1/* bnx2i_hwi.c: QLogic NetXtreme II iSCSI driver.
2 *
3 * Copyright (c) 2006 - 2013 Broadcom Corporation
4 * Copyright (c) 2007, 2008 Red Hat, Inc.  All rights reserved.
5 * Copyright (c) 2007, 2008 Mike Christie
6 * Copyright (c) 2014, QLogic Corporation
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation.
11 *
12 * Written by: Anil Veerabhadrappa (anilgv@broadcom.com)
13 * Previously Maintained by: Eddie Wai (eddie.wai@broadcom.com)
14 * Maintained by: QLogic-Storage-Upstream@qlogic.com
15 */
16
17#include <linux/gfp.h>
18#include <scsi/scsi_tcq.h>
19#include <scsi/libiscsi.h>
20#include "bnx2i.h"
21
22DECLARE_PER_CPU(struct bnx2i_percpu_s, bnx2i_percpu);
23
24/**
25 * bnx2i_get_cid_num - get cid from ep
26 * @ep: 	endpoint pointer
27 *
28 * Only applicable to 57710 family of devices
29 */
30static u32 bnx2i_get_cid_num(struct bnx2i_endpoint *ep)
31{
32	u32 cid;
33
34	if (test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type))
35		cid = ep->ep_cid;
36	else
37		cid = GET_CID_NUM(ep->ep_cid);
38	return cid;
39}
40
41
42/**
43 * bnx2i_adjust_qp_size - Adjust SQ/RQ/CQ size for 57710 device type
44 * @hba: 		Adapter for which adjustments is to be made
45 *
46 * Only applicable to 57710 family of devices
47 */
48static void bnx2i_adjust_qp_size(struct bnx2i_hba *hba)
49{
50	u32 num_elements_per_pg;
51
52	if (test_bit(BNX2I_NX2_DEV_5706, &hba->cnic_dev_type) ||
53	    test_bit(BNX2I_NX2_DEV_5708, &hba->cnic_dev_type) ||
54	    test_bit(BNX2I_NX2_DEV_5709, &hba->cnic_dev_type)) {
55		if (!is_power_of_2(hba->max_sqes))
56			hba->max_sqes = rounddown_pow_of_two(hba->max_sqes);
57
58		if (!is_power_of_2(hba->max_rqes))
59			hba->max_rqes = rounddown_pow_of_two(hba->max_rqes);
60	}
61
62	/* Adjust each queue size if the user selection does not
63	 * yield integral num of page buffers
64	 */
65	/* adjust SQ */
66	num_elements_per_pg = CNIC_PAGE_SIZE / BNX2I_SQ_WQE_SIZE;
67	if (hba->max_sqes < num_elements_per_pg)
68		hba->max_sqes = num_elements_per_pg;
69	else if (hba->max_sqes % num_elements_per_pg)
70		hba->max_sqes = (hba->max_sqes + num_elements_per_pg - 1) &
71				 ~(num_elements_per_pg - 1);
72
73	/* adjust CQ */
74	num_elements_per_pg = CNIC_PAGE_SIZE / BNX2I_CQE_SIZE;
75	if (hba->max_cqes < num_elements_per_pg)
76		hba->max_cqes = num_elements_per_pg;
77	else if (hba->max_cqes % num_elements_per_pg)
78		hba->max_cqes = (hba->max_cqes + num_elements_per_pg - 1) &
79				 ~(num_elements_per_pg - 1);
80
81	/* adjust RQ */
82	num_elements_per_pg = CNIC_PAGE_SIZE / BNX2I_RQ_WQE_SIZE;
83	if (hba->max_rqes < num_elements_per_pg)
84		hba->max_rqes = num_elements_per_pg;
85	else if (hba->max_rqes % num_elements_per_pg)
86		hba->max_rqes = (hba->max_rqes + num_elements_per_pg - 1) &
87				 ~(num_elements_per_pg - 1);
88}
89
90
91/**
92 * bnx2i_get_link_state - get network interface link state
93 * @hba:	adapter instance pointer
94 *
95 * updates adapter structure flag based on netdev state
96 */
97static void bnx2i_get_link_state(struct bnx2i_hba *hba)
98{
99	if (test_bit(__LINK_STATE_NOCARRIER, &hba->netdev->state))
100		set_bit(ADAPTER_STATE_LINK_DOWN, &hba->adapter_state);
101	else
102		clear_bit(ADAPTER_STATE_LINK_DOWN, &hba->adapter_state);
103}
104
105
106/**
107 * bnx2i_iscsi_license_error - displays iscsi license related error message
108 * @hba:		adapter instance pointer
109 * @error_code:		error classification
110 *
111 * Puts out an error log when driver is unable to offload iscsi connection
112 *	due to license restrictions
113 */
114static void bnx2i_iscsi_license_error(struct bnx2i_hba *hba, u32 error_code)
115{
116	if (error_code == ISCSI_KCQE_COMPLETION_STATUS_ISCSI_NOT_SUPPORTED)
117		/* iSCSI offload not supported on this device */
118		printk(KERN_ERR "bnx2i: iSCSI not supported, dev=%s\n",
119				hba->netdev->name);
120	if (error_code == ISCSI_KCQE_COMPLETION_STATUS_LOM_ISCSI_NOT_ENABLED)
121		/* iSCSI offload not supported on this LOM device */
122		printk(KERN_ERR "bnx2i: LOM is not enable to "
123				"offload iSCSI connections, dev=%s\n",
124				hba->netdev->name);
125	set_bit(ADAPTER_STATE_INIT_FAILED, &hba->adapter_state);
126}
127
128
129/**
130 * bnx2i_arm_cq_event_coalescing - arms CQ to enable EQ notification
131 * @ep:		endpoint (transport identifier) structure
132 * @action:	action, ARM or DISARM. For now only ARM_CQE is used
133 *
134 * Arm'ing CQ will enable chip to generate global EQ events inorder to interrupt
135 *	the driver. EQ event is generated CQ index is hit or at least 1 CQ is
136 *	outstanding and on chip timer expires
137 */
138int bnx2i_arm_cq_event_coalescing(struct bnx2i_endpoint *ep, u8 action)
139{
140	struct bnx2i_5771x_cq_db *cq_db;
141	u16 cq_index;
142	u16 next_index = 0;
143	u32 num_active_cmds;
144
145	/* Coalesce CQ entries only on 10G devices */
146	if (!test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type))
147		return 0;
148
149	/* Do not update CQ DB multiple times before firmware writes
150	 * '0xFFFF' to CQDB->SQN field. Deviation may cause spurious
151	 * interrupts and other unwanted results
152	 */
153	cq_db = (struct bnx2i_5771x_cq_db *) ep->qp.cq_pgtbl_virt;
154
155	if (action != CNIC_ARM_CQE_FP)
156		if (cq_db->sqn[0] && cq_db->sqn[0] != 0xFFFF)
157			return 0;
158
159	if (action == CNIC_ARM_CQE || action == CNIC_ARM_CQE_FP) {
160		num_active_cmds = atomic_read(&ep->num_active_cmds);
161		if (num_active_cmds <= event_coal_min)
162			next_index = 1;
163		else {
164			next_index = num_active_cmds >> ep->ec_shift;
165			if (next_index > num_active_cmds - event_coal_min)
166				next_index = num_active_cmds - event_coal_min;
167		}
168		if (!next_index)
169			next_index = 1;
170		cq_index = ep->qp.cqe_exp_seq_sn + next_index - 1;
171		if (cq_index > ep->qp.cqe_size * 2)
172			cq_index -= ep->qp.cqe_size * 2;
173		if (!cq_index)
174			cq_index = 1;
175
176		cq_db->sqn[0] = cq_index;
177	}
178	return next_index;
179}
180
181
182/**
183 * bnx2i_get_rq_buf - copy RQ buffer contents to driver buffer
184 * @conn:		iscsi connection on which RQ event occurred
185 * @ptr:		driver buffer to which RQ buffer contents is to
186 *			be copied
187 * @len:		length of valid data inside RQ buf
188 *
189 * Copies RQ buffer contents from shared (DMA'able) memory region to
190 *	driver buffer. RQ is used to DMA unsolicitated iscsi pdu's and
191 *	scsi sense info
192 */
193void bnx2i_get_rq_buf(struct bnx2i_conn *bnx2i_conn, char *ptr, int len)
194{
195	if (!bnx2i_conn->ep->qp.rqe_left)
196		return;
197
198	bnx2i_conn->ep->qp.rqe_left--;
199	memcpy(ptr, (u8 *) bnx2i_conn->ep->qp.rq_cons_qe, len);
200	if (bnx2i_conn->ep->qp.rq_cons_qe == bnx2i_conn->ep->qp.rq_last_qe) {
201		bnx2i_conn->ep->qp.rq_cons_qe = bnx2i_conn->ep->qp.rq_first_qe;
202		bnx2i_conn->ep->qp.rq_cons_idx = 0;
203	} else {
204		bnx2i_conn->ep->qp.rq_cons_qe++;
205		bnx2i_conn->ep->qp.rq_cons_idx++;
206	}
207}
208
209
210static void bnx2i_ring_577xx_doorbell(struct bnx2i_conn *conn)
211{
212	struct bnx2i_5771x_dbell dbell;
213	u32 msg;
214
215	memset(&dbell, 0, sizeof(dbell));
216	dbell.dbell.header = (B577XX_ISCSI_CONNECTION_TYPE <<
217			      B577XX_DOORBELL_HDR_CONN_TYPE_SHIFT);
218	msg = *((u32 *)&dbell);
219	/* TODO : get doorbell register mapping */
220	writel(cpu_to_le32(msg), conn->ep->qp.ctx_base);
221}
222
223
224/**
225 * bnx2i_put_rq_buf - Replenish RQ buffer, if required ring on chip doorbell
226 * @conn:	iscsi connection on which event to post
227 * @count:	number of RQ buffer being posted to chip
228 *
229 * No need to ring hardware doorbell for 57710 family of devices
230 */
231void bnx2i_put_rq_buf(struct bnx2i_conn *bnx2i_conn, int count)
232{
233	struct bnx2i_5771x_sq_rq_db *rq_db;
234	u16 hi_bit = (bnx2i_conn->ep->qp.rq_prod_idx & 0x8000);
235	struct bnx2i_endpoint *ep = bnx2i_conn->ep;
236
237	ep->qp.rqe_left += count;
238	ep->qp.rq_prod_idx &= 0x7FFF;
239	ep->qp.rq_prod_idx += count;
240
241	if (ep->qp.rq_prod_idx > bnx2i_conn->hba->max_rqes) {
242		ep->qp.rq_prod_idx %= bnx2i_conn->hba->max_rqes;
243		if (!hi_bit)
244			ep->qp.rq_prod_idx |= 0x8000;
245	} else
246		ep->qp.rq_prod_idx |= hi_bit;
247
248	if (test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type)) {
249		rq_db = (struct bnx2i_5771x_sq_rq_db *) ep->qp.rq_pgtbl_virt;
250		rq_db->prod_idx = ep->qp.rq_prod_idx;
251		/* no need to ring hardware doorbell for 57710 */
252	} else {
253		writew(ep->qp.rq_prod_idx,
254		       ep->qp.ctx_base + CNIC_RECV_DOORBELL);
255	}
256	mmiowb();
257}
258
259
260/**
261 * bnx2i_ring_sq_dbell - Ring SQ doorbell to wake-up the processing engine
262 * @conn: 		iscsi connection to which new SQ entries belong
263 * @count: 		number of SQ WQEs to post
264 *
265 * SQ DB is updated in host memory and TX Doorbell is rung for 57710 family
266 *	of devices. For 5706/5708/5709 new SQ WQE count is written into the
267 *	doorbell register
268 */
269static void bnx2i_ring_sq_dbell(struct bnx2i_conn *bnx2i_conn, int count)
270{
271	struct bnx2i_5771x_sq_rq_db *sq_db;
272	struct bnx2i_endpoint *ep = bnx2i_conn->ep;
273
274	atomic_inc(&ep->num_active_cmds);
275	wmb();	/* flush SQ WQE memory before the doorbell is rung */
276	if (test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type)) {
277		sq_db = (struct bnx2i_5771x_sq_rq_db *) ep->qp.sq_pgtbl_virt;
278		sq_db->prod_idx = ep->qp.sq_prod_idx;
279		bnx2i_ring_577xx_doorbell(bnx2i_conn);
280	} else
281		writew(count, ep->qp.ctx_base + CNIC_SEND_DOORBELL);
282
283	mmiowb(); /* flush posted PCI writes */
284}
285
286
287/**
288 * bnx2i_ring_dbell_update_sq_params - update SQ driver parameters
289 * @conn:	iscsi connection to which new SQ entries belong
290 * @count:	number of SQ WQEs to post
291 *
292 * this routine will update SQ driver parameters and ring the doorbell
293 */
294static void bnx2i_ring_dbell_update_sq_params(struct bnx2i_conn *bnx2i_conn,
295					      int count)
296{
297	int tmp_cnt;
298
299	if (count == 1) {
300		if (bnx2i_conn->ep->qp.sq_prod_qe ==
301		    bnx2i_conn->ep->qp.sq_last_qe)
302			bnx2i_conn->ep->qp.sq_prod_qe =
303						bnx2i_conn->ep->qp.sq_first_qe;
304		else
305			bnx2i_conn->ep->qp.sq_prod_qe++;
306	} else {
307		if ((bnx2i_conn->ep->qp.sq_prod_qe + count) <=
308		    bnx2i_conn->ep->qp.sq_last_qe)
309			bnx2i_conn->ep->qp.sq_prod_qe += count;
310		else {
311			tmp_cnt = bnx2i_conn->ep->qp.sq_last_qe -
312				bnx2i_conn->ep->qp.sq_prod_qe;
313			bnx2i_conn->ep->qp.sq_prod_qe =
314				&bnx2i_conn->ep->qp.sq_first_qe[count -
315								(tmp_cnt + 1)];
316		}
317	}
318	bnx2i_conn->ep->qp.sq_prod_idx += count;
319	/* Ring the doorbell */
320	bnx2i_ring_sq_dbell(bnx2i_conn, bnx2i_conn->ep->qp.sq_prod_idx);
321}
322
323
324/**
325 * bnx2i_send_iscsi_login - post iSCSI login request MP WQE to hardware
326 * @conn:	iscsi connection
327 * @cmd:	driver command structure which is requesting
328 *		a WQE to sent to chip for further processing
329 *
330 * prepare and post an iSCSI Login request WQE to CNIC firmware
331 */
332int bnx2i_send_iscsi_login(struct bnx2i_conn *bnx2i_conn,
333			   struct iscsi_task *task)
334{
335	struct bnx2i_cmd *bnx2i_cmd;
336	struct bnx2i_login_request *login_wqe;
337	struct iscsi_login_req *login_hdr;
338	u32 dword;
339
340	bnx2i_cmd = (struct bnx2i_cmd *)task->dd_data;
341	login_hdr = (struct iscsi_login_req *)task->hdr;
342	login_wqe = (struct bnx2i_login_request *)
343						bnx2i_conn->ep->qp.sq_prod_qe;
344
345	login_wqe->op_code = login_hdr->opcode;
346	login_wqe->op_attr = login_hdr->flags;
347	login_wqe->version_max = login_hdr->max_version;
348	login_wqe->version_min = login_hdr->min_version;
349	login_wqe->data_length = ntoh24(login_hdr->dlength);
350	login_wqe->isid_lo = *((u32 *) login_hdr->isid);
351	login_wqe->isid_hi = *((u16 *) login_hdr->isid + 2);
352	login_wqe->tsih = login_hdr->tsih;
353	login_wqe->itt = task->itt |
354		(ISCSI_TASK_TYPE_MPATH << ISCSI_LOGIN_REQUEST_TYPE_SHIFT);
355	login_wqe->cid = login_hdr->cid;
356
357	login_wqe->cmd_sn = be32_to_cpu(login_hdr->cmdsn);
358	login_wqe->exp_stat_sn = be32_to_cpu(login_hdr->exp_statsn);
359	login_wqe->flags = ISCSI_LOGIN_REQUEST_UPDATE_EXP_STAT_SN;
360
361	login_wqe->resp_bd_list_addr_lo = (u32) bnx2i_conn->gen_pdu.resp_bd_dma;
362	login_wqe->resp_bd_list_addr_hi =
363		(u32) ((u64) bnx2i_conn->gen_pdu.resp_bd_dma >> 32);
364
365	dword = ((1 << ISCSI_LOGIN_REQUEST_NUM_RESP_BDS_SHIFT) |
366		 (bnx2i_conn->gen_pdu.resp_buf_size <<
367		  ISCSI_LOGIN_REQUEST_RESP_BUFFER_LENGTH_SHIFT));
368	login_wqe->resp_buffer = dword;
369	login_wqe->bd_list_addr_lo = (u32) bnx2i_conn->gen_pdu.req_bd_dma;
370	login_wqe->bd_list_addr_hi =
371		(u32) ((u64) bnx2i_conn->gen_pdu.req_bd_dma >> 32);
372	login_wqe->num_bds = 1;
373	login_wqe->cq_index = 0; /* CQ# used for completion, 5771x only */
374
375	bnx2i_ring_dbell_update_sq_params(bnx2i_conn, 1);
376	return 0;
377}
378
379/**
380 * bnx2i_send_iscsi_tmf - post iSCSI task management request MP WQE to hardware
381 * @conn:	iscsi connection
382 * @mtask:	driver command structure which is requesting
383 *		a WQE to sent to chip for further processing
384 *
385 * prepare and post an iSCSI Login request WQE to CNIC firmware
386 */
387int bnx2i_send_iscsi_tmf(struct bnx2i_conn *bnx2i_conn,
388			 struct iscsi_task *mtask)
389{
390	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
391	struct iscsi_tm *tmfabort_hdr;
392	struct scsi_cmnd *ref_sc;
393	struct iscsi_task *ctask;
394	struct bnx2i_cmd *bnx2i_cmd;
395	struct bnx2i_tmf_request *tmfabort_wqe;
396	u32 dword;
397	u32 scsi_lun[2];
398
399	bnx2i_cmd = (struct bnx2i_cmd *)mtask->dd_data;
400	tmfabort_hdr = (struct iscsi_tm *)mtask->hdr;
401	tmfabort_wqe = (struct bnx2i_tmf_request *)
402						bnx2i_conn->ep->qp.sq_prod_qe;
403
404	tmfabort_wqe->op_code = tmfabort_hdr->opcode;
405	tmfabort_wqe->op_attr = tmfabort_hdr->flags;
406
407	tmfabort_wqe->itt = (mtask->itt | (ISCSI_TASK_TYPE_MPATH << 14));
408	tmfabort_wqe->reserved2 = 0;
409	tmfabort_wqe->cmd_sn = be32_to_cpu(tmfabort_hdr->cmdsn);
410
411	switch (tmfabort_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) {
412	case ISCSI_TM_FUNC_ABORT_TASK:
413	case ISCSI_TM_FUNC_TASK_REASSIGN:
414		ctask = iscsi_itt_to_task(conn, tmfabort_hdr->rtt);
415		if (!ctask || !ctask->sc)
416			/*
417			 * the iscsi layer must have completed the cmd while
418			 * was starting up.
419			 *
420			 * Note: In the case of a SCSI cmd timeout, the task's
421			 *       sc is still active; hence ctask->sc != 0
422			 *       In this case, the task must be aborted
423			 */
424			return 0;
425
426		ref_sc = ctask->sc;
427		if (ref_sc->sc_data_direction == DMA_TO_DEVICE)
428			dword = (ISCSI_TASK_TYPE_WRITE <<
429				 ISCSI_CMD_REQUEST_TYPE_SHIFT);
430		else
431			dword = (ISCSI_TASK_TYPE_READ <<
432				 ISCSI_CMD_REQUEST_TYPE_SHIFT);
433		tmfabort_wqe->ref_itt = (dword |
434					(tmfabort_hdr->rtt & ISCSI_ITT_MASK));
435		break;
436	default:
437		tmfabort_wqe->ref_itt = RESERVED_ITT;
438	}
439	memcpy(scsi_lun, &tmfabort_hdr->lun, sizeof(struct scsi_lun));
440	tmfabort_wqe->lun[0] = be32_to_cpu(scsi_lun[0]);
441	tmfabort_wqe->lun[1] = be32_to_cpu(scsi_lun[1]);
442
443	tmfabort_wqe->ref_cmd_sn = be32_to_cpu(tmfabort_hdr->refcmdsn);
444
445	tmfabort_wqe->bd_list_addr_lo = (u32) bnx2i_conn->hba->mp_bd_dma;
446	tmfabort_wqe->bd_list_addr_hi = (u32)
447				((u64) bnx2i_conn->hba->mp_bd_dma >> 32);
448	tmfabort_wqe->num_bds = 1;
449	tmfabort_wqe->cq_index = 0; /* CQ# used for completion, 5771x only */
450
451	bnx2i_ring_dbell_update_sq_params(bnx2i_conn, 1);
452	return 0;
453}
454
455/**
456 * bnx2i_send_iscsi_text - post iSCSI text WQE to hardware
457 * @conn:	iscsi connection
458 * @mtask:	driver command structure which is requesting
459 *		a WQE to sent to chip for further processing
460 *
461 * prepare and post an iSCSI Text request WQE to CNIC firmware
462 */
463int bnx2i_send_iscsi_text(struct bnx2i_conn *bnx2i_conn,
464			  struct iscsi_task *mtask)
465{
466	struct bnx2i_cmd *bnx2i_cmd;
467	struct bnx2i_text_request *text_wqe;
468	struct iscsi_text *text_hdr;
469	u32 dword;
470
471	bnx2i_cmd = (struct bnx2i_cmd *)mtask->dd_data;
472	text_hdr = (struct iscsi_text *)mtask->hdr;
473	text_wqe = (struct bnx2i_text_request *) bnx2i_conn->ep->qp.sq_prod_qe;
474
475	memset(text_wqe, 0, sizeof(struct bnx2i_text_request));
476
477	text_wqe->op_code = text_hdr->opcode;
478	text_wqe->op_attr = text_hdr->flags;
479	text_wqe->data_length = ntoh24(text_hdr->dlength);
480	text_wqe->itt = mtask->itt |
481		(ISCSI_TASK_TYPE_MPATH << ISCSI_TEXT_REQUEST_TYPE_SHIFT);
482	text_wqe->ttt = be32_to_cpu(text_hdr->ttt);
483
484	text_wqe->cmd_sn = be32_to_cpu(text_hdr->cmdsn);
485
486	text_wqe->resp_bd_list_addr_lo = (u32) bnx2i_conn->gen_pdu.resp_bd_dma;
487	text_wqe->resp_bd_list_addr_hi =
488			(u32) ((u64) bnx2i_conn->gen_pdu.resp_bd_dma >> 32);
489
490	dword = ((1 << ISCSI_TEXT_REQUEST_NUM_RESP_BDS_SHIFT) |
491		 (bnx2i_conn->gen_pdu.resp_buf_size <<
492		  ISCSI_TEXT_REQUEST_RESP_BUFFER_LENGTH_SHIFT));
493	text_wqe->resp_buffer = dword;
494	text_wqe->bd_list_addr_lo = (u32) bnx2i_conn->gen_pdu.req_bd_dma;
495	text_wqe->bd_list_addr_hi =
496			(u32) ((u64) bnx2i_conn->gen_pdu.req_bd_dma >> 32);
497	text_wqe->num_bds = 1;
498	text_wqe->cq_index = 0; /* CQ# used for completion, 5771x only */
499
500	bnx2i_ring_dbell_update_sq_params(bnx2i_conn, 1);
501	return 0;
502}
503
504
505/**
506 * bnx2i_send_iscsi_scsicmd - post iSCSI scsicmd request WQE to hardware
507 * @conn:	iscsi connection
508 * @cmd:	driver command structure which is requesting
509 *		a WQE to sent to chip for further processing
510 *
511 * prepare and post an iSCSI SCSI-CMD request WQE to CNIC firmware
512 */
513int bnx2i_send_iscsi_scsicmd(struct bnx2i_conn *bnx2i_conn,
514			     struct bnx2i_cmd *cmd)
515{
516	struct bnx2i_cmd_request *scsi_cmd_wqe;
517
518	scsi_cmd_wqe = (struct bnx2i_cmd_request *)
519						bnx2i_conn->ep->qp.sq_prod_qe;
520	memcpy(scsi_cmd_wqe, &cmd->req, sizeof(struct bnx2i_cmd_request));
521	scsi_cmd_wqe->cq_index = 0; /* CQ# used for completion, 5771x only */
522
523	bnx2i_ring_dbell_update_sq_params(bnx2i_conn, 1);
524	return 0;
525}
526
527/**
528 * bnx2i_send_iscsi_nopout - post iSCSI NOPOUT request WQE to hardware
529 * @conn:		iscsi connection
530 * @cmd:		driver command structure which is requesting
531 *			a WQE to sent to chip for further processing
532 * @datap:		payload buffer pointer
533 * @data_len:		payload data length
534 * @unsol:		indicated whether nopout pdu is unsolicited pdu or
535 *			in response to target's NOPIN w/ TTT != FFFFFFFF
536 *
537 * prepare and post a nopout request WQE to CNIC firmware
538 */
539int bnx2i_send_iscsi_nopout(struct bnx2i_conn *bnx2i_conn,
540			    struct iscsi_task *task,
541			    char *datap, int data_len, int unsol)
542{
543	struct bnx2i_endpoint *ep = bnx2i_conn->ep;
544	struct bnx2i_cmd *bnx2i_cmd;
545	struct bnx2i_nop_out_request *nopout_wqe;
546	struct iscsi_nopout *nopout_hdr;
547
548	bnx2i_cmd = (struct bnx2i_cmd *)task->dd_data;
549	nopout_hdr = (struct iscsi_nopout *)task->hdr;
550	nopout_wqe = (struct bnx2i_nop_out_request *)ep->qp.sq_prod_qe;
551
552	memset(nopout_wqe, 0x00, sizeof(struct bnx2i_nop_out_request));
553
554	nopout_wqe->op_code = nopout_hdr->opcode;
555	nopout_wqe->op_attr = ISCSI_FLAG_CMD_FINAL;
556	memcpy(nopout_wqe->lun, &nopout_hdr->lun, 8);
557
558	if (test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type)) {
559		u32 tmp = nopout_wqe->lun[0];
560		/* 57710 requires LUN field to be swapped */
561		nopout_wqe->lun[0] = nopout_wqe->lun[1];
562		nopout_wqe->lun[1] = tmp;
563	}
564
565	nopout_wqe->itt = ((u16)task->itt |
566			   (ISCSI_TASK_TYPE_MPATH <<
567			    ISCSI_TMF_REQUEST_TYPE_SHIFT));
568	nopout_wqe->ttt = be32_to_cpu(nopout_hdr->ttt);
569	nopout_wqe->flags = 0;
570	if (!unsol)
571		nopout_wqe->flags = ISCSI_NOP_OUT_REQUEST_LOCAL_COMPLETION;
572	else if (nopout_hdr->itt == RESERVED_ITT)
573		nopout_wqe->flags = ISCSI_NOP_OUT_REQUEST_LOCAL_COMPLETION;
574
575	nopout_wqe->cmd_sn = be32_to_cpu(nopout_hdr->cmdsn);
576	nopout_wqe->data_length = data_len;
577	if (data_len) {
578		/* handle payload data, not required in first release */
579		printk(KERN_ALERT "NOPOUT: WARNING!! payload len != 0\n");
580	} else {
581		nopout_wqe->bd_list_addr_lo = (u32)
582					bnx2i_conn->hba->mp_bd_dma;
583		nopout_wqe->bd_list_addr_hi =
584			(u32) ((u64) bnx2i_conn->hba->mp_bd_dma >> 32);
585		nopout_wqe->num_bds = 1;
586	}
587	nopout_wqe->cq_index = 0; /* CQ# used for completion, 5771x only */
588
589	bnx2i_ring_dbell_update_sq_params(bnx2i_conn, 1);
590	return 0;
591}
592
593
594/**
595 * bnx2i_send_iscsi_logout - post iSCSI logout request WQE to hardware
596 * @conn:	iscsi connection
597 * @cmd:	driver command structure which is requesting
598 *		a WQE to sent to chip for further processing
599 *
600 * prepare and post logout request WQE to CNIC firmware
601 */
602int bnx2i_send_iscsi_logout(struct bnx2i_conn *bnx2i_conn,
603			    struct iscsi_task *task)
604{
605	struct bnx2i_cmd *bnx2i_cmd;
606	struct bnx2i_logout_request *logout_wqe;
607	struct iscsi_logout *logout_hdr;
608
609	bnx2i_cmd = (struct bnx2i_cmd *)task->dd_data;
610	logout_hdr = (struct iscsi_logout *)task->hdr;
611
612	logout_wqe = (struct bnx2i_logout_request *)
613						bnx2i_conn->ep->qp.sq_prod_qe;
614	memset(logout_wqe, 0x00, sizeof(struct bnx2i_logout_request));
615
616	logout_wqe->op_code = logout_hdr->opcode;
617	logout_wqe->cmd_sn = be32_to_cpu(logout_hdr->cmdsn);
618	logout_wqe->op_attr =
619			logout_hdr->flags | ISCSI_LOGOUT_REQUEST_ALWAYS_ONE;
620	logout_wqe->itt = ((u16)task->itt |
621			   (ISCSI_TASK_TYPE_MPATH <<
622			    ISCSI_LOGOUT_REQUEST_TYPE_SHIFT));
623	logout_wqe->data_length = 0;
624	logout_wqe->cid = 0;
625
626	logout_wqe->bd_list_addr_lo = (u32) bnx2i_conn->hba->mp_bd_dma;
627	logout_wqe->bd_list_addr_hi = (u32)
628				((u64) bnx2i_conn->hba->mp_bd_dma >> 32);
629	logout_wqe->num_bds = 1;
630	logout_wqe->cq_index = 0; /* CQ# used for completion, 5771x only */
631
632	bnx2i_conn->ep->state = EP_STATE_LOGOUT_SENT;
633
634	bnx2i_ring_dbell_update_sq_params(bnx2i_conn, 1);
635	return 0;
636}
637
638
639/**
640 * bnx2i_update_iscsi_conn - post iSCSI logout request WQE to hardware
641 * @conn:	iscsi connection which requires iscsi parameter update
642 *
643 * sends down iSCSI Conn Update request to move iSCSI conn to FFP
644 */
645void bnx2i_update_iscsi_conn(struct iscsi_conn *conn)
646{
647	struct bnx2i_conn *bnx2i_conn = conn->dd_data;
648	struct bnx2i_hba *hba = bnx2i_conn->hba;
649	struct kwqe *kwqe_arr[2];
650	struct iscsi_kwqe_conn_update *update_wqe;
651	struct iscsi_kwqe_conn_update conn_update_kwqe;
652
653	update_wqe = &conn_update_kwqe;
654
655	update_wqe->hdr.op_code = ISCSI_KWQE_OPCODE_UPDATE_CONN;
656	update_wqe->hdr.flags =
657		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
658
659	/* 5771x requires conn context id to be passed as is */
660	if (test_bit(BNX2I_NX2_DEV_57710, &bnx2i_conn->ep->hba->cnic_dev_type))
661		update_wqe->context_id = bnx2i_conn->ep->ep_cid;
662	else
663		update_wqe->context_id = (bnx2i_conn->ep->ep_cid >> 7);
664	update_wqe->conn_flags = 0;
665	if (conn->hdrdgst_en)
666		update_wqe->conn_flags |= ISCSI_KWQE_CONN_UPDATE_HEADER_DIGEST;
667	if (conn->datadgst_en)
668		update_wqe->conn_flags |= ISCSI_KWQE_CONN_UPDATE_DATA_DIGEST;
669	if (conn->session->initial_r2t_en)
670		update_wqe->conn_flags |= ISCSI_KWQE_CONN_UPDATE_INITIAL_R2T;
671	if (conn->session->imm_data_en)
672		update_wqe->conn_flags |= ISCSI_KWQE_CONN_UPDATE_IMMEDIATE_DATA;
673
674	update_wqe->max_send_pdu_length = conn->max_xmit_dlength;
675	update_wqe->max_recv_pdu_length = conn->max_recv_dlength;
676	update_wqe->first_burst_length = conn->session->first_burst;
677	update_wqe->max_burst_length = conn->session->max_burst;
678	update_wqe->exp_stat_sn = conn->exp_statsn;
679	update_wqe->max_outstanding_r2ts = conn->session->max_r2t;
680	update_wqe->session_error_recovery_level = conn->session->erl;
681	iscsi_conn_printk(KERN_ALERT, conn,
682			  "bnx2i: conn update - MBL 0x%x FBL 0x%x"
683			  "MRDSL_I 0x%x MRDSL_T 0x%x \n",
684			  update_wqe->max_burst_length,
685			  update_wqe->first_burst_length,
686			  update_wqe->max_recv_pdu_length,
687			  update_wqe->max_send_pdu_length);
688
689	kwqe_arr[0] = (struct kwqe *) update_wqe;
690	if (hba->cnic && hba->cnic->submit_kwqes)
691		hba->cnic->submit_kwqes(hba->cnic, kwqe_arr, 1);
692}
693
694
695/**
696 * bnx2i_ep_ofld_timer - post iSCSI logout request WQE to hardware
697 * @data:	endpoint (transport handle) structure pointer
698 *
699 * routine to handle connection offload/destroy request timeout
700 */
701void bnx2i_ep_ofld_timer(unsigned long data)
702{
703	struct bnx2i_endpoint *ep = (struct bnx2i_endpoint *) data;
704
705	if (ep->state == EP_STATE_OFLD_START) {
706		printk(KERN_ALERT "ofld_timer: CONN_OFLD timeout\n");
707		ep->state = EP_STATE_OFLD_FAILED;
708	} else if (ep->state == EP_STATE_DISCONN_START) {
709		printk(KERN_ALERT "ofld_timer: CONN_DISCON timeout\n");
710		ep->state = EP_STATE_DISCONN_TIMEDOUT;
711	} else if (ep->state == EP_STATE_CLEANUP_START) {
712		printk(KERN_ALERT "ofld_timer: CONN_CLEANUP timeout\n");
713		ep->state = EP_STATE_CLEANUP_FAILED;
714	}
715
716	wake_up_interruptible(&ep->ofld_wait);
717}
718
719
720static int bnx2i_power_of2(u32 val)
721{
722	u32 power = 0;
723	if (val & (val - 1))
724		return power;
725	val--;
726	while (val) {
727		val = val >> 1;
728		power++;
729	}
730	return power;
731}
732
733
734/**
735 * bnx2i_send_cmd_cleanup_req - send iscsi cmd context clean-up request
736 * @hba:	adapter structure pointer
737 * @cmd:	driver command structure which is requesting
738 *		a WQE to sent to chip for further processing
739 *
740 * prepares and posts CONN_OFLD_REQ1/2 KWQE
741 */
742void bnx2i_send_cmd_cleanup_req(struct bnx2i_hba *hba, struct bnx2i_cmd *cmd)
743{
744	struct bnx2i_cleanup_request *cmd_cleanup;
745
746	cmd_cleanup =
747		(struct bnx2i_cleanup_request *)cmd->conn->ep->qp.sq_prod_qe;
748	memset(cmd_cleanup, 0x00, sizeof(struct bnx2i_cleanup_request));
749
750	cmd_cleanup->op_code = ISCSI_OPCODE_CLEANUP_REQUEST;
751	cmd_cleanup->itt = cmd->req.itt;
752	cmd_cleanup->cq_index = 0; /* CQ# used for completion, 5771x only */
753
754	bnx2i_ring_dbell_update_sq_params(cmd->conn, 1);
755}
756
757
758/**
759 * bnx2i_send_conn_destroy - initiates iscsi connection teardown process
760 * @hba:	adapter structure pointer
761 * @ep:		endpoint (transport identifier) structure
762 *
763 * this routine prepares and posts CONN_OFLD_REQ1/2 KWQE to initiate
764 * 	iscsi connection context clean-up process
765 */
766int bnx2i_send_conn_destroy(struct bnx2i_hba *hba, struct bnx2i_endpoint *ep)
767{
768	struct kwqe *kwqe_arr[2];
769	struct iscsi_kwqe_conn_destroy conn_cleanup;
770	int rc = -EINVAL;
771
772	memset(&conn_cleanup, 0x00, sizeof(struct iscsi_kwqe_conn_destroy));
773
774	conn_cleanup.hdr.op_code = ISCSI_KWQE_OPCODE_DESTROY_CONN;
775	conn_cleanup.hdr.flags =
776		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
777	/* 5771x requires conn context id to be passed as is */
778	if (test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type))
779		conn_cleanup.context_id = ep->ep_cid;
780	else
781		conn_cleanup.context_id = (ep->ep_cid >> 7);
782
783	conn_cleanup.reserved0 = (u16)ep->ep_iscsi_cid;
784
785	kwqe_arr[0] = (struct kwqe *) &conn_cleanup;
786	if (hba->cnic && hba->cnic->submit_kwqes)
787		rc = hba->cnic->submit_kwqes(hba->cnic, kwqe_arr, 1);
788
789	return rc;
790}
791
792
793/**
794 * bnx2i_570x_send_conn_ofld_req - initiates iscsi conn context setup process
795 * @hba: 		adapter structure pointer
796 * @ep: 		endpoint (transport identifier) structure
797 *
798 * 5706/5708/5709 specific - prepares and posts CONN_OFLD_REQ1/2 KWQE
799 */
800static int bnx2i_570x_send_conn_ofld_req(struct bnx2i_hba *hba,
801					 struct bnx2i_endpoint *ep)
802{
803	struct kwqe *kwqe_arr[2];
804	struct iscsi_kwqe_conn_offload1 ofld_req1;
805	struct iscsi_kwqe_conn_offload2 ofld_req2;
806	dma_addr_t dma_addr;
807	int num_kwqes = 2;
808	u32 *ptbl;
809	int rc = -EINVAL;
810
811	ofld_req1.hdr.op_code = ISCSI_KWQE_OPCODE_OFFLOAD_CONN1;
812	ofld_req1.hdr.flags =
813		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
814
815	ofld_req1.iscsi_conn_id = (u16) ep->ep_iscsi_cid;
816
817	dma_addr = ep->qp.sq_pgtbl_phys;
818	ofld_req1.sq_page_table_addr_lo = (u32) dma_addr;
819	ofld_req1.sq_page_table_addr_hi = (u32) ((u64) dma_addr >> 32);
820
821	dma_addr = ep->qp.cq_pgtbl_phys;
822	ofld_req1.cq_page_table_addr_lo = (u32) dma_addr;
823	ofld_req1.cq_page_table_addr_hi = (u32) ((u64) dma_addr >> 32);
824
825	ofld_req2.hdr.op_code = ISCSI_KWQE_OPCODE_OFFLOAD_CONN2;
826	ofld_req2.hdr.flags =
827		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
828
829	dma_addr = ep->qp.rq_pgtbl_phys;
830	ofld_req2.rq_page_table_addr_lo = (u32) dma_addr;
831	ofld_req2.rq_page_table_addr_hi = (u32) ((u64) dma_addr >> 32);
832
833	ptbl = (u32 *) ep->qp.sq_pgtbl_virt;
834
835	ofld_req2.sq_first_pte.hi = *ptbl++;
836	ofld_req2.sq_first_pte.lo = *ptbl;
837
838	ptbl = (u32 *) ep->qp.cq_pgtbl_virt;
839	ofld_req2.cq_first_pte.hi = *ptbl++;
840	ofld_req2.cq_first_pte.lo = *ptbl;
841
842	kwqe_arr[0] = (struct kwqe *) &ofld_req1;
843	kwqe_arr[1] = (struct kwqe *) &ofld_req2;
844	ofld_req2.num_additional_wqes = 0;
845
846	if (hba->cnic && hba->cnic->submit_kwqes)
847		rc = hba->cnic->submit_kwqes(hba->cnic, kwqe_arr, num_kwqes);
848
849	return rc;
850}
851
852
853/**
854 * bnx2i_5771x_send_conn_ofld_req - initiates iscsi connection context creation
855 * @hba: 		adapter structure pointer
856 * @ep: 		endpoint (transport identifier) structure
857 *
858 * 57710 specific - prepares and posts CONN_OFLD_REQ1/2 KWQE
859 */
860static int bnx2i_5771x_send_conn_ofld_req(struct bnx2i_hba *hba,
861					  struct bnx2i_endpoint *ep)
862{
863	struct kwqe *kwqe_arr[5];
864	struct iscsi_kwqe_conn_offload1 ofld_req1;
865	struct iscsi_kwqe_conn_offload2 ofld_req2;
866	struct iscsi_kwqe_conn_offload3 ofld_req3[1];
867	dma_addr_t dma_addr;
868	int num_kwqes = 2;
869	u32 *ptbl;
870	int rc = -EINVAL;
871
872	ofld_req1.hdr.op_code = ISCSI_KWQE_OPCODE_OFFLOAD_CONN1;
873	ofld_req1.hdr.flags =
874		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
875
876	ofld_req1.iscsi_conn_id = (u16) ep->ep_iscsi_cid;
877
878	dma_addr = ep->qp.sq_pgtbl_phys + ISCSI_SQ_DB_SIZE;
879	ofld_req1.sq_page_table_addr_lo = (u32) dma_addr;
880	ofld_req1.sq_page_table_addr_hi = (u32) ((u64) dma_addr >> 32);
881
882	dma_addr = ep->qp.cq_pgtbl_phys + ISCSI_CQ_DB_SIZE;
883	ofld_req1.cq_page_table_addr_lo = (u32) dma_addr;
884	ofld_req1.cq_page_table_addr_hi = (u32) ((u64) dma_addr >> 32);
885
886	ofld_req2.hdr.op_code = ISCSI_KWQE_OPCODE_OFFLOAD_CONN2;
887	ofld_req2.hdr.flags =
888		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
889
890	dma_addr = ep->qp.rq_pgtbl_phys + ISCSI_RQ_DB_SIZE;
891	ofld_req2.rq_page_table_addr_lo = (u32) dma_addr;
892	ofld_req2.rq_page_table_addr_hi = (u32) ((u64) dma_addr >> 32);
893
894	ptbl = (u32 *)((u8 *)ep->qp.sq_pgtbl_virt + ISCSI_SQ_DB_SIZE);
895	ofld_req2.sq_first_pte.hi = *ptbl++;
896	ofld_req2.sq_first_pte.lo = *ptbl;
897
898	ptbl = (u32 *)((u8 *)ep->qp.cq_pgtbl_virt + ISCSI_CQ_DB_SIZE);
899	ofld_req2.cq_first_pte.hi = *ptbl++;
900	ofld_req2.cq_first_pte.lo = *ptbl;
901
902	kwqe_arr[0] = (struct kwqe *) &ofld_req1;
903	kwqe_arr[1] = (struct kwqe *) &ofld_req2;
904
905	ofld_req2.num_additional_wqes = 1;
906	memset(ofld_req3, 0x00, sizeof(ofld_req3[0]));
907	ptbl = (u32 *)((u8 *)ep->qp.rq_pgtbl_virt + ISCSI_RQ_DB_SIZE);
908	ofld_req3[0].qp_first_pte[0].hi = *ptbl++;
909	ofld_req3[0].qp_first_pte[0].lo = *ptbl;
910
911	kwqe_arr[2] = (struct kwqe *) ofld_req3;
912	/* need if we decide to go with multiple KCQE's per conn */
913	num_kwqes += 1;
914
915	if (hba->cnic && hba->cnic->submit_kwqes)
916		rc = hba->cnic->submit_kwqes(hba->cnic, kwqe_arr, num_kwqes);
917
918	return rc;
919}
920
921/**
922 * bnx2i_send_conn_ofld_req - initiates iscsi connection context setup process
923 *
924 * @hba: 		adapter structure pointer
925 * @ep: 		endpoint (transport identifier) structure
926 *
927 * this routine prepares and posts CONN_OFLD_REQ1/2 KWQE
928 */
929int bnx2i_send_conn_ofld_req(struct bnx2i_hba *hba, struct bnx2i_endpoint *ep)
930{
931	int rc;
932
933	if (test_bit(BNX2I_NX2_DEV_57710, &hba->cnic_dev_type))
934		rc = bnx2i_5771x_send_conn_ofld_req(hba, ep);
935	else
936		rc = bnx2i_570x_send_conn_ofld_req(hba, ep);
937
938	return rc;
939}
940
941
942/**
943 * setup_qp_page_tables - iscsi QP page table setup function
944 * @ep:		endpoint (transport identifier) structure
945 *
946 * Sets up page tables for SQ/RQ/CQ, 1G/sec (5706/5708/5709) devices requires
947 * 	64-bit address in big endian format. Whereas 10G/sec (57710) requires
948 * 	PT in little endian format
949 */
950static void setup_qp_page_tables(struct bnx2i_endpoint *ep)
951{
952	int num_pages;
953	u32 *ptbl;
954	dma_addr_t page;
955	int cnic_dev_10g;
956
957	if (test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type))
958		cnic_dev_10g = 1;
959	else
960		cnic_dev_10g = 0;
961
962	/* SQ page table */
963	memset(ep->qp.sq_pgtbl_virt, 0, ep->qp.sq_pgtbl_size);
964	num_pages = ep->qp.sq_mem_size / CNIC_PAGE_SIZE;
965	page = ep->qp.sq_phys;
966
967	if (cnic_dev_10g)
968		ptbl = (u32 *)((u8 *)ep->qp.sq_pgtbl_virt + ISCSI_SQ_DB_SIZE);
969	else
970		ptbl = (u32 *) ep->qp.sq_pgtbl_virt;
971	while (num_pages--) {
972		if (cnic_dev_10g) {
973			/* PTE is written in little endian format for 57710 */
974			*ptbl = (u32) page;
975			ptbl++;
976			*ptbl = (u32) ((u64) page >> 32);
977			ptbl++;
978			page += CNIC_PAGE_SIZE;
979		} else {
980			/* PTE is written in big endian format for
981			 * 5706/5708/5709 devices */
982			*ptbl = (u32) ((u64) page >> 32);
983			ptbl++;
984			*ptbl = (u32) page;
985			ptbl++;
986			page += CNIC_PAGE_SIZE;
987		}
988	}
989
990	/* RQ page table */
991	memset(ep->qp.rq_pgtbl_virt, 0, ep->qp.rq_pgtbl_size);
992	num_pages = ep->qp.rq_mem_size / CNIC_PAGE_SIZE;
993	page = ep->qp.rq_phys;
994
995	if (cnic_dev_10g)
996		ptbl = (u32 *)((u8 *)ep->qp.rq_pgtbl_virt + ISCSI_RQ_DB_SIZE);
997	else
998		ptbl = (u32 *) ep->qp.rq_pgtbl_virt;
999	while (num_pages--) {
1000		if (cnic_dev_10g) {
1001			/* PTE is written in little endian format for 57710 */
1002			*ptbl = (u32) page;
1003			ptbl++;
1004			*ptbl = (u32) ((u64) page >> 32);
1005			ptbl++;
1006			page += CNIC_PAGE_SIZE;
1007		} else {
1008			/* PTE is written in big endian format for
1009			 * 5706/5708/5709 devices */
1010			*ptbl = (u32) ((u64) page >> 32);
1011			ptbl++;
1012			*ptbl = (u32) page;
1013			ptbl++;
1014			page += CNIC_PAGE_SIZE;
1015		}
1016	}
1017
1018	/* CQ page table */
1019	memset(ep->qp.cq_pgtbl_virt, 0, ep->qp.cq_pgtbl_size);
1020	num_pages = ep->qp.cq_mem_size / CNIC_PAGE_SIZE;
1021	page = ep->qp.cq_phys;
1022
1023	if (cnic_dev_10g)
1024		ptbl = (u32 *)((u8 *)ep->qp.cq_pgtbl_virt + ISCSI_CQ_DB_SIZE);
1025	else
1026		ptbl = (u32 *) ep->qp.cq_pgtbl_virt;
1027	while (num_pages--) {
1028		if (cnic_dev_10g) {
1029			/* PTE is written in little endian format for 57710 */
1030			*ptbl = (u32) page;
1031			ptbl++;
1032			*ptbl = (u32) ((u64) page >> 32);
1033			ptbl++;
1034			page += CNIC_PAGE_SIZE;
1035		} else {
1036			/* PTE is written in big endian format for
1037			 * 5706/5708/5709 devices */
1038			*ptbl = (u32) ((u64) page >> 32);
1039			ptbl++;
1040			*ptbl = (u32) page;
1041			ptbl++;
1042			page += CNIC_PAGE_SIZE;
1043		}
1044	}
1045}
1046
1047
1048/**
1049 * bnx2i_alloc_qp_resc - allocates required resources for QP.
1050 * @hba:	adapter structure pointer
1051 * @ep:		endpoint (transport identifier) structure
1052 *
1053 * Allocate QP (transport layer for iSCSI connection) resources, DMA'able
1054 *	memory for SQ/RQ/CQ and page tables. EP structure elements such
1055 *	as producer/consumer indexes/pointers, queue sizes and page table
1056 *	contents are setup
1057 */
1058int bnx2i_alloc_qp_resc(struct bnx2i_hba *hba, struct bnx2i_endpoint *ep)
1059{
1060	struct bnx2i_5771x_cq_db *cq_db;
1061
1062	ep->hba = hba;
1063	ep->conn = NULL;
1064	ep->ep_cid = ep->ep_iscsi_cid = ep->ep_pg_cid = 0;
1065
1066	/* Allocate page table memory for SQ which is page aligned */
1067	ep->qp.sq_mem_size = hba->max_sqes * BNX2I_SQ_WQE_SIZE;
1068	ep->qp.sq_mem_size =
1069		(ep->qp.sq_mem_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK;
1070	ep->qp.sq_pgtbl_size =
1071		(ep->qp.sq_mem_size / CNIC_PAGE_SIZE) * sizeof(void *);
1072	ep->qp.sq_pgtbl_size =
1073		(ep->qp.sq_pgtbl_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK;
1074
1075	ep->qp.sq_pgtbl_virt =
1076		dma_alloc_coherent(&hba->pcidev->dev, ep->qp.sq_pgtbl_size,
1077				   &ep->qp.sq_pgtbl_phys, GFP_KERNEL);
1078	if (!ep->qp.sq_pgtbl_virt) {
1079		printk(KERN_ALERT "bnx2i: unable to alloc SQ PT mem (%d)\n",
1080				  ep->qp.sq_pgtbl_size);
1081		goto mem_alloc_err;
1082	}
1083
1084	/* Allocate memory area for actual SQ element */
1085	ep->qp.sq_virt =
1086		dma_alloc_coherent(&hba->pcidev->dev, ep->qp.sq_mem_size,
1087				   &ep->qp.sq_phys, GFP_KERNEL);
1088	if (!ep->qp.sq_virt) {
1089		printk(KERN_ALERT "bnx2i: unable to alloc SQ BD memory %d\n",
1090				  ep->qp.sq_mem_size);
1091		goto mem_alloc_err;
1092	}
1093
1094	memset(ep->qp.sq_virt, 0x00, ep->qp.sq_mem_size);
1095	ep->qp.sq_first_qe = ep->qp.sq_virt;
1096	ep->qp.sq_prod_qe = ep->qp.sq_first_qe;
1097	ep->qp.sq_cons_qe = ep->qp.sq_first_qe;
1098	ep->qp.sq_last_qe = &ep->qp.sq_first_qe[hba->max_sqes - 1];
1099	ep->qp.sq_prod_idx = 0;
1100	ep->qp.sq_cons_idx = 0;
1101	ep->qp.sqe_left = hba->max_sqes;
1102
1103	/* Allocate page table memory for CQ which is page aligned */
1104	ep->qp.cq_mem_size = hba->max_cqes * BNX2I_CQE_SIZE;
1105	ep->qp.cq_mem_size =
1106		(ep->qp.cq_mem_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK;
1107	ep->qp.cq_pgtbl_size =
1108		(ep->qp.cq_mem_size / CNIC_PAGE_SIZE) * sizeof(void *);
1109	ep->qp.cq_pgtbl_size =
1110		(ep->qp.cq_pgtbl_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK;
1111
1112	ep->qp.cq_pgtbl_virt =
1113		dma_alloc_coherent(&hba->pcidev->dev, ep->qp.cq_pgtbl_size,
1114				   &ep->qp.cq_pgtbl_phys, GFP_KERNEL);
1115	if (!ep->qp.cq_pgtbl_virt) {
1116		printk(KERN_ALERT "bnx2i: unable to alloc CQ PT memory %d\n",
1117				  ep->qp.cq_pgtbl_size);
1118		goto mem_alloc_err;
1119	}
1120
1121	/* Allocate memory area for actual CQ element */
1122	ep->qp.cq_virt =
1123		dma_alloc_coherent(&hba->pcidev->dev, ep->qp.cq_mem_size,
1124				   &ep->qp.cq_phys, GFP_KERNEL);
1125	if (!ep->qp.cq_virt) {
1126		printk(KERN_ALERT "bnx2i: unable to alloc CQ BD memory %d\n",
1127				  ep->qp.cq_mem_size);
1128		goto mem_alloc_err;
1129	}
1130	memset(ep->qp.cq_virt, 0x00, ep->qp.cq_mem_size);
1131
1132	ep->qp.cq_first_qe = ep->qp.cq_virt;
1133	ep->qp.cq_prod_qe = ep->qp.cq_first_qe;
1134	ep->qp.cq_cons_qe = ep->qp.cq_first_qe;
1135	ep->qp.cq_last_qe = &ep->qp.cq_first_qe[hba->max_cqes - 1];
1136	ep->qp.cq_prod_idx = 0;
1137	ep->qp.cq_cons_idx = 0;
1138	ep->qp.cqe_left = hba->max_cqes;
1139	ep->qp.cqe_exp_seq_sn = ISCSI_INITIAL_SN;
1140	ep->qp.cqe_size = hba->max_cqes;
1141
1142	/* Invalidate all EQ CQE index, req only for 57710 */
1143	cq_db = (struct bnx2i_5771x_cq_db *) ep->qp.cq_pgtbl_virt;
1144	memset(cq_db->sqn, 0xFF, sizeof(cq_db->sqn[0]) * BNX2X_MAX_CQS);
1145
1146	/* Allocate page table memory for RQ which is page aligned */
1147	ep->qp.rq_mem_size = hba->max_rqes * BNX2I_RQ_WQE_SIZE;
1148	ep->qp.rq_mem_size =
1149		(ep->qp.rq_mem_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK;
1150	ep->qp.rq_pgtbl_size =
1151		(ep->qp.rq_mem_size / CNIC_PAGE_SIZE) * sizeof(void *);
1152	ep->qp.rq_pgtbl_size =
1153		(ep->qp.rq_pgtbl_size + (CNIC_PAGE_SIZE - 1)) & CNIC_PAGE_MASK;
1154
1155	ep->qp.rq_pgtbl_virt =
1156		dma_alloc_coherent(&hba->pcidev->dev, ep->qp.rq_pgtbl_size,
1157				   &ep->qp.rq_pgtbl_phys, GFP_KERNEL);
1158	if (!ep->qp.rq_pgtbl_virt) {
1159		printk(KERN_ALERT "bnx2i: unable to alloc RQ PT mem %d\n",
1160				  ep->qp.rq_pgtbl_size);
1161		goto mem_alloc_err;
1162	}
1163
1164	/* Allocate memory area for actual RQ element */
1165	ep->qp.rq_virt =
1166		dma_alloc_coherent(&hba->pcidev->dev, ep->qp.rq_mem_size,
1167				   &ep->qp.rq_phys, GFP_KERNEL);
1168	if (!ep->qp.rq_virt) {
1169		printk(KERN_ALERT "bnx2i: unable to alloc RQ BD memory %d\n",
1170				  ep->qp.rq_mem_size);
1171		goto mem_alloc_err;
1172	}
1173
1174	ep->qp.rq_first_qe = ep->qp.rq_virt;
1175	ep->qp.rq_prod_qe = ep->qp.rq_first_qe;
1176	ep->qp.rq_cons_qe = ep->qp.rq_first_qe;
1177	ep->qp.rq_last_qe = &ep->qp.rq_first_qe[hba->max_rqes - 1];
1178	ep->qp.rq_prod_idx = 0x8000;
1179	ep->qp.rq_cons_idx = 0;
1180	ep->qp.rqe_left = hba->max_rqes;
1181
1182	setup_qp_page_tables(ep);
1183
1184	return 0;
1185
1186mem_alloc_err:
1187	bnx2i_free_qp_resc(hba, ep);
1188	return -ENOMEM;
1189}
1190
1191
1192
1193/**
1194 * bnx2i_free_qp_resc - free memory resources held by QP
1195 * @hba:	adapter structure pointer
1196 * @ep:	endpoint (transport identifier) structure
1197 *
1198 * Free QP resources - SQ/RQ/CQ memory and page tables.
1199 */
1200void bnx2i_free_qp_resc(struct bnx2i_hba *hba, struct bnx2i_endpoint *ep)
1201{
1202	if (ep->qp.ctx_base) {
1203		iounmap(ep->qp.ctx_base);
1204		ep->qp.ctx_base = NULL;
1205	}
1206	/* Free SQ mem */
1207	if (ep->qp.sq_pgtbl_virt) {
1208		dma_free_coherent(&hba->pcidev->dev, ep->qp.sq_pgtbl_size,
1209				  ep->qp.sq_pgtbl_virt, ep->qp.sq_pgtbl_phys);
1210		ep->qp.sq_pgtbl_virt = NULL;
1211		ep->qp.sq_pgtbl_phys = 0;
1212	}
1213	if (ep->qp.sq_virt) {
1214		dma_free_coherent(&hba->pcidev->dev, ep->qp.sq_mem_size,
1215				  ep->qp.sq_virt, ep->qp.sq_phys);
1216		ep->qp.sq_virt = NULL;
1217		ep->qp.sq_phys = 0;
1218	}
1219
1220	/* Free RQ mem */
1221	if (ep->qp.rq_pgtbl_virt) {
1222		dma_free_coherent(&hba->pcidev->dev, ep->qp.rq_pgtbl_size,
1223				  ep->qp.rq_pgtbl_virt, ep->qp.rq_pgtbl_phys);
1224		ep->qp.rq_pgtbl_virt = NULL;
1225		ep->qp.rq_pgtbl_phys = 0;
1226	}
1227	if (ep->qp.rq_virt) {
1228		dma_free_coherent(&hba->pcidev->dev, ep->qp.rq_mem_size,
1229				  ep->qp.rq_virt, ep->qp.rq_phys);
1230		ep->qp.rq_virt = NULL;
1231		ep->qp.rq_phys = 0;
1232	}
1233
1234	/* Free CQ mem */
1235	if (ep->qp.cq_pgtbl_virt) {
1236		dma_free_coherent(&hba->pcidev->dev, ep->qp.cq_pgtbl_size,
1237				  ep->qp.cq_pgtbl_virt, ep->qp.cq_pgtbl_phys);
1238		ep->qp.cq_pgtbl_virt = NULL;
1239		ep->qp.cq_pgtbl_phys = 0;
1240	}
1241	if (ep->qp.cq_virt) {
1242		dma_free_coherent(&hba->pcidev->dev, ep->qp.cq_mem_size,
1243				  ep->qp.cq_virt, ep->qp.cq_phys);
1244		ep->qp.cq_virt = NULL;
1245		ep->qp.cq_phys = 0;
1246	}
1247}
1248
1249
1250/**
1251 * bnx2i_send_fw_iscsi_init_msg - initiates initial handshake with iscsi f/w
1252 * @hba:	adapter structure pointer
1253 *
1254 * Send down iscsi_init KWQEs which initiates the initial handshake with the f/w
1255 * 	This results in iSCSi support validation and on-chip context manager
1256 * 	initialization.  Firmware completes this handshake with a CQE carrying
1257 * 	the result of iscsi support validation. Parameter carried by
1258 * 	iscsi init request determines the number of offloaded connection and
1259 * 	tolerance level for iscsi protocol violation this hba/chip can support
1260 */
1261int bnx2i_send_fw_iscsi_init_msg(struct bnx2i_hba *hba)
1262{
1263	struct kwqe *kwqe_arr[3];
1264	struct iscsi_kwqe_init1 iscsi_init;
1265	struct iscsi_kwqe_init2 iscsi_init2;
1266	int rc = 0;
1267	u64 mask64;
1268
1269	memset(&iscsi_init, 0x00, sizeof(struct iscsi_kwqe_init1));
1270	memset(&iscsi_init2, 0x00, sizeof(struct iscsi_kwqe_init2));
1271
1272	bnx2i_adjust_qp_size(hba);
1273
1274	iscsi_init.flags =
1275		(CNIC_PAGE_BITS - 8) << ISCSI_KWQE_INIT1_PAGE_SIZE_SHIFT;
1276	if (en_tcp_dack)
1277		iscsi_init.flags |= ISCSI_KWQE_INIT1_DELAYED_ACK_ENABLE;
1278	iscsi_init.reserved0 = 0;
1279	iscsi_init.num_cqs = 1;
1280	iscsi_init.hdr.op_code = ISCSI_KWQE_OPCODE_INIT1;
1281	iscsi_init.hdr.flags =
1282		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
1283
1284	iscsi_init.dummy_buffer_addr_lo = (u32) hba->dummy_buf_dma;
1285	iscsi_init.dummy_buffer_addr_hi =
1286		(u32) ((u64) hba->dummy_buf_dma >> 32);
1287
1288	hba->num_ccell = hba->max_sqes >> 1;
1289	hba->ctx_ccell_tasks =
1290			((hba->num_ccell & 0xFFFF) | (hba->max_sqes << 16));
1291	iscsi_init.num_ccells_per_conn = hba->num_ccell;
1292	iscsi_init.num_tasks_per_conn = hba->max_sqes;
1293	iscsi_init.sq_wqes_per_page = CNIC_PAGE_SIZE / BNX2I_SQ_WQE_SIZE;
1294	iscsi_init.sq_num_wqes = hba->max_sqes;
1295	iscsi_init.cq_log_wqes_per_page =
1296		(u8) bnx2i_power_of2(CNIC_PAGE_SIZE / BNX2I_CQE_SIZE);
1297	iscsi_init.cq_num_wqes = hba->max_cqes;
1298	iscsi_init.cq_num_pages = (hba->max_cqes * BNX2I_CQE_SIZE +
1299				   (CNIC_PAGE_SIZE - 1)) / CNIC_PAGE_SIZE;
1300	iscsi_init.sq_num_pages = (hba->max_sqes * BNX2I_SQ_WQE_SIZE +
1301				   (CNIC_PAGE_SIZE - 1)) / CNIC_PAGE_SIZE;
1302	iscsi_init.rq_buffer_size = BNX2I_RQ_WQE_SIZE;
1303	iscsi_init.rq_num_wqes = hba->max_rqes;
1304
1305
1306	iscsi_init2.hdr.op_code = ISCSI_KWQE_OPCODE_INIT2;
1307	iscsi_init2.hdr.flags =
1308		(ISCSI_KWQE_LAYER_CODE << ISCSI_KWQE_HEADER_LAYER_CODE_SHIFT);
1309	iscsi_init2.max_cq_sqn = hba->max_cqes * 2 + 1;
1310	mask64 = 0x0ULL;
1311	mask64 |= (
1312		/* CISCO MDS */
1313		(1UL <<
1314		  ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_TTT_NOT_RSRV) |
1315		/* HP MSA1510i */
1316		(1UL <<
1317		  ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_EXP_DATASN) |
1318		/* EMC */
1319		(1ULL << ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_LUN));
1320	if (error_mask1) {
1321		iscsi_init2.error_bit_map[0] = error_mask1;
1322		mask64 ^= (u32)(mask64);
1323		mask64 |= error_mask1;
1324	} else
1325		iscsi_init2.error_bit_map[0] = (u32) mask64;
1326
1327	if (error_mask2) {
1328		iscsi_init2.error_bit_map[1] = error_mask2;
1329		mask64 &= 0xffffffff;
1330		mask64 |= ((u64)error_mask2 << 32);
1331	} else
1332		iscsi_init2.error_bit_map[1] = (u32) (mask64 >> 32);
1333
1334	iscsi_error_mask = mask64;
1335
1336	kwqe_arr[0] = (struct kwqe *) &iscsi_init;
1337	kwqe_arr[1] = (struct kwqe *) &iscsi_init2;
1338
1339	if (hba->cnic && hba->cnic->submit_kwqes)
1340		rc = hba->cnic->submit_kwqes(hba->cnic, kwqe_arr, 2);
1341	return rc;
1342}
1343
1344
1345/**
1346 * bnx2i_process_scsi_cmd_resp - this function handles scsi cmd completion.
1347 * @session:	iscsi session
1348 * @bnx2i_conn:	bnx2i connection
1349 * @cqe:	pointer to newly DMA'ed CQE entry for processing
1350 *
1351 * process SCSI CMD Response CQE & complete the request to SCSI-ML
1352 */
1353int bnx2i_process_scsi_cmd_resp(struct iscsi_session *session,
1354				struct bnx2i_conn *bnx2i_conn,
1355				struct cqe *cqe)
1356{
1357	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1358	struct bnx2i_hba *hba = bnx2i_conn->hba;
1359	struct bnx2i_cmd_response *resp_cqe;
1360	struct bnx2i_cmd *bnx2i_cmd;
1361	struct iscsi_task *task;
1362	struct iscsi_scsi_rsp *hdr;
1363	u32 datalen = 0;
1364
1365	resp_cqe = (struct bnx2i_cmd_response *)cqe;
1366	spin_lock_bh(&session->back_lock);
1367	task = iscsi_itt_to_task(conn,
1368				 resp_cqe->itt & ISCSI_CMD_RESPONSE_INDEX);
1369	if (!task)
1370		goto fail;
1371
1372	bnx2i_cmd = task->dd_data;
1373
1374	if (bnx2i_cmd->req.op_attr & ISCSI_CMD_REQUEST_READ) {
1375		conn->datain_pdus_cnt +=
1376			resp_cqe->task_stat.read_stat.num_data_ins;
1377		conn->rxdata_octets +=
1378			bnx2i_cmd->req.total_data_transfer_length;
1379		ADD_STATS_64(hba, rx_pdus,
1380			     resp_cqe->task_stat.read_stat.num_data_ins);
1381		ADD_STATS_64(hba, rx_bytes,
1382			     bnx2i_cmd->req.total_data_transfer_length);
1383	} else {
1384		conn->dataout_pdus_cnt +=
1385			resp_cqe->task_stat.write_stat.num_data_outs;
1386		conn->r2t_pdus_cnt +=
1387			resp_cqe->task_stat.write_stat.num_r2ts;
1388		conn->txdata_octets +=
1389			bnx2i_cmd->req.total_data_transfer_length;
1390		ADD_STATS_64(hba, tx_pdus,
1391			     resp_cqe->task_stat.write_stat.num_data_outs);
1392		ADD_STATS_64(hba, tx_bytes,
1393			     bnx2i_cmd->req.total_data_transfer_length);
1394		ADD_STATS_64(hba, rx_pdus,
1395			     resp_cqe->task_stat.write_stat.num_r2ts);
1396	}
1397	bnx2i_iscsi_unmap_sg_list(bnx2i_cmd);
1398
1399	hdr = (struct iscsi_scsi_rsp *)task->hdr;
1400	resp_cqe = (struct bnx2i_cmd_response *)cqe;
1401	hdr->opcode = resp_cqe->op_code;
1402	hdr->max_cmdsn = cpu_to_be32(resp_cqe->max_cmd_sn);
1403	hdr->exp_cmdsn = cpu_to_be32(resp_cqe->exp_cmd_sn);
1404	hdr->response = resp_cqe->response;
1405	hdr->cmd_status = resp_cqe->status;
1406	hdr->flags = resp_cqe->response_flags;
1407	hdr->residual_count = cpu_to_be32(resp_cqe->residual_count);
1408
1409	if (resp_cqe->op_code == ISCSI_OP_SCSI_DATA_IN)
1410		goto done;
1411
1412	if (resp_cqe->status == SAM_STAT_CHECK_CONDITION) {
1413		datalen = resp_cqe->data_length;
1414		if (datalen < 2)
1415			goto done;
1416
1417		if (datalen > BNX2I_RQ_WQE_SIZE) {
1418			iscsi_conn_printk(KERN_ERR, conn,
1419					  "sense data len %d > RQ sz\n",
1420					  datalen);
1421			datalen = BNX2I_RQ_WQE_SIZE;
1422		} else if (datalen > ISCSI_DEF_MAX_RECV_SEG_LEN) {
1423			iscsi_conn_printk(KERN_ERR, conn,
1424					  "sense data len %d > conn data\n",
1425					  datalen);
1426			datalen = ISCSI_DEF_MAX_RECV_SEG_LEN;
1427		}
1428
1429		bnx2i_get_rq_buf(bnx2i_cmd->conn, conn->data, datalen);
1430		bnx2i_put_rq_buf(bnx2i_cmd->conn, 1);
1431	}
1432
1433done:
1434	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr,
1435			     conn->data, datalen);
1436fail:
1437	spin_unlock_bh(&session->back_lock);
1438	return 0;
1439}
1440
1441
1442/**
1443 * bnx2i_process_login_resp - this function handles iscsi login response
1444 * @session:		iscsi session pointer
1445 * @bnx2i_conn:		iscsi connection pointer
1446 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1447 *
1448 * process Login Response CQE & complete it to open-iscsi user daemon
1449 */
1450static int bnx2i_process_login_resp(struct iscsi_session *session,
1451				    struct bnx2i_conn *bnx2i_conn,
1452				    struct cqe *cqe)
1453{
1454	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1455	struct iscsi_task *task;
1456	struct bnx2i_login_response *login;
1457	struct iscsi_login_rsp *resp_hdr;
1458	int pld_len;
1459	int pad_len;
1460
1461	login = (struct bnx2i_login_response *) cqe;
1462	spin_lock(&session->back_lock);
1463	task = iscsi_itt_to_task(conn,
1464				 login->itt & ISCSI_LOGIN_RESPONSE_INDEX);
1465	if (!task)
1466		goto done;
1467
1468	resp_hdr = (struct iscsi_login_rsp *) &bnx2i_conn->gen_pdu.resp_hdr;
1469	memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
1470	resp_hdr->opcode = login->op_code;
1471	resp_hdr->flags = login->response_flags;
1472	resp_hdr->max_version = login->version_max;
1473	resp_hdr->active_version = login->version_active;
1474	resp_hdr->hlength = 0;
1475
1476	hton24(resp_hdr->dlength, login->data_length);
1477	memcpy(resp_hdr->isid, &login->isid_lo, 6);
1478	resp_hdr->tsih = cpu_to_be16(login->tsih);
1479	resp_hdr->itt = task->hdr->itt;
1480	resp_hdr->statsn = cpu_to_be32(login->stat_sn);
1481	resp_hdr->exp_cmdsn = cpu_to_be32(login->exp_cmd_sn);
1482	resp_hdr->max_cmdsn = cpu_to_be32(login->max_cmd_sn);
1483	resp_hdr->status_class = login->status_class;
1484	resp_hdr->status_detail = login->status_detail;
1485	pld_len = login->data_length;
1486	bnx2i_conn->gen_pdu.resp_wr_ptr =
1487					bnx2i_conn->gen_pdu.resp_buf + pld_len;
1488
1489	pad_len = 0;
1490	if (pld_len & 0x3)
1491		pad_len = 4 - (pld_len % 4);
1492
1493	if (pad_len) {
1494		int i = 0;
1495		for (i = 0; i < pad_len; i++) {
1496			bnx2i_conn->gen_pdu.resp_wr_ptr[0] = 0;
1497			bnx2i_conn->gen_pdu.resp_wr_ptr++;
1498		}
1499	}
1500
1501	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr,
1502		bnx2i_conn->gen_pdu.resp_buf,
1503		bnx2i_conn->gen_pdu.resp_wr_ptr - bnx2i_conn->gen_pdu.resp_buf);
1504done:
1505	spin_unlock(&session->back_lock);
1506	return 0;
1507}
1508
1509
1510/**
1511 * bnx2i_process_text_resp - this function handles iscsi text response
1512 * @session:	iscsi session pointer
1513 * @bnx2i_conn:	iscsi connection pointer
1514 * @cqe:	pointer to newly DMA'ed CQE entry for processing
1515 *
1516 * process iSCSI Text Response CQE&  complete it to open-iscsi user daemon
1517 */
1518static int bnx2i_process_text_resp(struct iscsi_session *session,
1519				   struct bnx2i_conn *bnx2i_conn,
1520				   struct cqe *cqe)
1521{
1522	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1523	struct iscsi_task *task;
1524	struct bnx2i_text_response *text;
1525	struct iscsi_text_rsp *resp_hdr;
1526	int pld_len;
1527	int pad_len;
1528
1529	text = (struct bnx2i_text_response *) cqe;
1530	spin_lock(&session->back_lock);
1531	task = iscsi_itt_to_task(conn, text->itt & ISCSI_LOGIN_RESPONSE_INDEX);
1532	if (!task)
1533		goto done;
1534
1535	resp_hdr = (struct iscsi_text_rsp *)&bnx2i_conn->gen_pdu.resp_hdr;
1536	memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
1537	resp_hdr->opcode = text->op_code;
1538	resp_hdr->flags = text->response_flags;
1539	resp_hdr->hlength = 0;
1540
1541	hton24(resp_hdr->dlength, text->data_length);
1542	resp_hdr->itt = task->hdr->itt;
1543	resp_hdr->ttt = cpu_to_be32(text->ttt);
1544	resp_hdr->statsn = task->hdr->exp_statsn;
1545	resp_hdr->exp_cmdsn = cpu_to_be32(text->exp_cmd_sn);
1546	resp_hdr->max_cmdsn = cpu_to_be32(text->max_cmd_sn);
1547	pld_len = text->data_length;
1548	bnx2i_conn->gen_pdu.resp_wr_ptr = bnx2i_conn->gen_pdu.resp_buf +
1549					  pld_len;
1550	pad_len = 0;
1551	if (pld_len & 0x3)
1552		pad_len = 4 - (pld_len % 4);
1553
1554	if (pad_len) {
1555		int i = 0;
1556		for (i = 0; i < pad_len; i++) {
1557			bnx2i_conn->gen_pdu.resp_wr_ptr[0] = 0;
1558			bnx2i_conn->gen_pdu.resp_wr_ptr++;
1559		}
1560	}
1561	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr,
1562			     bnx2i_conn->gen_pdu.resp_buf,
1563			     bnx2i_conn->gen_pdu.resp_wr_ptr -
1564			     bnx2i_conn->gen_pdu.resp_buf);
1565done:
1566	spin_unlock(&session->back_lock);
1567	return 0;
1568}
1569
1570
1571/**
1572 * bnx2i_process_tmf_resp - this function handles iscsi TMF response
1573 * @session:		iscsi session pointer
1574 * @bnx2i_conn:		iscsi connection pointer
1575 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1576 *
1577 * process iSCSI TMF Response CQE and wake up the driver eh thread.
1578 */
1579static int bnx2i_process_tmf_resp(struct iscsi_session *session,
1580				  struct bnx2i_conn *bnx2i_conn,
1581				  struct cqe *cqe)
1582{
1583	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1584	struct iscsi_task *task;
1585	struct bnx2i_tmf_response *tmf_cqe;
1586	struct iscsi_tm_rsp *resp_hdr;
1587
1588	tmf_cqe = (struct bnx2i_tmf_response *)cqe;
1589	spin_lock(&session->back_lock);
1590	task = iscsi_itt_to_task(conn,
1591				 tmf_cqe->itt & ISCSI_TMF_RESPONSE_INDEX);
1592	if (!task)
1593		goto done;
1594
1595	resp_hdr = (struct iscsi_tm_rsp *) &bnx2i_conn->gen_pdu.resp_hdr;
1596	memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
1597	resp_hdr->opcode = tmf_cqe->op_code;
1598	resp_hdr->max_cmdsn = cpu_to_be32(tmf_cqe->max_cmd_sn);
1599	resp_hdr->exp_cmdsn = cpu_to_be32(tmf_cqe->exp_cmd_sn);
1600	resp_hdr->itt = task->hdr->itt;
1601	resp_hdr->response = tmf_cqe->response;
1602
1603	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, NULL, 0);
1604done:
1605	spin_unlock(&session->back_lock);
1606	return 0;
1607}
1608
1609/**
1610 * bnx2i_process_logout_resp - this function handles iscsi logout response
1611 * @session:		iscsi session pointer
1612 * @bnx2i_conn:		iscsi connection pointer
1613 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1614 *
1615 * process iSCSI Logout Response CQE & make function call to
1616 * notify the user daemon.
1617 */
1618static int bnx2i_process_logout_resp(struct iscsi_session *session,
1619				     struct bnx2i_conn *bnx2i_conn,
1620				     struct cqe *cqe)
1621{
1622	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1623	struct iscsi_task *task;
1624	struct bnx2i_logout_response *logout;
1625	struct iscsi_logout_rsp *resp_hdr;
1626
1627	logout = (struct bnx2i_logout_response *) cqe;
1628	spin_lock(&session->back_lock);
1629	task = iscsi_itt_to_task(conn,
1630				 logout->itt & ISCSI_LOGOUT_RESPONSE_INDEX);
1631	if (!task)
1632		goto done;
1633
1634	resp_hdr = (struct iscsi_logout_rsp *) &bnx2i_conn->gen_pdu.resp_hdr;
1635	memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
1636	resp_hdr->opcode = logout->op_code;
1637	resp_hdr->flags = logout->response;
1638	resp_hdr->hlength = 0;
1639
1640	resp_hdr->itt = task->hdr->itt;
1641	resp_hdr->statsn = task->hdr->exp_statsn;
1642	resp_hdr->exp_cmdsn = cpu_to_be32(logout->exp_cmd_sn);
1643	resp_hdr->max_cmdsn = cpu_to_be32(logout->max_cmd_sn);
1644
1645	resp_hdr->t2wait = cpu_to_be32(logout->time_to_wait);
1646	resp_hdr->t2retain = cpu_to_be32(logout->time_to_retain);
1647
1648	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, NULL, 0);
1649
1650	bnx2i_conn->ep->state = EP_STATE_LOGOUT_RESP_RCVD;
1651done:
1652	spin_unlock(&session->back_lock);
1653	return 0;
1654}
1655
1656/**
1657 * bnx2i_process_nopin_local_cmpl - this function handles iscsi nopin CQE
1658 * @session:		iscsi session pointer
1659 * @bnx2i_conn:		iscsi connection pointer
1660 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1661 *
1662 * process iSCSI NOPIN local completion CQE, frees IIT and command structures
1663 */
1664static void bnx2i_process_nopin_local_cmpl(struct iscsi_session *session,
1665					   struct bnx2i_conn *bnx2i_conn,
1666					   struct cqe *cqe)
1667{
1668	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1669	struct bnx2i_nop_in_msg *nop_in;
1670	struct iscsi_task *task;
1671
1672	nop_in = (struct bnx2i_nop_in_msg *)cqe;
1673	spin_lock(&session->back_lock);
1674	task = iscsi_itt_to_task(conn,
1675				 nop_in->itt & ISCSI_NOP_IN_MSG_INDEX);
1676	if (task)
1677		__iscsi_put_task(task);
1678	spin_unlock(&session->back_lock);
1679}
1680
1681/**
1682 * bnx2i_unsol_pdu_adjust_rq - makes adjustments to RQ after unsol pdu is recvd
1683 * @conn:	iscsi connection
1684 *
1685 * Firmware advances RQ producer index for every unsolicited PDU even if
1686 *	payload data length is '0'. This function makes corresponding
1687 *	adjustments on the driver side to match this f/w behavior
1688 */
1689static void bnx2i_unsol_pdu_adjust_rq(struct bnx2i_conn *bnx2i_conn)
1690{
1691	char dummy_rq_data[2];
1692	bnx2i_get_rq_buf(bnx2i_conn, dummy_rq_data, 1);
1693	bnx2i_put_rq_buf(bnx2i_conn, 1);
1694}
1695
1696
1697/**
1698 * bnx2i_process_nopin_mesg - this function handles iscsi nopin CQE
1699 * @session:		iscsi session pointer
1700 * @bnx2i_conn:		iscsi connection pointer
1701 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1702 *
1703 * process iSCSI target's proactive iSCSI NOPIN request
1704 */
1705static int bnx2i_process_nopin_mesg(struct iscsi_session *session,
1706				     struct bnx2i_conn *bnx2i_conn,
1707				     struct cqe *cqe)
1708{
1709	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1710	struct iscsi_task *task;
1711	struct bnx2i_nop_in_msg *nop_in;
1712	struct iscsi_nopin *hdr;
1713	int tgt_async_nop = 0;
1714
1715	nop_in = (struct bnx2i_nop_in_msg *)cqe;
1716
1717	spin_lock(&session->back_lock);
1718	hdr = (struct iscsi_nopin *)&bnx2i_conn->gen_pdu.resp_hdr;
1719	memset(hdr, 0, sizeof(struct iscsi_hdr));
1720	hdr->opcode = nop_in->op_code;
1721	hdr->max_cmdsn = cpu_to_be32(nop_in->max_cmd_sn);
1722	hdr->exp_cmdsn = cpu_to_be32(nop_in->exp_cmd_sn);
1723	hdr->ttt = cpu_to_be32(nop_in->ttt);
1724
1725	if (nop_in->itt == (u16) RESERVED_ITT) {
1726		bnx2i_unsol_pdu_adjust_rq(bnx2i_conn);
1727		hdr->itt = RESERVED_ITT;
1728		tgt_async_nop = 1;
1729		goto done;
1730	}
1731
1732	/* this is a response to one of our nop-outs */
1733	task = iscsi_itt_to_task(conn,
1734			 (itt_t) (nop_in->itt & ISCSI_NOP_IN_MSG_INDEX));
1735	if (task) {
1736		hdr->flags = ISCSI_FLAG_CMD_FINAL;
1737		hdr->itt = task->hdr->itt;
1738		hdr->ttt = cpu_to_be32(nop_in->ttt);
1739		memcpy(&hdr->lun, nop_in->lun, 8);
1740	}
1741done:
1742	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1743	spin_unlock(&session->back_lock);
1744
1745	return tgt_async_nop;
1746}
1747
1748
1749/**
1750 * bnx2i_process_async_mesg - this function handles iscsi async message
1751 * @session:		iscsi session pointer
1752 * @bnx2i_conn:		iscsi connection pointer
1753 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1754 *
1755 * process iSCSI ASYNC Message
1756 */
1757static void bnx2i_process_async_mesg(struct iscsi_session *session,
1758				     struct bnx2i_conn *bnx2i_conn,
1759				     struct cqe *cqe)
1760{
1761	struct bnx2i_async_msg *async_cqe;
1762	struct iscsi_async *resp_hdr;
1763	u8 async_event;
1764
1765	bnx2i_unsol_pdu_adjust_rq(bnx2i_conn);
1766
1767	async_cqe = (struct bnx2i_async_msg *)cqe;
1768	async_event = async_cqe->async_event;
1769
1770	if (async_event == ISCSI_ASYNC_MSG_SCSI_EVENT) {
1771		iscsi_conn_printk(KERN_ALERT, bnx2i_conn->cls_conn->dd_data,
1772				  "async: scsi events not supported\n");
1773		return;
1774	}
1775
1776	spin_lock(&session->back_lock);
1777	resp_hdr = (struct iscsi_async *) &bnx2i_conn->gen_pdu.resp_hdr;
1778	memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
1779	resp_hdr->opcode = async_cqe->op_code;
1780	resp_hdr->flags = 0x80;
1781
1782	memcpy(&resp_hdr->lun, async_cqe->lun, 8);
1783	resp_hdr->exp_cmdsn = cpu_to_be32(async_cqe->exp_cmd_sn);
1784	resp_hdr->max_cmdsn = cpu_to_be32(async_cqe->max_cmd_sn);
1785
1786	resp_hdr->async_event = async_cqe->async_event;
1787	resp_hdr->async_vcode = async_cqe->async_vcode;
1788
1789	resp_hdr->param1 = cpu_to_be16(async_cqe->param1);
1790	resp_hdr->param2 = cpu_to_be16(async_cqe->param2);
1791	resp_hdr->param3 = cpu_to_be16(async_cqe->param3);
1792
1793	__iscsi_complete_pdu(bnx2i_conn->cls_conn->dd_data,
1794			     (struct iscsi_hdr *)resp_hdr, NULL, 0);
1795	spin_unlock(&session->back_lock);
1796}
1797
1798
1799/**
1800 * bnx2i_process_reject_mesg - process iscsi reject pdu
1801 * @session:		iscsi session pointer
1802 * @bnx2i_conn:		iscsi connection pointer
1803 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1804 *
1805 * process iSCSI REJECT message
1806 */
1807static void bnx2i_process_reject_mesg(struct iscsi_session *session,
1808				      struct bnx2i_conn *bnx2i_conn,
1809				      struct cqe *cqe)
1810{
1811	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1812	struct bnx2i_reject_msg *reject;
1813	struct iscsi_reject *hdr;
1814
1815	reject = (struct bnx2i_reject_msg *) cqe;
1816	if (reject->data_length) {
1817		bnx2i_get_rq_buf(bnx2i_conn, conn->data, reject->data_length);
1818		bnx2i_put_rq_buf(bnx2i_conn, 1);
1819	} else
1820		bnx2i_unsol_pdu_adjust_rq(bnx2i_conn);
1821
1822	spin_lock(&session->back_lock);
1823	hdr = (struct iscsi_reject *) &bnx2i_conn->gen_pdu.resp_hdr;
1824	memset(hdr, 0, sizeof(struct iscsi_hdr));
1825	hdr->opcode = reject->op_code;
1826	hdr->reason = reject->reason;
1827	hton24(hdr->dlength, reject->data_length);
1828	hdr->max_cmdsn = cpu_to_be32(reject->max_cmd_sn);
1829	hdr->exp_cmdsn = cpu_to_be32(reject->exp_cmd_sn);
1830	hdr->ffffffff = cpu_to_be32(RESERVED_ITT);
1831	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, conn->data,
1832			     reject->data_length);
1833	spin_unlock(&session->back_lock);
1834}
1835
1836/**
1837 * bnx2i_process_cmd_cleanup_resp - process scsi command clean-up completion
1838 * @session:		iscsi session pointer
1839 * @bnx2i_conn:		iscsi connection pointer
1840 * @cqe:		pointer to newly DMA'ed CQE entry for processing
1841 *
1842 * process command cleanup response CQE during conn shutdown or error recovery
1843 */
1844static void bnx2i_process_cmd_cleanup_resp(struct iscsi_session *session,
1845					   struct bnx2i_conn *bnx2i_conn,
1846					   struct cqe *cqe)
1847{
1848	struct bnx2i_cleanup_response *cmd_clean_rsp;
1849	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1850	struct iscsi_task *task;
1851
1852	cmd_clean_rsp = (struct bnx2i_cleanup_response *)cqe;
1853	spin_lock(&session->back_lock);
1854	task = iscsi_itt_to_task(conn,
1855			cmd_clean_rsp->itt & ISCSI_CLEANUP_RESPONSE_INDEX);
1856	if (!task)
1857		printk(KERN_ALERT "bnx2i: cmd clean ITT %x not active\n",
1858			cmd_clean_rsp->itt & ISCSI_CLEANUP_RESPONSE_INDEX);
1859	spin_unlock(&session->back_lock);
1860	complete(&bnx2i_conn->cmd_cleanup_cmpl);
1861}
1862
1863
1864/**
1865 * bnx2i_percpu_io_thread - thread per cpu for ios
1866 *
1867 * @arg:	ptr to bnx2i_percpu_info structure
1868 */
1869int bnx2i_percpu_io_thread(void *arg)
1870{
1871	struct bnx2i_percpu_s *p = arg;
1872	struct bnx2i_work *work, *tmp;
1873	LIST_HEAD(work_list);
1874
1875	set_user_nice(current, MIN_NICE);
1876
1877	while (!kthread_should_stop()) {
1878		spin_lock_bh(&p->p_work_lock);
1879		while (!list_empty(&p->work_list)) {
1880			list_splice_init(&p->work_list, &work_list);
1881			spin_unlock_bh(&p->p_work_lock);
1882
1883			list_for_each_entry_safe(work, tmp, &work_list, list) {
1884				list_del_init(&work->list);
1885				/* work allocated in the bh, freed here */
1886				bnx2i_process_scsi_cmd_resp(work->session,
1887							    work->bnx2i_conn,
1888							    &work->cqe);
1889				atomic_dec(&work->bnx2i_conn->work_cnt);
1890				kfree(work);
1891			}
1892			spin_lock_bh(&p->p_work_lock);
1893		}
1894		set_current_state(TASK_INTERRUPTIBLE);
1895		spin_unlock_bh(&p->p_work_lock);
1896		schedule();
1897	}
1898	__set_current_state(TASK_RUNNING);
1899
1900	return 0;
1901}
1902
1903
1904/**
1905 * bnx2i_queue_scsi_cmd_resp - queue cmd completion to the percpu thread
1906 * @bnx2i_conn:		bnx2i connection
1907 *
1908 * this function is called by generic KCQ handler to queue all pending cmd
1909 * completion CQEs
1910 *
1911 * The implementation is to queue the cmd response based on the
1912 * last recorded command for the given connection.  The
1913 * cpu_id gets recorded upon task_xmit.  No out-of-order completion!
1914 */
1915static int bnx2i_queue_scsi_cmd_resp(struct iscsi_session *session,
1916				     struct bnx2i_conn *bnx2i_conn,
1917				     struct bnx2i_nop_in_msg *cqe)
1918{
1919	struct bnx2i_work *bnx2i_work = NULL;
1920	struct bnx2i_percpu_s *p = NULL;
1921	struct iscsi_task *task;
1922	struct scsi_cmnd *sc;
1923	int rc = 0;
1924	int cpu;
1925
1926	spin_lock(&session->back_lock);
1927	task = iscsi_itt_to_task(bnx2i_conn->cls_conn->dd_data,
1928				 cqe->itt & ISCSI_CMD_RESPONSE_INDEX);
1929	if (!task || !task->sc) {
1930		spin_unlock(&session->back_lock);
1931		return -EINVAL;
1932	}
1933	sc = task->sc;
1934
1935	if (!blk_rq_cpu_valid(sc->request))
1936		cpu = smp_processor_id();
1937	else
1938		cpu = sc->request->cpu;
1939
1940	spin_unlock(&session->back_lock);
1941
1942	p = &per_cpu(bnx2i_percpu, cpu);
1943	spin_lock(&p->p_work_lock);
1944	if (unlikely(!p->iothread)) {
1945		rc = -EINVAL;
1946		goto err;
1947	}
1948	/* Alloc and copy to the cqe */
1949	bnx2i_work = kzalloc(sizeof(struct bnx2i_work), GFP_ATOMIC);
1950	if (bnx2i_work) {
1951		INIT_LIST_HEAD(&bnx2i_work->list);
1952		bnx2i_work->session = session;
1953		bnx2i_work->bnx2i_conn = bnx2i_conn;
1954		memcpy(&bnx2i_work->cqe, cqe, sizeof(struct cqe));
1955		list_add_tail(&bnx2i_work->list, &p->work_list);
1956		atomic_inc(&bnx2i_conn->work_cnt);
1957		wake_up_process(p->iothread);
1958		spin_unlock(&p->p_work_lock);
1959		goto done;
1960	} else
1961		rc = -ENOMEM;
1962err:
1963	spin_unlock(&p->p_work_lock);
1964	bnx2i_process_scsi_cmd_resp(session, bnx2i_conn, (struct cqe *)cqe);
1965done:
1966	return rc;
1967}
1968
1969
1970/**
1971 * bnx2i_process_new_cqes - process newly DMA'ed CQE's
1972 * @bnx2i_conn:		bnx2i connection
1973 *
1974 * this function is called by generic KCQ handler to process all pending CQE's
1975 */
1976static int bnx2i_process_new_cqes(struct bnx2i_conn *bnx2i_conn)
1977{
1978	struct iscsi_conn *conn = bnx2i_conn->cls_conn->dd_data;
1979	struct iscsi_session *session = conn->session;
1980	struct bnx2i_hba *hba = bnx2i_conn->hba;
1981	struct qp_info *qp;
1982	struct bnx2i_nop_in_msg *nopin;
1983	int tgt_async_msg;
1984	int cqe_cnt = 0;
1985
1986	if (bnx2i_conn->ep == NULL)
1987		return 0;
1988
1989	qp = &bnx2i_conn->ep->qp;
1990
1991	if (!qp->cq_virt) {
1992		printk(KERN_ALERT "bnx2i (%s): cq resr freed in bh execution!",
1993		       hba->netdev->name);
1994		goto out;
1995	}
1996	while (1) {
1997		nopin = (struct bnx2i_nop_in_msg *) qp->cq_cons_qe;
1998		if (nopin->cq_req_sn != qp->cqe_exp_seq_sn)
1999			break;
2000
2001		if (unlikely(test_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx))) {
2002			if (nopin->op_code == ISCSI_OP_NOOP_IN &&
2003			    nopin->itt == (u16) RESERVED_ITT) {
2004				printk(KERN_ALERT "bnx2i: Unsolicited "
2005				       "NOP-In detected for suspended "
2006				       "connection dev=%s!\n",
2007				       hba->netdev->name);
2008				bnx2i_unsol_pdu_adjust_rq(bnx2i_conn);
2009				goto cqe_out;
2010			}
2011			break;
2012		}
2013		tgt_async_msg = 0;
2014
2015		switch (nopin->op_code) {
2016		case ISCSI_OP_SCSI_CMD_RSP:
2017		case ISCSI_OP_SCSI_DATA_IN:
2018			/* Run the kthread engine only for data cmds
2019			   All other cmds will be completed in this bh! */
2020			bnx2i_queue_scsi_cmd_resp(session, bnx2i_conn, nopin);
2021			goto done;
2022		case ISCSI_OP_LOGIN_RSP:
2023			bnx2i_process_login_resp(session, bnx2i_conn,
2024						 qp->cq_cons_qe);
2025			break;
2026		case ISCSI_OP_SCSI_TMFUNC_RSP:
2027			bnx2i_process_tmf_resp(session, bnx2i_conn,
2028					       qp->cq_cons_qe);
2029			break;
2030		case ISCSI_OP_TEXT_RSP:
2031			bnx2i_process_text_resp(session, bnx2i_conn,
2032						qp->cq_cons_qe);
2033			break;
2034		case ISCSI_OP_LOGOUT_RSP:
2035			bnx2i_process_logout_resp(session, bnx2i_conn,
2036						  qp->cq_cons_qe);
2037			break;
2038		case ISCSI_OP_NOOP_IN:
2039			if (bnx2i_process_nopin_mesg(session, bnx2i_conn,
2040						     qp->cq_cons_qe))
2041				tgt_async_msg = 1;
2042			break;
2043		case ISCSI_OPCODE_NOPOUT_LOCAL_COMPLETION:
2044			bnx2i_process_nopin_local_cmpl(session, bnx2i_conn,
2045						       qp->cq_cons_qe);
2046			break;
2047		case ISCSI_OP_ASYNC_EVENT:
2048			bnx2i_process_async_mesg(session, bnx2i_conn,
2049						 qp->cq_cons_qe);
2050			tgt_async_msg = 1;
2051			break;
2052		case ISCSI_OP_REJECT:
2053			bnx2i_process_reject_mesg(session, bnx2i_conn,
2054						  qp->cq_cons_qe);
2055			break;
2056		case ISCSI_OPCODE_CLEANUP_RESPONSE:
2057			bnx2i_process_cmd_cleanup_resp(session, bnx2i_conn,
2058						       qp->cq_cons_qe);
2059			break;
2060		default:
2061			printk(KERN_ALERT "bnx2i: unknown opcode 0x%x\n",
2062					  nopin->op_code);
2063		}
2064
2065		ADD_STATS_64(hba, rx_pdus, 1);
2066		ADD_STATS_64(hba, rx_bytes, nopin->data_length);
2067done:
2068		if (!tgt_async_msg) {
2069			if (!atomic_read(&bnx2i_conn->ep->num_active_cmds))
2070				printk(KERN_ALERT "bnx2i (%s): no active cmd! "
2071				       "op 0x%x\n",
2072				       hba->netdev->name,
2073				       nopin->op_code);
2074			else
2075				atomic_dec(&bnx2i_conn->ep->num_active_cmds);
2076		}
2077cqe_out:
2078		/* clear out in production version only, till beta keep opcode
2079		 * field intact, will be helpful in debugging (context dump)
2080		 * nopin->op_code = 0;
2081		 */
2082		cqe_cnt++;
2083		qp->cqe_exp_seq_sn++;
2084		if (qp->cqe_exp_seq_sn == (qp->cqe_size * 2 + 1))
2085			qp->cqe_exp_seq_sn = ISCSI_INITIAL_SN;
2086
2087		if (qp->cq_cons_qe == qp->cq_last_qe) {
2088			qp->cq_cons_qe = qp->cq_first_qe;
2089			qp->cq_cons_idx = 0;
2090		} else {
2091			qp->cq_cons_qe++;
2092			qp->cq_cons_idx++;
2093		}
2094	}
2095out:
2096	return cqe_cnt;
2097}
2098
2099/**
2100 * bnx2i_fastpath_notification - process global event queue (KCQ)
2101 * @hba:		adapter structure pointer
2102 * @new_cqe_kcqe:	pointer to newly DMA'ed KCQE entry
2103 *
2104 * Fast path event notification handler, KCQ entry carries context id
2105 *	of the connection that has 1 or more pending CQ entries
2106 */
2107static void bnx2i_fastpath_notification(struct bnx2i_hba *hba,
2108					struct iscsi_kcqe *new_cqe_kcqe)
2109{
2110	struct bnx2i_conn *bnx2i_conn;
2111	u32 iscsi_cid;
2112	int nxt_idx;
2113
2114	iscsi_cid = new_cqe_kcqe->iscsi_conn_id;
2115	bnx2i_conn = bnx2i_get_conn_from_id(hba, iscsi_cid);
2116
2117	if (!bnx2i_conn) {
2118		printk(KERN_ALERT "cid #%x not valid\n", iscsi_cid);
2119		return;
2120	}
2121	if (!bnx2i_conn->ep) {
2122		printk(KERN_ALERT "cid #%x - ep not bound\n", iscsi_cid);
2123		return;
2124	}
2125
2126	bnx2i_process_new_cqes(bnx2i_conn);
2127	nxt_idx = bnx2i_arm_cq_event_coalescing(bnx2i_conn->ep,
2128						CNIC_ARM_CQE_FP);
2129	if (nxt_idx && nxt_idx == bnx2i_process_new_cqes(bnx2i_conn))
2130		bnx2i_arm_cq_event_coalescing(bnx2i_conn->ep, CNIC_ARM_CQE_FP);
2131}
2132
2133
2134/**
2135 * bnx2i_process_update_conn_cmpl - process iscsi conn update completion KCQE
2136 * @hba:		adapter structure pointer
2137 * @update_kcqe:	kcqe pointer
2138 *
2139 * CONN_UPDATE completion handler, this completes iSCSI connection FFP migration
2140 */
2141static void bnx2i_process_update_conn_cmpl(struct bnx2i_hba *hba,
2142					   struct iscsi_kcqe *update_kcqe)
2143{
2144	struct bnx2i_conn *conn;
2145	u32 iscsi_cid;
2146
2147	iscsi_cid = update_kcqe->iscsi_conn_id;
2148	conn = bnx2i_get_conn_from_id(hba, iscsi_cid);
2149
2150	if (!conn) {
2151		printk(KERN_ALERT "conn_update: cid %x not valid\n", iscsi_cid);
2152		return;
2153	}
2154	if (!conn->ep) {
2155		printk(KERN_ALERT "cid %x does not have ep bound\n", iscsi_cid);
2156		return;
2157	}
2158
2159	if (update_kcqe->completion_status) {
2160		printk(KERN_ALERT "request failed cid %x\n", iscsi_cid);
2161		conn->ep->state = EP_STATE_ULP_UPDATE_FAILED;
2162	} else
2163		conn->ep->state = EP_STATE_ULP_UPDATE_COMPL;
2164
2165	wake_up_interruptible(&conn->ep->ofld_wait);
2166}
2167
2168
2169/**
2170 * bnx2i_recovery_que_add_conn - add connection to recovery queue
2171 * @hba:		adapter structure pointer
2172 * @bnx2i_conn:		iscsi connection
2173 *
2174 * Add connection to recovery queue and schedule adapter eh worker
2175 */
2176static void bnx2i_recovery_que_add_conn(struct bnx2i_hba *hba,
2177					struct bnx2i_conn *bnx2i_conn)
2178{
2179	iscsi_conn_failure(bnx2i_conn->cls_conn->dd_data,
2180			   ISCSI_ERR_CONN_FAILED);
2181}
2182
2183
2184/**
2185 * bnx2i_process_tcp_error - process error notification on a given connection
2186 *
2187 * @hba: 		adapter structure pointer
2188 * @tcp_err: 		tcp error kcqe pointer
2189 *
2190 * handles tcp level error notifications from FW.
2191 */
2192static void bnx2i_process_tcp_error(struct bnx2i_hba *hba,
2193				    struct iscsi_kcqe *tcp_err)
2194{
2195	struct bnx2i_conn *bnx2i_conn;
2196	u32 iscsi_cid;
2197
2198	iscsi_cid = tcp_err->iscsi_conn_id;
2199	bnx2i_conn = bnx2i_get_conn_from_id(hba, iscsi_cid);
2200
2201	if (!bnx2i_conn) {
2202		printk(KERN_ALERT "bnx2i - cid 0x%x not valid\n", iscsi_cid);
2203		return;
2204	}
2205
2206	printk(KERN_ALERT "bnx2i - cid 0x%x had TCP errors, error code 0x%x\n",
2207			  iscsi_cid, tcp_err->completion_status);
2208	bnx2i_recovery_que_add_conn(bnx2i_conn->hba, bnx2i_conn);
2209}
2210
2211
2212/**
2213 * bnx2i_process_iscsi_error - process error notification on a given connection
2214 * @hba:		adapter structure pointer
2215 * @iscsi_err:		iscsi error kcqe pointer
2216 *
2217 * handles iscsi error notifications from the FW. Firmware based in initial
2218 *	handshake classifies iscsi protocol / TCP rfc violation into either
2219 *	warning or error indications. If indication is of "Error" type, driver
2220 *	will initiate session recovery for that connection/session. For
2221 *	"Warning" type indication, driver will put out a system log message
2222 *	(there will be only one message for each type for the life of the
2223 *	session, this is to avoid un-necessarily overloading the system)
2224 */
2225static void bnx2i_process_iscsi_error(struct bnx2i_hba *hba,
2226				      struct iscsi_kcqe *iscsi_err)
2227{
2228	struct bnx2i_conn *bnx2i_conn;
2229	u32 iscsi_cid;
2230	char warn_notice[] = "iscsi_warning";
2231	char error_notice[] = "iscsi_error";
2232	char additional_notice[64];
2233	char *message;
2234	int need_recovery;
2235	u64 err_mask64;
2236
2237	iscsi_cid = iscsi_err->iscsi_conn_id;
2238	bnx2i_conn = bnx2i_get_conn_from_id(hba, iscsi_cid);
2239	if (!bnx2i_conn) {
2240		printk(KERN_ALERT "bnx2i - cid 0x%x not valid\n", iscsi_cid);
2241		return;
2242	}
2243
2244	err_mask64 = (0x1ULL << iscsi_err->completion_status);
2245
2246	if (err_mask64 & iscsi_error_mask) {
2247		need_recovery = 0;
2248		message = warn_notice;
2249	} else {
2250		need_recovery = 1;
2251		message = error_notice;
2252	}
2253
2254	switch (iscsi_err->completion_status) {
2255	case ISCSI_KCQE_COMPLETION_STATUS_HDR_DIG_ERR:
2256		strcpy(additional_notice, "hdr digest err");
2257		break;
2258	case ISCSI_KCQE_COMPLETION_STATUS_DATA_DIG_ERR:
2259		strcpy(additional_notice, "data digest err");
2260		break;
2261	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_OPCODE:
2262		strcpy(additional_notice, "wrong opcode rcvd");
2263		break;
2264	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_AHS_LEN:
2265		strcpy(additional_notice, "AHS len > 0 rcvd");
2266		break;
2267	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_ITT:
2268		strcpy(additional_notice, "invalid ITT rcvd");
2269		break;
2270	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_STATSN:
2271		strcpy(additional_notice, "wrong StatSN rcvd");
2272		break;
2273	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_EXP_DATASN:
2274		strcpy(additional_notice, "wrong DataSN rcvd");
2275		break;
2276	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_PEND_R2T:
2277		strcpy(additional_notice, "pend R2T violation");
2278		break;
2279	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_O_U_0:
2280		strcpy(additional_notice, "ERL0, UO");
2281		break;
2282	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_O_U_1:
2283		strcpy(additional_notice, "ERL0, U1");
2284		break;
2285	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_O_U_2:
2286		strcpy(additional_notice, "ERL0, U2");
2287		break;
2288	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_O_U_3:
2289		strcpy(additional_notice, "ERL0, U3");
2290		break;
2291	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_O_U_4:
2292		strcpy(additional_notice, "ERL0, U4");
2293		break;
2294	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_O_U_5:
2295		strcpy(additional_notice, "ERL0, U5");
2296		break;
2297	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_O_U_6:
2298		strcpy(additional_notice, "ERL0, U6");
2299		break;
2300	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_REMAIN_RCV_LEN:
2301		strcpy(additional_notice, "invalid resi len");
2302		break;
2303	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_MAX_RCV_PDU_LEN:
2304		strcpy(additional_notice, "MRDSL violation");
2305		break;
2306	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_F_BIT_ZERO:
2307		strcpy(additional_notice, "F-bit not set");
2308		break;
2309	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_TTT_NOT_RSRV:
2310		strcpy(additional_notice, "invalid TTT");
2311		break;
2312	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_DATASN:
2313		strcpy(additional_notice, "invalid DataSN");
2314		break;
2315	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_REMAIN_BURST_LEN:
2316		strcpy(additional_notice, "burst len violation");
2317		break;
2318	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_BUFFER_OFF:
2319		strcpy(additional_notice, "buf offset violation");
2320		break;
2321	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_LUN:
2322		strcpy(additional_notice, "invalid LUN field");
2323		break;
2324	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_R2TSN:
2325		strcpy(additional_notice, "invalid R2TSN field");
2326		break;
2327#define BNX2I_ERR_DESIRED_DATA_TRNS_LEN_0 	\
2328	ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_DESIRED_DATA_TRNS_LEN_0
2329	case BNX2I_ERR_DESIRED_DATA_TRNS_LEN_0:
2330		strcpy(additional_notice, "invalid cmd len1");
2331		break;
2332#define BNX2I_ERR_DESIRED_DATA_TRNS_LEN_1 	\
2333	ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_DESIRED_DATA_TRNS_LEN_1
2334	case BNX2I_ERR_DESIRED_DATA_TRNS_LEN_1:
2335		strcpy(additional_notice, "invalid cmd len2");
2336		break;
2337	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_PEND_R2T_EXCEED:
2338		strcpy(additional_notice,
2339		       "pend r2t exceeds MaxOutstandingR2T value");
2340		break;
2341	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_TTT_IS_RSRV:
2342		strcpy(additional_notice, "TTT is rsvd");
2343		break;
2344	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_MAX_BURST_LEN:
2345		strcpy(additional_notice, "MBL violation");
2346		break;
2347#define BNX2I_ERR_DATA_SEG_LEN_NOT_ZERO 	\
2348	ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_DATA_SEG_LEN_NOT_ZERO
2349	case BNX2I_ERR_DATA_SEG_LEN_NOT_ZERO:
2350		strcpy(additional_notice, "data seg len != 0");
2351		break;
2352	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_REJECT_PDU_LEN:
2353		strcpy(additional_notice, "reject pdu len error");
2354		break;
2355	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_ASYNC_PDU_LEN:
2356		strcpy(additional_notice, "async pdu len error");
2357		break;
2358	case ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_NOPIN_PDU_LEN:
2359		strcpy(additional_notice, "nopin pdu len error");
2360		break;
2361#define BNX2_ERR_PEND_R2T_IN_CLEANUP			\
2362	ISCSI_KCQE_COMPLETION_STATUS_PROTOCOL_ERR_PEND_R2T_IN_CLEANUP
2363	case BNX2_ERR_PEND_R2T_IN_CLEANUP:
2364		strcpy(additional_notice, "pend r2t in cleanup");
2365		break;
2366
2367	case ISCI_KCQE_COMPLETION_STATUS_TCP_ERROR_IP_FRAGMENT:
2368		strcpy(additional_notice, "IP fragments rcvd");
2369		break;
2370	case ISCI_KCQE_COMPLETION_STATUS_TCP_ERROR_IP_OPTIONS:
2371		strcpy(additional_notice, "IP options error");
2372		break;
2373	case ISCI_KCQE_COMPLETION_STATUS_TCP_ERROR_URGENT_FLAG:
2374		strcpy(additional_notice, "urgent flag error");
2375		break;
2376	default:
2377		printk(KERN_ALERT "iscsi_err - unknown err %x\n",
2378				  iscsi_err->completion_status);
2379	}
2380
2381	if (need_recovery) {
2382		iscsi_conn_printk(KERN_ALERT,
2383				  bnx2i_conn->cls_conn->dd_data,
2384				  "bnx2i: %s - %s\n",
2385				  message, additional_notice);
2386
2387		iscsi_conn_printk(KERN_ALERT,
2388				  bnx2i_conn->cls_conn->dd_data,
2389				  "conn_err - hostno %d conn %p, "
2390				  "iscsi_cid %x cid %x\n",
2391				  bnx2i_conn->hba->shost->host_no,
2392				  bnx2i_conn, bnx2i_conn->ep->ep_iscsi_cid,
2393				  bnx2i_conn->ep->ep_cid);
2394		bnx2i_recovery_que_add_conn(bnx2i_conn->hba, bnx2i_conn);
2395	} else
2396		if (!test_and_set_bit(iscsi_err->completion_status,
2397				      (void *) &bnx2i_conn->violation_notified))
2398			iscsi_conn_printk(KERN_ALERT,
2399					  bnx2i_conn->cls_conn->dd_data,
2400					  "bnx2i: %s - %s\n",
2401					  message, additional_notice);
2402}
2403
2404
2405/**
2406 * bnx2i_process_conn_destroy_cmpl - process iscsi conn destroy completion
2407 * @hba:		adapter structure pointer
2408 * @conn_destroy:	conn destroy kcqe pointer
2409 *
2410 * handles connection destroy completion request.
2411 */
2412static void bnx2i_process_conn_destroy_cmpl(struct bnx2i_hba *hba,
2413					    struct iscsi_kcqe *conn_destroy)
2414{
2415	struct bnx2i_endpoint *ep;
2416
2417	ep = bnx2i_find_ep_in_destroy_list(hba, conn_destroy->iscsi_conn_id);
2418	if (!ep) {
2419		printk(KERN_ALERT "bnx2i_conn_destroy_cmpl: no pending "
2420				  "offload request, unexpected complection\n");
2421		return;
2422	}
2423
2424	if (hba != ep->hba) {
2425		printk(KERN_ALERT "conn destroy- error hba mis-match\n");
2426		return;
2427	}
2428
2429	if (conn_destroy->completion_status) {
2430		printk(KERN_ALERT "conn_destroy_cmpl: op failed\n");
2431		ep->state = EP_STATE_CLEANUP_FAILED;
2432	} else
2433		ep->state = EP_STATE_CLEANUP_CMPL;
2434	wake_up_interruptible(&ep->ofld_wait);
2435}
2436
2437
2438/**
2439 * bnx2i_process_ofld_cmpl - process initial iscsi conn offload completion
2440 * @hba:		adapter structure pointer
2441 * @ofld_kcqe:		conn offload kcqe pointer
2442 *
2443 * handles initial connection offload completion, ep_connect() thread is
2444 *	woken-up to continue with LLP connect process
2445 */
2446static void bnx2i_process_ofld_cmpl(struct bnx2i_hba *hba,
2447				    struct iscsi_kcqe *ofld_kcqe)
2448{
2449	u32 cid_addr;
2450	struct bnx2i_endpoint *ep;
2451	u32 cid_num;
2452
2453	ep = bnx2i_find_ep_in_ofld_list(hba, ofld_kcqe->iscsi_conn_id);
2454	if (!ep) {
2455		printk(KERN_ALERT "ofld_cmpl: no pend offload request\n");
2456		return;
2457	}
2458
2459	if (hba != ep->hba) {
2460		printk(KERN_ALERT "ofld_cmpl: error hba mis-match\n");
2461		return;
2462	}
2463
2464	if (ofld_kcqe->completion_status) {
2465		ep->state = EP_STATE_OFLD_FAILED;
2466		if (ofld_kcqe->completion_status ==
2467		    ISCSI_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE)
2468			printk(KERN_ALERT "bnx2i (%s): ofld1 cmpl - unable "
2469				"to allocate iSCSI context resources\n",
2470				hba->netdev->name);
2471		else if (ofld_kcqe->completion_status ==
2472			 ISCSI_KCQE_COMPLETION_STATUS_INVALID_OPCODE)
2473			printk(KERN_ALERT "bnx2i (%s): ofld1 cmpl - invalid "
2474				"opcode\n", hba->netdev->name);
2475		else if (ofld_kcqe->completion_status ==
2476			 ISCSI_KCQE_COMPLETION_STATUS_CID_BUSY)
2477			/* error status code valid only for 5771x chipset */
2478			ep->state = EP_STATE_OFLD_FAILED_CID_BUSY;
2479		else
2480			printk(KERN_ALERT "bnx2i (%s): ofld1 cmpl - invalid "
2481				"error code %d\n", hba->netdev->name,
2482				ofld_kcqe->completion_status);
2483	} else {
2484		ep->state = EP_STATE_OFLD_COMPL;
2485		cid_addr = ofld_kcqe->iscsi_conn_context_id;
2486		cid_num = bnx2i_get_cid_num(ep);
2487		ep->ep_cid = cid_addr;
2488		ep->qp.ctx_base = NULL;
2489	}
2490	wake_up_interruptible(&ep->ofld_wait);
2491}
2492
2493/**
2494 * bnx2i_indicate_kcqe - process iscsi conn update completion KCQE
2495 * @hba:		adapter structure pointer
2496 * @update_kcqe:	kcqe pointer
2497 *
2498 * Generic KCQ event handler/dispatcher
2499 */
2500static void bnx2i_indicate_kcqe(void *context, struct kcqe *kcqe[],
2501				u32 num_cqe)
2502{
2503	struct bnx2i_hba *hba = context;
2504	int i = 0;
2505	struct iscsi_kcqe *ikcqe = NULL;
2506
2507	while (i < num_cqe) {
2508		ikcqe = (struct iscsi_kcqe *) kcqe[i++];
2509
2510		if (ikcqe->op_code ==
2511		    ISCSI_KCQE_OPCODE_CQ_EVENT_NOTIFICATION)
2512			bnx2i_fastpath_notification(hba, ikcqe);
2513		else if (ikcqe->op_code == ISCSI_KCQE_OPCODE_OFFLOAD_CONN)
2514			bnx2i_process_ofld_cmpl(hba, ikcqe);
2515		else if (ikcqe->op_code == ISCSI_KCQE_OPCODE_UPDATE_CONN)
2516			bnx2i_process_update_conn_cmpl(hba, ikcqe);
2517		else if (ikcqe->op_code == ISCSI_KCQE_OPCODE_INIT) {
2518			if (ikcqe->completion_status !=
2519			    ISCSI_KCQE_COMPLETION_STATUS_SUCCESS)
2520				bnx2i_iscsi_license_error(hba, ikcqe->\
2521							  completion_status);
2522			else {
2523				set_bit(ADAPTER_STATE_UP, &hba->adapter_state);
2524				bnx2i_get_link_state(hba);
2525				printk(KERN_INFO "bnx2i [%.2x:%.2x.%.2x]: "
2526						 "ISCSI_INIT passed\n",
2527						 (u8)hba->pcidev->bus->number,
2528						 hba->pci_devno,
2529						 (u8)hba->pci_func);
2530
2531
2532			}
2533		} else if (ikcqe->op_code == ISCSI_KCQE_OPCODE_DESTROY_CONN)
2534			bnx2i_process_conn_destroy_cmpl(hba, ikcqe);
2535		else if (ikcqe->op_code == ISCSI_KCQE_OPCODE_ISCSI_ERROR)
2536			bnx2i_process_iscsi_error(hba, ikcqe);
2537		else if (ikcqe->op_code == ISCSI_KCQE_OPCODE_TCP_ERROR)
2538			bnx2i_process_tcp_error(hba, ikcqe);
2539		else
2540			printk(KERN_ALERT "bnx2i: unknown opcode 0x%x\n",
2541					  ikcqe->op_code);
2542	}
2543}
2544
2545
2546/**
2547 * bnx2i_indicate_netevent - Generic netdev event handler
2548 * @context:	adapter structure pointer
2549 * @event:	event type
2550 * @vlan_id:	vlans id - associated vlan id with this event
2551 *
2552 * Handles four netdev events, NETDEV_UP, NETDEV_DOWN,
2553 *	NETDEV_GOING_DOWN and NETDEV_CHANGE
2554 */
2555static void bnx2i_indicate_netevent(void *context, unsigned long event,
2556				    u16 vlan_id)
2557{
2558	struct bnx2i_hba *hba = context;
2559
2560	/* Ignore all netevent coming from vlans */
2561	if (vlan_id != 0)
2562		return;
2563
2564	switch (event) {
2565	case NETDEV_UP:
2566		if (!test_bit(ADAPTER_STATE_UP, &hba->adapter_state))
2567			bnx2i_send_fw_iscsi_init_msg(hba);
2568		break;
2569	case NETDEV_DOWN:
2570		clear_bit(ADAPTER_STATE_GOING_DOWN, &hba->adapter_state);
2571		clear_bit(ADAPTER_STATE_UP, &hba->adapter_state);
2572		break;
2573	case NETDEV_GOING_DOWN:
2574		set_bit(ADAPTER_STATE_GOING_DOWN, &hba->adapter_state);
2575		iscsi_host_for_each_session(hba->shost,
2576					    bnx2i_drop_session);
2577		break;
2578	case NETDEV_CHANGE:
2579		bnx2i_get_link_state(hba);
2580		break;
2581	default:
2582		;
2583	}
2584}
2585
2586
2587/**
2588 * bnx2i_cm_connect_cmpl - process iscsi conn establishment completion
2589 * @cm_sk: 		cnic sock structure pointer
2590 *
2591 * function callback exported via bnx2i - cnic driver interface to
2592 *	indicate completion of option-2 TCP connect request.
2593 */
2594static void bnx2i_cm_connect_cmpl(struct cnic_sock *cm_sk)
2595{
2596	struct bnx2i_endpoint *ep = (struct bnx2i_endpoint *) cm_sk->context;
2597
2598	if (test_bit(ADAPTER_STATE_GOING_DOWN, &ep->hba->adapter_state))
2599		ep->state = EP_STATE_CONNECT_FAILED;
2600	else if (test_bit(SK_F_OFFLD_COMPLETE, &cm_sk->flags))
2601		ep->state = EP_STATE_CONNECT_COMPL;
2602	else
2603		ep->state = EP_STATE_CONNECT_FAILED;
2604
2605	wake_up_interruptible(&ep->ofld_wait);
2606}
2607
2608
2609/**
2610 * bnx2i_cm_close_cmpl - process tcp conn close completion
2611 * @cm_sk:	cnic sock structure pointer
2612 *
2613 * function callback exported via bnx2i - cnic driver interface to
2614 *	indicate completion of option-2 graceful TCP connect shutdown
2615 */
2616static void bnx2i_cm_close_cmpl(struct cnic_sock *cm_sk)
2617{
2618	struct bnx2i_endpoint *ep = (struct bnx2i_endpoint *) cm_sk->context;
2619
2620	ep->state = EP_STATE_DISCONN_COMPL;
2621	wake_up_interruptible(&ep->ofld_wait);
2622}
2623
2624
2625/**
2626 * bnx2i_cm_abort_cmpl - process abortive tcp conn teardown completion
2627 * @cm_sk:	cnic sock structure pointer
2628 *
2629 * function callback exported via bnx2i - cnic driver interface to
2630 *	indicate completion of option-2 abortive TCP connect termination
2631 */
2632static void bnx2i_cm_abort_cmpl(struct cnic_sock *cm_sk)
2633{
2634	struct bnx2i_endpoint *ep = (struct bnx2i_endpoint *) cm_sk->context;
2635
2636	ep->state = EP_STATE_DISCONN_COMPL;
2637	wake_up_interruptible(&ep->ofld_wait);
2638}
2639
2640
2641/**
2642 * bnx2i_cm_remote_close - process received TCP FIN
2643 * @hba:		adapter structure pointer
2644 * @update_kcqe:	kcqe pointer
2645 *
2646 * function callback exported via bnx2i - cnic driver interface to indicate
2647 *	async TCP events such as FIN
2648 */
2649static void bnx2i_cm_remote_close(struct cnic_sock *cm_sk)
2650{
2651	struct bnx2i_endpoint *ep = (struct bnx2i_endpoint *) cm_sk->context;
2652
2653	ep->state = EP_STATE_TCP_FIN_RCVD;
2654	if (ep->conn)
2655		bnx2i_recovery_que_add_conn(ep->hba, ep->conn);
2656}
2657
2658/**
2659 * bnx2i_cm_remote_abort - process TCP RST and start conn cleanup
2660 * @hba:		adapter structure pointer
2661 * @update_kcqe:	kcqe pointer
2662 *
2663 * function callback exported via bnx2i - cnic driver interface to
2664 *	indicate async TCP events (RST) sent by the peer.
2665 */
2666static void bnx2i_cm_remote_abort(struct cnic_sock *cm_sk)
2667{
2668	struct bnx2i_endpoint *ep = (struct bnx2i_endpoint *) cm_sk->context;
2669	u32 old_state = ep->state;
2670
2671	ep->state = EP_STATE_TCP_RST_RCVD;
2672	if (old_state == EP_STATE_DISCONN_START)
2673		wake_up_interruptible(&ep->ofld_wait);
2674	else
2675		if (ep->conn)
2676			bnx2i_recovery_que_add_conn(ep->hba, ep->conn);
2677}
2678
2679
2680static int bnx2i_send_nl_mesg(void *context, u32 msg_type,
2681			      char *buf, u16 buflen)
2682{
2683	struct bnx2i_hba *hba = context;
2684	int rc;
2685
2686	if (!hba)
2687		return -ENODEV;
2688
2689	rc = iscsi_offload_mesg(hba->shost, &bnx2i_iscsi_transport,
2690				msg_type, buf, buflen);
2691	if (rc)
2692		printk(KERN_ALERT "bnx2i: private nl message send error\n");
2693
2694	return rc;
2695}
2696
2697
2698/**
2699 * bnx2i_cnic_cb - global template of bnx2i - cnic driver interface structure
2700 *			carrying callback function pointers
2701 *
2702 */
2703struct cnic_ulp_ops bnx2i_cnic_cb = {
2704	.cnic_init = bnx2i_ulp_init,
2705	.cnic_exit = bnx2i_ulp_exit,
2706	.cnic_start = bnx2i_start,
2707	.cnic_stop = bnx2i_stop,
2708	.indicate_kcqes = bnx2i_indicate_kcqe,
2709	.indicate_netevent = bnx2i_indicate_netevent,
2710	.cm_connect_complete = bnx2i_cm_connect_cmpl,
2711	.cm_close_complete = bnx2i_cm_close_cmpl,
2712	.cm_abort_complete = bnx2i_cm_abort_cmpl,
2713	.cm_remote_close = bnx2i_cm_remote_close,
2714	.cm_remote_abort = bnx2i_cm_remote_abort,
2715	.iscsi_nl_send_msg = bnx2i_send_nl_mesg,
2716	.cnic_get_stats = bnx2i_get_stats,
2717	.owner = THIS_MODULE
2718};
2719
2720
2721/**
2722 * bnx2i_map_ep_dbell_regs - map connection doorbell registers
2723 * @ep: bnx2i endpoint
2724 *
2725 * maps connection's SQ and RQ doorbell registers, 5706/5708/5709 hosts these
2726 *	register in BAR #0. Whereas in 57710 these register are accessed by
2727 *	mapping BAR #1
2728 */
2729int bnx2i_map_ep_dbell_regs(struct bnx2i_endpoint *ep)
2730{
2731	u32 cid_num;
2732	u32 reg_off;
2733	u32 first_l4l5;
2734	u32 ctx_sz;
2735	u32 config2;
2736	resource_size_t reg_base;
2737
2738	cid_num = bnx2i_get_cid_num(ep);
2739
2740	if (test_bit(BNX2I_NX2_DEV_57710, &ep->hba->cnic_dev_type)) {
2741		reg_base = pci_resource_start(ep->hba->pcidev,
2742					      BNX2X_DOORBELL_PCI_BAR);
2743		reg_off = (1 << BNX2X_DB_SHIFT) * (cid_num & 0x1FFFF);
2744		ep->qp.ctx_base = ioremap_nocache(reg_base + reg_off, 4);
2745		goto arm_cq;
2746	}
2747
2748	if ((test_bit(BNX2I_NX2_DEV_5709, &ep->hba->cnic_dev_type)) &&
2749	    (ep->hba->mail_queue_access == BNX2I_MQ_BIN_MODE)) {
2750		config2 = REG_RD(ep->hba, BNX2_MQ_CONFIG2);
2751		first_l4l5 = config2 & BNX2_MQ_CONFIG2_FIRST_L4L5;
2752		ctx_sz = (config2 & BNX2_MQ_CONFIG2_CONT_SZ) >> 3;
2753		if (ctx_sz)
2754			reg_off = CTX_OFFSET + MAX_CID_CNT * MB_KERNEL_CTX_SIZE
2755				  + BNX2I_570X_PAGE_SIZE_DEFAULT *
2756				  (((cid_num - first_l4l5) / ctx_sz) + 256);
2757		else
2758			reg_off = CTX_OFFSET + (MB_KERNEL_CTX_SIZE * cid_num);
2759	} else
2760		/* 5709 device in normal node and 5706/5708 devices */
2761		reg_off = CTX_OFFSET + (MB_KERNEL_CTX_SIZE * cid_num);
2762
2763	ep->qp.ctx_base = ioremap_nocache(ep->hba->reg_base + reg_off,
2764					  MB_KERNEL_CTX_SIZE);
2765	if (!ep->qp.ctx_base)
2766		return -ENOMEM;
2767
2768arm_cq:
2769	bnx2i_arm_cq_event_coalescing(ep, CNIC_ARM_CQE);
2770	return 0;
2771}
2772