62 	struct iscsi_cmd *cmd,  in iscsit_add_r2t_to_list()  argument
78 	r2t->r2t_sn = (!r2t_sn) ? cmd->r2t_sn++ : r2t_sn;  in iscsit_add_r2t_to_list()
81 	list_add_tail(&r2t->r2t_list, &cmd->cmd_r2t_list);  in iscsit_add_r2t_to_list()
82 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_add_r2t_to_list()
84 	iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);  in iscsit_add_r2t_to_list()
86 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_add_r2t_to_list()
91 	struct iscsi_cmd *cmd,  in iscsit_get_r2t_for_eos()  argument
97 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_for_eos()
98 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {  in iscsit_get_r2t_for_eos()
101 			spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_for_eos()
105 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_for_eos()
112 struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *cmd)  in iscsit_get_r2t_from_list()  argument
116 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_from_list()
117 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {  in iscsit_get_r2t_from_list()
119 			spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_from_list()
123 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_from_list()
126 			" 0x%08x.\n", cmd->init_task_tag);  in iscsit_get_r2t_from_list()
133 void iscsit_free_r2t(struct iscsi_r2t *r2t, struct iscsi_cmd *cmd)  in iscsit_free_r2t()  argument
139 void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)  in iscsit_free_r2ts_from_list()  argument
143 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_free_r2ts_from_list()
144 	list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list)  in iscsit_free_r2ts_from_list()
145 		iscsit_free_r2t(r2t, cmd);  in iscsit_free_r2ts_from_list()
146 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_free_r2ts_from_list()
155 	struct iscsi_cmd *cmd;  in iscsit_allocate_cmd()  local
164 	cmd = (struct iscsi_cmd *)(se_sess->sess_cmd_map + (tag * size));  in iscsit_allocate_cmd()
165 	memset(cmd, 0, size);  in iscsit_allocate_cmd()
167 	cmd->se_cmd.map_tag = tag;  in iscsit_allocate_cmd()
168 	cmd->conn = conn;  in iscsit_allocate_cmd()
169 	INIT_LIST_HEAD(&cmd->i_conn_node);  in iscsit_allocate_cmd()
170 	INIT_LIST_HEAD(&cmd->datain_list);  in iscsit_allocate_cmd()
171 	INIT_LIST_HEAD(&cmd->cmd_r2t_list);  in iscsit_allocate_cmd()
172 	spin_lock_init(&cmd->datain_lock);  in iscsit_allocate_cmd()
173 	spin_lock_init(&cmd->dataout_timeout_lock);  in iscsit_allocate_cmd()
174 	spin_lock_init(&cmd->istate_lock);  in iscsit_allocate_cmd()
175 	spin_lock_init(&cmd->error_lock);  in iscsit_allocate_cmd()
176 	spin_lock_init(&cmd->r2t_lock);  in iscsit_allocate_cmd()
178 	return cmd;  in iscsit_allocate_cmd()
183 	struct iscsi_cmd *cmd,  in iscsit_get_seq_holder_for_datain()  argument
188 	for (i = 0; i < cmd->seq_count; i++)  in iscsit_get_seq_holder_for_datain()
189 		if (cmd->seq_list[i].seq_send_order == seq_send_order)  in iscsit_get_seq_holder_for_datain()
190 			return &cmd->seq_list[i];  in iscsit_get_seq_holder_for_datain()
195 struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *cmd)  in iscsit_get_seq_holder_for_r2t()  argument
199 	if (!cmd->seq_list) {  in iscsit_get_seq_holder_for_r2t()
204 	for (i = 0; i < cmd->seq_count; i++) {  in iscsit_get_seq_holder_for_r2t()
205 		if (cmd->seq_list[i].type != SEQTYPE_NORMAL)  in iscsit_get_seq_holder_for_r2t()
207 		if (cmd->seq_list[i].seq_send_order == cmd->seq_send_order) {  in iscsit_get_seq_holder_for_r2t()
208 			cmd->seq_send_order++;  in iscsit_get_seq_holder_for_r2t()
209 			return &cmd->seq_list[i];  in iscsit_get_seq_holder_for_r2t()
217 	struct iscsi_cmd *cmd,  in iscsit_get_holder_for_r2tsn()  argument
222 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_get_holder_for_r2tsn()
223 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {  in iscsit_get_holder_for_r2tsn()
225 			spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_holder_for_r2tsn()
229 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_holder_for_r2tsn()
278 int iscsit_sequence_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,  in iscsit_sequence_cmd()  argument
290 		ret = iscsit_execute_cmd(cmd, 0);  in iscsit_sequence_cmd()
299 		ret = iscsit_handle_ooo_cmdsn(conn->sess, cmd, be32_to_cpu(cmdsn));  in iscsit_sequence_cmd()
310 		cmd->i_state = ISTATE_REMOVE;  in iscsit_sequence_cmd()
311 		iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);  in iscsit_sequence_cmd()
323 		iscsit_reject_cmd(cmd, reason, buf);  in iscsit_sequence_cmd()
329 int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)  in iscsit_check_unsolicited_dataout()  argument
331 	struct iscsi_conn *conn = cmd->conn;  in iscsit_check_unsolicited_dataout()
332 	struct se_cmd *se_cmd = &cmd->se_cmd;  in iscsit_check_unsolicited_dataout()
344 	if ((cmd->first_burst_len + payload_length) >  in iscsit_check_unsolicited_dataout()
348 			(cmd->first_burst_len + payload_length),  in iscsit_check_unsolicited_dataout()
358 	if (((cmd->first_burst_len + payload_length) != cmd->se_cmd.data_length) &&  in iscsit_check_unsolicited_dataout()
359 	    ((cmd->first_burst_len + payload_length) !=  in iscsit_check_unsolicited_dataout()
364 			(cmd->first_burst_len + payload_length),  in iscsit_check_unsolicited_dataout()
365 			conn->sess->sess_ops->FirstBurstLength, cmd->se_cmd.data_length);  in iscsit_check_unsolicited_dataout()
377 	struct iscsi_cmd *cmd;  in iscsit_find_cmd_from_itt()  local
380 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {  in iscsit_find_cmd_from_itt()
381 		if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_from_itt()
383 			return cmd;  in iscsit_find_cmd_from_itt()
399 	struct iscsi_cmd *cmd;  in iscsit_find_cmd_from_itt_or_dump()  local
402 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {  in iscsit_find_cmd_from_itt_or_dump()
403 		if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT)  in iscsit_find_cmd_from_itt_or_dump()
405 		if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_from_itt_or_dump()
407 			return cmd;  in iscsit_find_cmd_from_itt_or_dump()
424 	struct iscsi_cmd *cmd = NULL;  in iscsit_find_cmd_from_ttt()  local
427 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {  in iscsit_find_cmd_from_ttt()
428 		if (cmd->targ_xfer_tag == targ_xfer_tag) {  in iscsit_find_cmd_from_ttt()
430 			return cmd;  in iscsit_find_cmd_from_ttt()
446 	struct iscsi_cmd *cmd = NULL;  in iscsit_find_cmd_for_recovery()  local
455 		list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {  in iscsit_find_cmd_for_recovery()
456 			if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_for_recovery()
461 				*cmd_ptr = cmd;  in iscsit_find_cmd_for_recovery()
475 		list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {  in iscsit_find_cmd_for_recovery()
476 			if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_for_recovery()
481 				*cmd_ptr = cmd;  in iscsit_find_cmd_for_recovery()
493 	struct iscsi_cmd *cmd,  in iscsit_add_cmd_to_immediate_queue()  argument
506 	qr->cmd = cmd;  in iscsit_add_cmd_to_immediate_queue()
511 	atomic_inc(&cmd->immed_queue_count);  in iscsit_add_cmd_to_immediate_queue()
531 	if (qr->cmd)  in iscsit_get_cmd_from_immediate_queue()
532 		atomic_dec(&qr->cmd->immed_queue_count);  in iscsit_get_cmd_from_immediate_queue()
539 	struct iscsi_cmd *cmd,  in iscsit_remove_cmd_from_immediate_queue()  argument
545 	if (!atomic_read(&cmd->immed_queue_count)) {  in iscsit_remove_cmd_from_immediate_queue()
551 		if (qr->cmd != cmd)  in iscsit_remove_cmd_from_immediate_queue()
554 		atomic_dec(&qr->cmd->immed_queue_count);  in iscsit_remove_cmd_from_immediate_queue()
560 	if (atomic_read(&cmd->immed_queue_count)) {  in iscsit_remove_cmd_from_immediate_queue()
562 			cmd->init_task_tag,  in iscsit_remove_cmd_from_immediate_queue()
563 			atomic_read(&cmd->immed_queue_count));  in iscsit_remove_cmd_from_immediate_queue()
568 	struct iscsi_cmd *cmd,  in iscsit_add_cmd_to_response_queue()  argument
581 	qr->cmd = cmd;  in iscsit_add_cmd_to_response_queue()
586 	atomic_inc(&cmd->response_queue_count);  in iscsit_add_cmd_to_response_queue()
606 	if (qr->cmd)  in iscsit_get_cmd_from_response_queue()
607 		atomic_dec(&qr->cmd->response_queue_count);  in iscsit_get_cmd_from_response_queue()
614 	struct iscsi_cmd *cmd,  in iscsit_remove_cmd_from_response_queue()  argument
620 	if (!atomic_read(&cmd->response_queue_count)) {  in iscsit_remove_cmd_from_response_queue()
627 		if (qr->cmd != cmd)  in iscsit_remove_cmd_from_response_queue()
630 		atomic_dec(&qr->cmd->response_queue_count);  in iscsit_remove_cmd_from_response_queue()
636 	if (atomic_read(&cmd->response_queue_count)) {  in iscsit_remove_cmd_from_response_queue()
638 			cmd->init_task_tag,  in iscsit_remove_cmd_from_response_queue()
639 			atomic_read(&cmd->response_queue_count));  in iscsit_remove_cmd_from_response_queue()
668 		if (qr->cmd)  in iscsit_free_queue_reqs_for_conn()
669 			atomic_dec(&qr->cmd->immed_queue_count);  in iscsit_free_queue_reqs_for_conn()
679 		if (qr->cmd)  in iscsit_free_queue_reqs_for_conn()
680 			atomic_dec(&qr->cmd->response_queue_count);  in iscsit_free_queue_reqs_for_conn()
687 void iscsit_release_cmd(struct iscsi_cmd *cmd)  in iscsit_release_cmd()  argument
690 	struct se_cmd *se_cmd = &cmd->se_cmd;  in iscsit_release_cmd()
692 	if (cmd->conn)  in iscsit_release_cmd()
693 		sess = cmd->conn->sess;  in iscsit_release_cmd()
695 		sess = cmd->sess;  in iscsit_release_cmd()
699 	kfree(cmd->buf_ptr);  in iscsit_release_cmd()
700 	kfree(cmd->pdu_list);  in iscsit_release_cmd()
701 	kfree(cmd->seq_list);  in iscsit_release_cmd()
702 	kfree(cmd->tmr_req);  in iscsit_release_cmd()
703 	kfree(cmd->iov_data);  in iscsit_release_cmd()
704 	kfree(cmd->text_in_ptr);  in iscsit_release_cmd()
710 void __iscsit_free_cmd(struct iscsi_cmd *cmd, bool scsi_cmd,  in __iscsit_free_cmd()  argument
713 	struct iscsi_conn *conn = cmd->conn;  in __iscsit_free_cmd()
716 		if (cmd->data_direction == DMA_TO_DEVICE) {  in __iscsit_free_cmd()
717 			iscsit_stop_dataout_timer(cmd);  in __iscsit_free_cmd()
718 			iscsit_free_r2ts_from_list(cmd);  in __iscsit_free_cmd()
720 		if (cmd->data_direction == DMA_FROM_DEVICE)  in __iscsit_free_cmd()
721 			iscsit_free_all_datain_reqs(cmd);  in __iscsit_free_cmd()
725 		iscsit_remove_cmd_from_immediate_queue(cmd, conn);  in __iscsit_free_cmd()
726 		iscsit_remove_cmd_from_response_queue(cmd, conn);  in __iscsit_free_cmd()
730 void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)  in iscsit_free_cmd()  argument
738 	switch (cmd->iscsi_opcode) {  in iscsit_free_cmd()
740 		se_cmd = &cmd->se_cmd;  in iscsit_free_cmd()
741 		__iscsit_free_cmd(cmd, true, shutdown);  in iscsit_free_cmd()
746 		rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);  in iscsit_free_cmd()
748 			__iscsit_free_cmd(cmd, true, shutdown);  in iscsit_free_cmd()
758 		if (cmd->se_cmd.se_tfo != NULL) {  in iscsit_free_cmd()
759 			se_cmd = &cmd->se_cmd;  in iscsit_free_cmd()
760 			__iscsit_free_cmd(cmd, true, shutdown);  in iscsit_free_cmd()
762 			rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);  in iscsit_free_cmd()
764 				__iscsit_free_cmd(cmd, true, shutdown);  in iscsit_free_cmd()
771 		__iscsit_free_cmd(cmd, false, shutdown);  in iscsit_free_cmd()
772 		iscsit_release_cmd(cmd);  in iscsit_free_cmd()
884 	struct iscsi_cmd *cmd;  in iscsit_add_nopin()  local
886 	cmd = iscsit_allocate_cmd(conn, TASK_RUNNING);  in iscsit_add_nopin()
887 	if (!cmd)  in iscsit_add_nopin()
890 	cmd->iscsi_opcode = ISCSI_OP_NOOP_IN;  in iscsit_add_nopin()
893 	cmd->init_task_tag = RESERVED_ITT;  in iscsit_add_nopin()
894 	cmd->targ_xfer_tag = (want_response) ?  in iscsit_add_nopin()
897 	list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);  in iscsit_add_nopin()
902 	iscsit_add_cmd_to_immediate_queue(cmd, conn, state);  in iscsit_add_nopin()
1101 	struct iscsi_cmd *cmd,  in iscsit_send_tx_data()  argument
1110 	tx_size = cmd->tx_size;  in iscsit_send_tx_data()
1113 		iov = &cmd->iov_data[0];  in iscsit_send_tx_data()
1114 		iov_count = cmd->iov_data_count;  in iscsit_send_tx_data()
1116 		iov = &cmd->iov_misc[0];  in iscsit_send_tx_data()
1117 		iov_count = cmd->iov_misc_count;  in iscsit_send_tx_data()
1128 	cmd->tx_size = 0;  in iscsit_send_tx_data()
1134 	struct iscsi_cmd *cmd,  in iscsit_fe_sendpage_sg()  argument
1137 	struct scatterlist *sg = cmd->first_data_sg;  in iscsit_fe_sendpage_sg()
1140 	u32 offset = cmd->first_data_sg_off;  in iscsit_fe_sendpage_sg()
1148 	iov.iov_base = cmd->pdu;  in iscsit_fe_sendpage_sg()
1160 	data_len = cmd->tx_size - tx_hdr_size - cmd->padding;  in iscsit_fe_sendpage_sg()
1167 		if (cmd->padding)  in iscsit_fe_sendpage_sg()
1168 			iov_off = (cmd->iov_data_count - 2);  in iscsit_fe_sendpage_sg()
1170 			iov_off = (cmd->iov_data_count - 1);  in iscsit_fe_sendpage_sg()
1172 		iov_off = (cmd->iov_data_count - 1);  in iscsit_fe_sendpage_sg()
1201 	if (cmd->padding) {  in iscsit_fe_sendpage_sg()
1202 		struct kvec *iov_p = &cmd->iov_data[iov_off++];  in iscsit_fe_sendpage_sg()
1204 		tx_sent = tx_data(conn, iov_p, 1, cmd->padding);  in iscsit_fe_sendpage_sg()
1205 		if (cmd->padding != tx_sent) {  in iscsit_fe_sendpage_sg()
1216 		struct kvec *iov_d = &cmd->iov_data[iov_off];  in iscsit_fe_sendpage_sg()