53 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
58 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
61 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
72 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)  in isert_prot_cmd()  argument
75 		cmd->prot_op != TARGET_PROT_NORMAL);  in isert_prot_cmd()
1263 	struct iscsi_cmd *cmd;  in isert_allocate_cmd()  local
1265 	cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);  in isert_allocate_cmd()
1266 	if (!cmd) {  in isert_allocate_cmd()
1270 	isert_cmd = iscsit_priv_cmd(cmd);  in isert_allocate_cmd()
1272 	isert_cmd->iscsi_cmd = cmd;  in isert_allocate_cmd()
1275 	return cmd;  in isert_allocate_cmd()
1280 		      struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,  in isert_handle_scsi_cmd()  argument
1289 	rc = iscsit_setup_scsi_cmd(conn, cmd, buf);  in isert_handle_scsi_cmd()
1293 	imm_data = cmd->immediate_data;  in isert_handle_scsi_cmd()
1294 	imm_data_len = cmd->first_burst_len;  in isert_handle_scsi_cmd()
1295 	unsol_data = cmd->unsolicited_data;  in isert_handle_scsi_cmd()
1296 	data_len = cmd->se_cmd.data_length;  in isert_handle_scsi_cmd()
1299 		cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;  in isert_handle_scsi_cmd()
1300 	rc = iscsit_process_scsi_cmd(conn, cmd, hdr);  in isert_handle_scsi_cmd()
1313 		sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,  in isert_handle_scsi_cmd()
1319 		cmd->se_cmd.t_data_sg = &isert_cmd->sg;  in isert_handle_scsi_cmd()
1320 		cmd->se_cmd.t_data_nents = 1;  in isert_handle_scsi_cmd()
1326 	cmd->write_data_done += imm_data_len;  in isert_handle_scsi_cmd()
1328 	if (cmd->write_data_done == cmd->se_cmd.data_length) {  in isert_handle_scsi_cmd()
1329 		spin_lock_bh(&cmd->istate_lock);  in isert_handle_scsi_cmd()
1330 		cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;  in isert_handle_scsi_cmd()
1331 		cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;  in isert_handle_scsi_cmd()
1332 		spin_unlock_bh(&cmd->istate_lock);  in isert_handle_scsi_cmd()
1336 	rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);  in isert_handle_scsi_cmd()
1339 		iscsit_set_unsoliticed_dataout(cmd);  in isert_handle_scsi_cmd()
1341 		target_put_sess_cmd(&cmd->se_cmd);  in isert_handle_scsi_cmd()
1352 	struct iscsi_cmd *cmd = NULL;  in isert_handle_iscsi_dataout()  local
1357 	rc = iscsit_check_dataout_hdr(conn, buf, &cmd);  in isert_handle_iscsi_dataout()
1360 	else if (!cmd)  in isert_handle_iscsi_dataout()
1365 	if (!cmd->unsolicited_data) {  in isert_handle_iscsi_dataout()
1373 		  unsol_data_len,  cmd->write_data_done,  in isert_handle_iscsi_dataout()
1374 		  cmd->se_cmd.data_length);  in isert_handle_iscsi_dataout()
1376 	sg_off = cmd->write_data_done / PAGE_SIZE;  in isert_handle_iscsi_dataout()
1377 	sg_start = &cmd->se_cmd.t_data_sg[sg_off];  in isert_handle_iscsi_dataout()
1379 	page_off = cmd->write_data_done % PAGE_SIZE;  in isert_handle_iscsi_dataout()
1395 	rc = iscsit_check_dataout_payload(cmd, hdr, false);  in isert_handle_iscsi_dataout()
1413 		     struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,  in isert_handle_nop_out()  argument
1420 	rc = iscsit_setup_nop_out(conn, cmd, hdr);  in isert_handle_nop_out()
1427 	return iscsit_process_nop_out(conn, cmd, hdr);  in isert_handle_nop_out()
1432 		      struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,  in isert_handle_text_cmd()  argument
1440 	rc = iscsit_setup_text_cmd(conn, cmd, hdr);  in isert_handle_text_cmd()
1452 	cmd->text_in_ptr = text_in;  in isert_handle_text_cmd()
1454 	memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length);  in isert_handle_text_cmd()
1456 	return iscsit_process_text_cmd(conn, cmd, hdr);  in isert_handle_text_cmd()
1466 	struct iscsi_cmd *cmd;  in isert_rx_opcode()  local
1480 		cmd = isert_allocate_cmd(conn, rx_desc);  in isert_rx_opcode()
1481 		if (!cmd)  in isert_rx_opcode()
1484 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1490 		ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1494 		cmd = isert_allocate_cmd(conn, rx_desc);  in isert_rx_opcode()
1495 		if (!cmd)  in isert_rx_opcode()
1498 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1499 		ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1507 		cmd = isert_allocate_cmd(conn, rx_desc);  in isert_rx_opcode()
1508 		if (!cmd)  in isert_rx_opcode()
1511 		ret = iscsit_handle_task_mgt_cmd(conn, cmd,  in isert_rx_opcode()
1515 		cmd = isert_allocate_cmd(conn, rx_desc);  in isert_rx_opcode()
1516 		if (!cmd)  in isert_rx_opcode()
1519 		ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);  in isert_rx_opcode()
1523 			cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);  in isert_rx_opcode()
1525 			cmd = isert_allocate_cmd(conn, rx_desc);  in isert_rx_opcode()
1527 		if (!cmd)  in isert_rx_opcode()
1530 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1531 		ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1728 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_put_cmd()  local
1736 	switch (cmd->iscsi_opcode) {  in isert_put_cmd()
1739 		if (!list_empty(&cmd->i_conn_node))  in isert_put_cmd()
1740 			list_del_init(&cmd->i_conn_node);  in isert_put_cmd()
1743 		if (cmd->data_direction == DMA_TO_DEVICE) {  in isert_put_cmd()
1744 			iscsit_stop_dataout_timer(cmd);  in isert_put_cmd()
1752 			    cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {  in isert_put_cmd()
1753 				struct se_cmd *se_cmd = &cmd->se_cmd;  in isert_put_cmd()
1760 		transport_generic_free_cmd(&cmd->se_cmd, 0);  in isert_put_cmd()
1764 		if (!list_empty(&cmd->i_conn_node))  in isert_put_cmd()
1765 			list_del_init(&cmd->i_conn_node);  in isert_put_cmd()
1768 		transport_generic_free_cmd(&cmd->se_cmd, 0);  in isert_put_cmd()
1779 		if (!list_empty(&cmd->i_conn_node))  in isert_put_cmd()
1780 			list_del_init(&cmd->i_conn_node);  in isert_put_cmd()
1788 		if (cmd->se_cmd.se_tfo != NULL) {  in isert_put_cmd()
1790 				 cmd->iscsi_opcode);  in isert_put_cmd()
1791 			transport_generic_free_cmd(&cmd->se_cmd, 0);  in isert_put_cmd()
1798 		iscsit_release_cmd(cmd);  in isert_put_cmd()
1878 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_completion_rdma_write()  local
1879 	struct se_cmd *se_cmd = &cmd->se_cmd;  in isert_completion_rdma_write()
1896 		isert_put_response(isert_conn->conn, cmd);  in isert_completion_rdma_write()
1904 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_completion_rdma_read()  local
1905 	struct se_cmd *se_cmd = &cmd->se_cmd;  in isert_completion_rdma_read()
1916 	iscsit_stop_dataout_timer(cmd);  in isert_completion_rdma_read()
1918 	cmd->write_data_done = wr->data.len;  in isert_completion_rdma_read()
1922 	spin_lock_bh(&cmd->istate_lock);  in isert_completion_rdma_read()
1923 	cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;  in isert_completion_rdma_read()
1924 	cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;  in isert_completion_rdma_read()
1925 	spin_unlock_bh(&cmd->istate_lock);  in isert_completion_rdma_read()
1943 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_do_control_comp()  local
1945 	isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);  in isert_do_control_comp()
1947 	switch (cmd->i_state) {  in isert_do_control_comp()
1949 		iscsit_tmr_post_handler(cmd, cmd->conn);  in isert_do_control_comp()
1952 		cmd->i_state = ISTATE_SENT_STATUS;  in isert_do_control_comp()
1957 		iscsit_logout_post_handler(cmd, cmd->conn);  in isert_do_control_comp()
1960 		isert_err("Unknown i_state %d\n", cmd->i_state);  in isert_do_control_comp()
1972 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_response_completion()  local
1974 	if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||  in isert_response_completion()
1975 	    cmd->i_state == ISTATE_SEND_LOGOUTRSP ||  in isert_response_completion()
1976 	    cmd->i_state == ISTATE_SEND_REJECT ||  in isert_response_completion()
1977 	    cmd->i_state == ISTATE_SEND_TEXTRSP) {  in isert_response_completion()
1985 	cmd->i_state = ISTATE_SENT_STATUS;  in isert_response_completion()
2150 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)  in isert_put_response()  argument
2152 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_response()
2159 	iscsit_build_rsp_pdu(cmd, conn, true, hdr);  in isert_put_response()
2164 	if (cmd->se_cmd.sense_buffer &&  in isert_put_response()
2165 	    ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||  in isert_put_response()
2166 	    (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {  in isert_put_response()
2172 		put_unaligned_be16(cmd->se_cmd.scsi_sense_length,  in isert_put_response()
2173 				   cmd->sense_buffer);  in isert_put_response()
2174 		cmd->se_cmd.scsi_sense_length += sizeof(__be16);  in isert_put_response()
2176 		padding = -(cmd->se_cmd.scsi_sense_length) & 3;  in isert_put_response()
2177 		hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);  in isert_put_response()
2178 		pdu_len = cmd->se_cmd.scsi_sense_length + padding;  in isert_put_response()
2181 				(void *)cmd->sense_buffer, pdu_len,  in isert_put_response()
2199 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)  in isert_aborted_task()  argument
2201 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_aborted_task()
2206 	if (!list_empty(&cmd->i_conn_node))  in isert_aborted_task()
2207 		list_del_init(&cmd->i_conn_node);  in isert_aborted_task()
2210 	if (cmd->data_direction == DMA_TO_DEVICE)  in isert_aborted_task()
2211 		iscsit_stop_dataout_timer(cmd);  in isert_aborted_task()
2237 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,  in isert_put_nopin()  argument
2240 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_nopin()
2245 	iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)  in isert_put_nopin()
2257 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)  in isert_put_logout_rsp()  argument
2259 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_logout_rsp()
2264 	iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)  in isert_put_logout_rsp()
2275 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)  in isert_put_tm_rsp()  argument
2277 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_tm_rsp()
2282 	iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)  in isert_put_tm_rsp()
2293 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)  in isert_put_reject()  argument
2295 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_reject()
2305 	iscsit_build_reject(cmd, conn, hdr);  in isert_put_reject()
2310 			(void *)cmd->buf_ptr, ISCSI_HDR_LEN,  in isert_put_reject()
2326 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)  in isert_put_text_rsp()  argument
2328 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_text_rsp()
2337 	rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);  in isert_put_text_rsp()
2348 		void *txt_rsp_buf = cmd->buf_ptr;  in isert_put_text_rsp()
2371 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_build_rdma_wr()  local
2379 	sg_start = &cmd->se_cmd.t_data_sg[sg_off];  in isert_build_rdma_wr()
2380 	sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);  in isert_build_rdma_wr()
2417 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,  in isert_map_rdma()  argument
2420 	struct se_cmd *se_cmd = &cmd->se_cmd;  in isert_map_rdma()
2421 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_map_rdma()
2431 	offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;  in isert_map_rdma()
2772 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,  in isert_reg_rdma()  argument
2775 	struct se_cmd *se_cmd = &cmd->se_cmd;  in isert_reg_rdma()
2776 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_reg_rdma()
2787 	offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;  in isert_reg_rdma()
2856 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)  in isert_put_datain()  argument
2858 	struct se_cmd *se_cmd = &cmd->se_cmd;  in isert_put_datain()
2859 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_datain()
2870 	rc = device->reg_rdma_mem(conn, cmd, wr);  in isert_put_datain()
2882 		iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)  in isert_put_datain()
2912 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)  in isert_get_dataout()  argument
2914 	struct se_cmd *se_cmd = &cmd->se_cmd;  in isert_get_dataout()
2915 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_get_dataout()
2923 		 isert_cmd, se_cmd->data_length, cmd->write_data_done);  in isert_get_dataout()
2925 	rc = device->reg_rdma_mem(conn, cmd, wr);  in isert_get_dataout()
2942 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)  in isert_immediate_queue()  argument
2944 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_immediate_queue()
2950 		list_del_init(&cmd->i_conn_node);  in isert_immediate_queue()
2955 		ret = isert_put_nopin(cmd, conn, false);  in isert_immediate_queue()
2967 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)  in isert_response_queue()  argument
2974 		ret = isert_put_logout_rsp(cmd, conn);  in isert_response_queue()
2979 		ret = isert_put_nopin(cmd, conn, true);  in isert_response_queue()
2982 		ret = isert_put_tm_rsp(cmd, conn);  in isert_response_queue()
2985 		ret = isert_put_reject(cmd, conn);  in isert_response_queue()
2988 		ret = isert_put_text_rsp(cmd, conn);  in isert_response_queue()
2995 		ret = isert_put_response(conn, cmd);  in isert_response_queue()
3321 	struct iscsi_cmd *cmd, *tmp;  in isert_put_unsol_pending_cmds()  local
3325 	list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {  in isert_put_unsol_pending_cmds()
3326 		if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&  in isert_put_unsol_pending_cmds()
3327 		    (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&  in isert_put_unsol_pending_cmds()
3328 		    (cmd->write_data_done < cmd->se_cmd.data_length))  in isert_put_unsol_pending_cmds()
3329 			list_move_tail(&cmd->i_conn_node, &drop_cmd_list);  in isert_put_unsol_pending_cmds()
3333 	list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {  in isert_put_unsol_pending_cmds()
3334 		list_del_init(&cmd->i_conn_node);  in isert_put_unsol_pending_cmds()
3335 		if (cmd->i_state != ISTATE_REMOVE) {  in isert_put_unsol_pending_cmds()
3336 			struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_unsol_pending_cmds()
3338 			isert_info("conn %p dropping cmd %p\n", conn, cmd);  in isert_put_unsol_pending_cmds()