51 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
56 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
1047 		       struct isert_cmd *isert_cmd,  in isert_create_send_desc()  argument
1060 	tx_desc->isert_cmd = isert_cmd;  in isert_create_send_desc()
1096 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_init_send_wr()  argument
1099 	struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;  in isert_init_send_wr()
1101 	isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;  in isert_init_send_wr()
1102 	send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_init_send_wr()
1105 	send_wr->num_sge = isert_cmd->tx_desc.num_sge;  in isert_init_send_wr()
1262 	struct isert_cmd *isert_cmd;  in isert_allocate_cmd()  local
1270 	isert_cmd = iscsit_priv_cmd(cmd);  in isert_allocate_cmd()
1271 	isert_cmd->conn = isert_conn;  in isert_allocate_cmd()
1272 	isert_cmd->iscsi_cmd = cmd;  in isert_allocate_cmd()
1273 	isert_cmd->rx_desc = rx_desc;  in isert_allocate_cmd()
1280 		      struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,  in isert_handle_scsi_cmd()  argument
1318 		sg_init_table(&isert_cmd->sg, 1);  in isert_handle_scsi_cmd()
1319 		cmd->se_cmd.t_data_sg = &isert_cmd->sg;  in isert_handle_scsi_cmd()
1321 		sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len);  in isert_handle_scsi_cmd()
1412 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_handle_nop_out()  argument
1431 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_handle_text_cmd()  argument
1467 	struct isert_cmd *isert_cmd;  in isert_rx_opcode()  local
1484 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1485 		isert_cmd->read_stag = read_stag;  in isert_rx_opcode()
1486 		isert_cmd->read_va = read_va;  in isert_rx_opcode()
1487 		isert_cmd->write_stag = write_stag;  in isert_rx_opcode()
1488 		isert_cmd->write_va = write_va;  in isert_rx_opcode()
1490 		ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1498 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1499 		ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1530 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1531 		ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1629 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_map_data_buf()  argument
1656 		  isert_cmd, data->dma_nents, data->sg, data->nents, data->len);  in isert_map_data_buf()
1673 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)  in isert_unmap_cmd()  argument
1675 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unmap_cmd()
1677 	isert_dbg("Cmd %p\n", isert_cmd);  in isert_unmap_cmd()
1680 		isert_dbg("Cmd %p unmap_sg op\n", isert_cmd);  in isert_unmap_cmd()
1685 		isert_dbg("Cmd %p free send_wr\n", isert_cmd);  in isert_unmap_cmd()
1691 		isert_dbg("Cmd %p free ib_sge\n", isert_cmd);  in isert_unmap_cmd()
1698 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)  in isert_unreg_rdma()  argument
1700 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unreg_rdma()
1702 	isert_dbg("Cmd %p\n", isert_cmd);  in isert_unreg_rdma()
1705 		isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc);  in isert_unreg_rdma()
1717 		isert_dbg("Cmd %p unmap_sg op\n", isert_cmd);  in isert_unreg_rdma()
1726 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)  in isert_put_cmd()  argument
1728 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_put_cmd()
1729 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_put_cmd()
1734 	isert_dbg("Cmd %p\n", isert_cmd);  in isert_put_cmd()
1759 		device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_put_cmd()
1773 		hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;  in isert_put_cmd()
1815 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,  in isert_completion_put()  argument
1818 	if (isert_cmd->pdu_buf_dma != 0) {  in isert_completion_put()
1820 		ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,  in isert_completion_put()
1821 				    isert_cmd->pdu_buf_len, DMA_TO_DEVICE);  in isert_completion_put()
1822 		isert_cmd->pdu_buf_dma = 0;  in isert_completion_put()
1826 	isert_put_cmd(isert_cmd, comp_err);  in isert_completion_put()
1875 			    struct isert_cmd *isert_cmd)  in isert_completion_rdma_write()  argument
1877 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_write()
1878 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_completion_rdma_write()
1880 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_completion_rdma_write()
1890 	device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_completion_rdma_write()
1901 			   struct isert_cmd *isert_cmd)  in isert_completion_rdma_read()  argument
1903 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_read()
1904 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_completion_rdma_read()
1906 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_completion_rdma_read()
1917 	device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_completion_rdma_read()
1921 	isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);  in isert_completion_rdma_read()
1939 	struct isert_cmd *isert_cmd = container_of(work,  in isert_do_control_comp()  local
1940 			struct isert_cmd, comp_work);  in isert_do_control_comp()
1941 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_do_control_comp()
1943 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_do_control_comp()
1945 	isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);  in isert_do_control_comp()
1953 		isert_completion_put(&isert_cmd->tx_desc, isert_cmd,  in isert_do_control_comp()
1968 			  struct isert_cmd *isert_cmd,  in isert_response_completion()  argument
1972 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_response_completion()
1980 		INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);  in isert_response_completion()
1981 		queue_work(isert_comp_wq, &isert_cmd->comp_work);  in isert_response_completion()
1986 	isert_completion_put(tx_desc, isert_cmd, ib_dev, false);  in isert_response_completion()
1994 	struct isert_cmd *isert_cmd = tx_desc->isert_cmd;  in isert_snd_completion()  local
1997 	if (!isert_cmd) {  in isert_snd_completion()
2001 	wr = &isert_cmd->rdma_wr;  in isert_snd_completion()
2003 	isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op);  in isert_snd_completion()
2007 		isert_response_completion(tx_desc, isert_cmd,  in isert_snd_completion()
2011 		isert_completion_rdma_write(tx_desc, isert_cmd);  in isert_snd_completion()
2014 		isert_completion_rdma_read(tx_desc, isert_cmd);  in isert_snd_completion()
2056 		struct isert_cmd *isert_cmd;  in isert_cq_comp_err()  local
2060 		isert_cmd = desc->isert_cmd;  in isert_cq_comp_err()
2061 		if (!isert_cmd)  in isert_cq_comp_err()
2064 			isert_completion_put(desc, isert_cmd, ib_dev, true);  in isert_cq_comp_err()
2129 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)  in isert_post_response()  argument
2134 	ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);  in isert_post_response()
2140 	ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr,  in isert_post_response()
2152 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_response()  local
2154 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_response()
2156 				&isert_cmd->tx_desc.iscsi_header;  in isert_put_response()
2158 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_response()
2160 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_response()
2169 		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];  in isert_put_response()
2180 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,  in isert_put_response()
2184 		isert_cmd->pdu_buf_len = pdu_len;  in isert_put_response()
2185 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;  in isert_put_response()
2188 		isert_cmd->tx_desc.num_sge = 2;  in isert_put_response()
2191 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_response()
2195 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_response()
2201 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_aborted_task()  local
2213 	device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_aborted_task()
2240 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_nopin()  local
2242 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_nopin()
2244 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_nopin()
2246 			       &isert_cmd->tx_desc.iscsi_header,  in isert_put_nopin()
2248 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_nopin()
2249 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_nopin()
2253 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_nopin()
2259 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_logout_rsp()  local
2261 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_logout_rsp()
2263 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_logout_rsp()
2265 				&isert_cmd->tx_desc.iscsi_header);  in isert_put_logout_rsp()
2266 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_logout_rsp()
2267 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_logout_rsp()
2271 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_logout_rsp()
2277 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_tm_rsp()  local
2279 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_tm_rsp()
2281 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_tm_rsp()
2283 				  &isert_cmd->tx_desc.iscsi_header);  in isert_put_tm_rsp()
2284 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_tm_rsp()
2285 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_tm_rsp()
2289 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_tm_rsp()
2295 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_reject()  local
2297 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_reject()
2300 	struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];  in isert_put_reject()
2302 		(struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;  in isert_put_reject()
2304 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_reject()
2306 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_reject()
2309 	isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,  in isert_put_reject()
2312 	isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;  in isert_put_reject()
2313 	tx_dsg->addr	= isert_cmd->pdu_buf_dma;  in isert_put_reject()
2316 	isert_cmd->tx_desc.num_sge = 2;  in isert_put_reject()
2318 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_reject()
2322 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_reject()
2328 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_text_rsp()  local
2330 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_text_rsp()
2332 		(struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;  in isert_put_text_rsp()
2336 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_text_rsp()
2342 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_text_rsp()
2347 		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];  in isert_put_text_rsp()
2350 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,  in isert_put_text_rsp()
2353 		isert_cmd->pdu_buf_len = txt_rsp_len;  in isert_put_text_rsp()
2354 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;  in isert_put_text_rsp()
2357 		isert_cmd->tx_desc.num_sge = 2;  in isert_put_text_rsp()
2359 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_text_rsp()
2363 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_text_rsp()
2367 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_build_rdma_wr()  argument
2371 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_build_rdma_wr()
2384 	rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_build_rdma_wr()
2421 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_map_rdma()  local
2429 	isert_cmd->tx_desc.isert_cmd = isert_cmd;  in isert_map_rdma()
2432 	ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,  in isert_map_rdma()
2458 	wr->isert_cmd = isert_cmd;  in isert_map_rdma()
2462 		rdma_wr = &isert_cmd->rdma_wr.rdma_wr[i];  in isert_map_rdma()
2468 			rdma_wr->remote_addr = isert_cmd->read_va + offset;  in isert_map_rdma()
2469 			rdma_wr->rkey = isert_cmd->read_stag;  in isert_map_rdma()
2471 				rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr;  in isert_map_rdma()
2476 			rdma_wr->remote_addr = isert_cmd->write_va + va_offset;  in isert_map_rdma()
2477 			rdma_wr->rkey = isert_cmd->write_stag;  in isert_map_rdma()
2484 		ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,  in isert_map_rdma()
2714 		      struct isert_cmd *isert_cmd,  in isert_handle_prot_cmd()  argument
2718 	struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd;  in isert_handle_prot_cmd()
2733 		ret = isert_map_data_buf(isert_conn, isert_cmd,  in isert_handle_prot_cmd()
2776 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_reg_rdma()  local
2785 	isert_cmd->tx_desc.isert_cmd = isert_cmd;  in isert_reg_rdma()
2788 	ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,  in isert_reg_rdma()
2809 		ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr);  in isert_reg_rdma()
2823 	wr->isert_cmd = isert_cmd;  in isert_reg_rdma()
2825 	rdma_wr = &isert_cmd->rdma_wr.s_rdma_wr;  in isert_reg_rdma()
2828 	rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_reg_rdma()
2831 		rdma_wr->remote_addr = isert_cmd->read_va;  in isert_reg_rdma()
2832 		rdma_wr->rkey = isert_cmd->read_stag;  in isert_reg_rdma()
2837 		rdma_wr->remote_addr = isert_cmd->write_va;  in isert_reg_rdma()
2838 		rdma_wr->rkey = isert_cmd->write_stag;  in isert_reg_rdma()
2859 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_datain()  local
2860 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_put_datain()
2867 		 isert_cmd, se_cmd->data_length);  in isert_put_datain()
2872 		isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);  in isert_put_datain()
2880 		isert_create_send_desc(isert_conn, isert_cmd,  in isert_put_datain()
2881 				       &isert_cmd->tx_desc);  in isert_put_datain()
2883 				     &isert_cmd->tx_desc.iscsi_header);  in isert_put_datain()
2884 		isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_datain()
2885 		isert_init_send_wr(isert_conn, isert_cmd,  in isert_put_datain()
2886 				   &isert_cmd->tx_desc.send_wr);  in isert_put_datain()
2887 		isert_cmd->rdma_wr.s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr;  in isert_put_datain()
2890 		rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);  in isert_put_datain()
2903 			 "READ\n", isert_cmd);  in isert_put_datain()
2906 			 isert_cmd);  in isert_put_datain()
2915 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_get_dataout()  local
2916 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_get_dataout()
2923 		 isert_cmd, se_cmd->data_length, cmd->write_data_done);  in isert_get_dataout()
2927 		isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);  in isert_get_dataout()
2936 		 isert_cmd);  in isert_get_dataout()
2944 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_immediate_queue()  local
2952 		isert_put_cmd(isert_cmd, true);  in isert_immediate_queue()
3336 			struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_unsol_pending_cmds()  local
3339 			isert_put_cmd(isert_cmd, true);  in isert_put_unsol_pending_cmds()
3381 	.priv_size		= sizeof(struct isert_cmd),