Lines Matching refs:ioctx
619 struct srpt_ioctx *ioctx; in srpt_alloc_ioctx() local
621 ioctx = kmalloc(ioctx_size, GFP_KERNEL); in srpt_alloc_ioctx()
622 if (!ioctx) in srpt_alloc_ioctx()
625 ioctx->buf = kmalloc(dma_size, GFP_KERNEL); in srpt_alloc_ioctx()
626 if (!ioctx->buf) in srpt_alloc_ioctx()
629 ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir); in srpt_alloc_ioctx()
630 if (ib_dma_mapping_error(sdev->device, ioctx->dma)) in srpt_alloc_ioctx()
633 return ioctx; in srpt_alloc_ioctx()
636 kfree(ioctx->buf); in srpt_alloc_ioctx()
638 kfree(ioctx); in srpt_alloc_ioctx()
646 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx, in srpt_free_ioctx() argument
649 if (!ioctx) in srpt_free_ioctx()
652 ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir); in srpt_free_ioctx()
653 kfree(ioctx->buf); in srpt_free_ioctx()
654 kfree(ioctx); in srpt_free_ioctx()
712 static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx) in srpt_get_cmd_state() argument
717 BUG_ON(!ioctx); in srpt_get_cmd_state()
719 spin_lock_irqsave(&ioctx->spinlock, flags); in srpt_get_cmd_state()
720 state = ioctx->state; in srpt_get_cmd_state()
721 spin_unlock_irqrestore(&ioctx->spinlock, flags); in srpt_get_cmd_state()
731 static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx, in srpt_set_cmd_state() argument
737 BUG_ON(!ioctx); in srpt_set_cmd_state()
739 spin_lock_irqsave(&ioctx->spinlock, flags); in srpt_set_cmd_state()
740 previous = ioctx->state; in srpt_set_cmd_state()
742 ioctx->state = new; in srpt_set_cmd_state()
743 spin_unlock_irqrestore(&ioctx->spinlock, flags); in srpt_set_cmd_state()
753 static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx, in srpt_test_and_set_cmd_state() argument
760 WARN_ON(!ioctx); in srpt_test_and_set_cmd_state()
764 spin_lock_irqsave(&ioctx->spinlock, flags); in srpt_test_and_set_cmd_state()
765 previous = ioctx->state; in srpt_test_and_set_cmd_state()
767 ioctx->state = new; in srpt_test_and_set_cmd_state()
768 spin_unlock_irqrestore(&ioctx->spinlock, flags); in srpt_test_and_set_cmd_state()
776 struct srpt_recv_ioctx *ioctx) in srpt_post_recv() argument
782 wr.wr_id = encode_wr_id(SRPT_RECV, ioctx->ioctx.index); in srpt_post_recv()
784 list.addr = ioctx->ioctx.dma; in srpt_post_recv()
801 struct srpt_send_ioctx *ioctx, int len) in srpt_post_send() argument
816 ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, len, in srpt_post_send()
819 list.addr = ioctx->ioctx.dma; in srpt_post_send()
824 wr.wr_id = encode_wr_id(SRPT_SEND, ioctx->ioctx.index); in srpt_post_send()
854 static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx, in srpt_get_desc_tbl() argument
898 ioctx->n_rbuf = 1; in srpt_get_desc_tbl()
899 ioctx->rbufs = &ioctx->single_rbuf; in srpt_get_desc_tbl()
903 memcpy(ioctx->rbufs, db, sizeof *db); in srpt_get_desc_tbl()
910 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db; in srpt_get_desc_tbl()
912 if (ioctx->n_rbuf > in srpt_get_desc_tbl()
920 ioctx->n_rbuf = 0; in srpt_get_desc_tbl()
925 if (ioctx->n_rbuf == 1) in srpt_get_desc_tbl()
926 ioctx->rbufs = &ioctx->single_rbuf; in srpt_get_desc_tbl()
928 ioctx->rbufs = in srpt_get_desc_tbl()
929 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC); in srpt_get_desc_tbl()
930 if (!ioctx->rbufs) { in srpt_get_desc_tbl()
931 ioctx->n_rbuf = 0; in srpt_get_desc_tbl()
938 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db); in srpt_get_desc_tbl()
1049 struct srpt_send_ioctx *ioctx) in srpt_unmap_sg_to_ib_sge() argument
1055 BUG_ON(!ioctx); in srpt_unmap_sg_to_ib_sge()
1056 BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius); in srpt_unmap_sg_to_ib_sge()
1058 while (ioctx->n_rdma) in srpt_unmap_sg_to_ib_sge()
1059 kfree(ioctx->rdma_ius[--ioctx->n_rdma].sge); in srpt_unmap_sg_to_ib_sge()
1061 kfree(ioctx->rdma_ius); in srpt_unmap_sg_to_ib_sge()
1062 ioctx->rdma_ius = NULL; in srpt_unmap_sg_to_ib_sge()
1064 if (ioctx->mapped_sg_count) { in srpt_unmap_sg_to_ib_sge()
1065 sg = ioctx->sg; in srpt_unmap_sg_to_ib_sge()
1067 dir = ioctx->cmd.data_direction; in srpt_unmap_sg_to_ib_sge()
1069 ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt, in srpt_unmap_sg_to_ib_sge()
1071 ioctx->mapped_sg_count = 0; in srpt_unmap_sg_to_ib_sge()
1079 struct srpt_send_ioctx *ioctx) in srpt_map_sg_to_ib_sge() argument
1098 BUG_ON(!ioctx); in srpt_map_sg_to_ib_sge()
1099 cmd = &ioctx->cmd; in srpt_map_sg_to_ib_sge()
1103 ioctx->sg = sg = sg_orig = cmd->t_data_sg; in srpt_map_sg_to_ib_sge()
1104 ioctx->sg_cnt = sg_cnt = cmd->t_data_nents; in srpt_map_sg_to_ib_sge()
1111 ioctx->mapped_sg_count = count; in srpt_map_sg_to_ib_sge()
1113 if (ioctx->rdma_ius && ioctx->n_rdma_ius) in srpt_map_sg_to_ib_sge()
1114 nrdma = ioctx->n_rdma_ius; in srpt_map_sg_to_ib_sge()
1117 + ioctx->n_rbuf; in srpt_map_sg_to_ib_sge()
1119 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu, GFP_KERNEL); in srpt_map_sg_to_ib_sge()
1120 if (!ioctx->rdma_ius) in srpt_map_sg_to_ib_sge()
1123 ioctx->n_rdma_ius = nrdma; in srpt_map_sg_to_ib_sge()
1126 db = ioctx->rbufs; in srpt_map_sg_to_ib_sge()
1129 riu = ioctx->rdma_ius; in srpt_map_sg_to_ib_sge()
1140 j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) { in srpt_map_sg_to_ib_sge()
1172 ++ioctx->n_rdma; in srpt_map_sg_to_ib_sge()
1186 ++ioctx->n_rdma; in srpt_map_sg_to_ib_sge()
1193 db = ioctx->rbufs; in srpt_map_sg_to_ib_sge()
1195 riu = ioctx->rdma_ius; in srpt_map_sg_to_ib_sge()
1202 j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) { in srpt_map_sg_to_ib_sge()
1248 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_map_sg_to_ib_sge()
1258 struct srpt_send_ioctx *ioctx; in srpt_get_send_ioctx() local
1263 ioctx = NULL; in srpt_get_send_ioctx()
1266 ioctx = list_first_entry(&ch->free_list, in srpt_get_send_ioctx()
1268 list_del(&ioctx->free_list); in srpt_get_send_ioctx()
1272 if (!ioctx) in srpt_get_send_ioctx()
1273 return ioctx; in srpt_get_send_ioctx()
1275 BUG_ON(ioctx->ch != ch); in srpt_get_send_ioctx()
1276 spin_lock_init(&ioctx->spinlock); in srpt_get_send_ioctx()
1277 ioctx->state = SRPT_STATE_NEW; in srpt_get_send_ioctx()
1278 ioctx->n_rbuf = 0; in srpt_get_send_ioctx()
1279 ioctx->rbufs = NULL; in srpt_get_send_ioctx()
1280 ioctx->n_rdma = 0; in srpt_get_send_ioctx()
1281 ioctx->n_rdma_ius = 0; in srpt_get_send_ioctx()
1282 ioctx->rdma_ius = NULL; in srpt_get_send_ioctx()
1283 ioctx->mapped_sg_count = 0; in srpt_get_send_ioctx()
1284 init_completion(&ioctx->tx_done); in srpt_get_send_ioctx()
1285 ioctx->queue_status_only = false; in srpt_get_send_ioctx()
1290 memset(&ioctx->cmd, 0, sizeof(ioctx->cmd)); in srpt_get_send_ioctx()
1291 memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data)); in srpt_get_send_ioctx()
1293 return ioctx; in srpt_get_send_ioctx()
1301 static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx) in srpt_abort_cmd() argument
1306 BUG_ON(!ioctx); in srpt_abort_cmd()
1316 spin_lock_irqsave(&ioctx->spinlock, flags); in srpt_abort_cmd()
1317 state = ioctx->state; in srpt_abort_cmd()
1320 ioctx->state = SRPT_STATE_DATA_IN; in srpt_abort_cmd()
1325 ioctx->state = SRPT_STATE_DONE; in srpt_abort_cmd()
1330 spin_unlock_irqrestore(&ioctx->spinlock, flags); in srpt_abort_cmd()
1333 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_abort_cmd()
1337 target_put_sess_cmd(&ioctx->cmd); in srpt_abort_cmd()
1342 ioctx->tag); in srpt_abort_cmd()
1352 WARN_ON(!transport_check_aborted_status(&ioctx->cmd, false)); in srpt_abort_cmd()
1358 spin_lock_irqsave(&ioctx->cmd.t_state_lock, flags); in srpt_abort_cmd()
1359 ioctx->cmd.transport_state &= ~CMD_T_ACTIVE; in srpt_abort_cmd()
1360 spin_unlock_irqrestore(&ioctx->cmd.t_state_lock, flags); in srpt_abort_cmd()
1367 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); in srpt_abort_cmd()
1368 target_put_sess_cmd(&ioctx->cmd); in srpt_abort_cmd()
1371 srpt_set_cmd_state(ioctx, SRPT_STATE_DONE); in srpt_abort_cmd()
1372 target_put_sess_cmd(&ioctx->cmd); in srpt_abort_cmd()
1388 struct srpt_send_ioctx *ioctx; in srpt_handle_send_err_comp() local
1396 ioctx = ch->ioctx_ring[index]; in srpt_handle_send_err_comp()
1397 state = srpt_get_cmd_state(ioctx); in srpt_handle_send_err_comp()
1398 cmd = &ioctx->cmd; in srpt_handle_send_err_comp()
1410 srpt_abort_cmd(ioctx); in srpt_handle_send_err_comp()
1417 struct srpt_send_ioctx *ioctx) in srpt_handle_send_comp() argument
1423 state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE); in srpt_handle_send_comp()
1431 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_handle_send_comp()
1432 transport_generic_free_cmd(&ioctx->cmd, 0); in srpt_handle_send_comp()
1435 " wr_id = %u.\n", ioctx->ioctx.index); in srpt_handle_send_comp()
1448 struct srpt_send_ioctx *ioctx, in srpt_handle_rdma_comp() argument
1451 WARN_ON(ioctx->n_rdma <= 0); in srpt_handle_rdma_comp()
1452 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_handle_rdma_comp()
1455 if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA, in srpt_handle_rdma_comp()
1457 target_execute_cmd(&ioctx->cmd); in srpt_handle_rdma_comp()
1460 __LINE__, srpt_get_cmd_state(ioctx)); in srpt_handle_rdma_comp()
1462 ioctx->rdma_aborted = true; in srpt_handle_rdma_comp()
1472 struct srpt_send_ioctx *ioctx, in srpt_handle_rdma_err_comp() argument
1478 cmd = &ioctx->cmd; in srpt_handle_rdma_err_comp()
1479 state = srpt_get_cmd_state(ioctx); in srpt_handle_rdma_err_comp()
1482 if (ioctx->n_rdma <= 0) { in srpt_handle_rdma_err_comp()
1485 ioctx->ioctx.index); in srpt_handle_rdma_err_comp()
1488 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_handle_rdma_err_comp()
1490 srpt_abort_cmd(ioctx); in srpt_handle_rdma_err_comp()
1519 struct srpt_send_ioctx *ioctx, u64 tag, in srpt_build_cmd_rsp() argument
1532 srp_rsp = ioctx->ioctx.buf; in srpt_build_cmd_rsp()
1535 sense_data = ioctx->sense_data; in srpt_build_cmd_rsp()
1536 sense_data_len = ioctx->cmd.scsi_sense_length; in srpt_build_cmd_rsp()
1537 WARN_ON(sense_data_len > sizeof(ioctx->sense_data)); in srpt_build_cmd_rsp()
1577 struct srpt_send_ioctx *ioctx, in srpt_build_tskmgmt_rsp() argument
1587 srp_rsp = ioctx->ioctx.buf; in srpt_build_tskmgmt_rsp()
1679 struct srpt_send_ioctx *ioctx = container_of(cmd, in srpt_check_stop_free() local
1682 return target_put_sess_cmd(&ioctx->cmd); in srpt_check_stop_free()
1702 srp_cmd = recv_ioctx->ioctx.buf; in srpt_handle_cmd()
1784 srp_tsk = recv_ioctx->ioctx.buf; in srpt_handle_tsk_mgmt()
1824 recv_ioctx->ioctx.dma, srp_max_req_size, in srpt_handle_new_iu()
1836 srp_cmd = recv_ioctx->ioctx.buf; in srpt_handle_new_iu()
1882 struct srpt_recv_ioctx *ioctx; in srpt_process_rcv_completion() local
1892 ioctx = sdev->ioctx_ring[index]; in srpt_process_rcv_completion()
1893 srpt_handle_new_iu(ch, ioctx, NULL); in srpt_process_rcv_completion()
2641 struct srpt_recv_ioctx *ioctx, *ioctx_tmp; in srpt_cm_rtu_recv() local
2645 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list, in srpt_cm_rtu_recv()
2647 list_del(&ioctx->wait_list); in srpt_cm_rtu_recv()
2648 srpt_handle_new_iu(ch, ioctx, NULL); in srpt_cm_rtu_recv()
2772 struct srpt_send_ioctx *ioctx) in srpt_perform_rdmas() argument
2781 const int n_rdma = ioctx->n_rdma; in srpt_perform_rdmas()
2783 dir = ioctx->cmd.data_direction; in srpt_perform_rdmas()
2795 ioctx->rdma_aborted = false; in srpt_perform_rdmas()
2797 riu = ioctx->rdma_ius; in srpt_perform_rdmas()
2806 ioctx->ioctx.index); in srpt_perform_rdmas()
2812 ioctx->ioctx.index); in srpt_perform_rdmas()
2834 wr.wr_id = encode_wr_id(SRPT_RDMA_ABORT, ioctx->ioctx.index); in srpt_perform_rdmas()
2839 ioctx->ioctx.index); in srpt_perform_rdmas()
2842 while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) { in srpt_perform_rdmas()
2844 ioctx->ioctx.index); in srpt_perform_rdmas()
2858 struct srpt_send_ioctx *ioctx) in srpt_xfer_data() argument
2862 ret = srpt_map_sg_to_ib_sge(ch, ioctx); in srpt_xfer_data()
2868 ret = srpt_perform_rdmas(ch, ioctx); in srpt_xfer_data()
2882 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_xfer_data()
2888 struct srpt_send_ioctx *ioctx; in srpt_write_pending_status() local
2890 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); in srpt_write_pending_status()
2891 return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA; in srpt_write_pending_status()
2900 struct srpt_send_ioctx *ioctx; in srpt_write_pending() local
2905 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); in srpt_write_pending()
2907 new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA); in srpt_write_pending()
2910 ch = ioctx->ch; in srpt_write_pending()
2925 ioctx->tag); in srpt_write_pending()
2926 srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN); in srpt_write_pending()
2930 ret = srpt_xfer_data(ch, ioctx); in srpt_write_pending()
2956 struct srpt_send_ioctx *ioctx; in srpt_queue_response() local
2964 ioctx = container_of(cmd, struct srpt_send_ioctx, cmd); in srpt_queue_response()
2965 ch = ioctx->ch; in srpt_queue_response()
2968 spin_lock_irqsave(&ioctx->spinlock, flags); in srpt_queue_response()
2969 state = ioctx->state; in srpt_queue_response()
2973 ioctx->state = SRPT_STATE_CMD_RSP_SENT; in srpt_queue_response()
2976 ioctx->state = SRPT_STATE_MGMT_RSP_SENT; in srpt_queue_response()
2980 ch, ioctx->ioctx.index, ioctx->state); in srpt_queue_response()
2983 spin_unlock_irqrestore(&ioctx->spinlock, flags); in srpt_queue_response()
2985 if (unlikely(transport_check_aborted_status(&ioctx->cmd, false) in srpt_queue_response()
2988 srpt_abort_cmd(ioctx); in srpt_queue_response()
2992 dir = ioctx->cmd.data_direction; in srpt_queue_response()
2995 if (dir == DMA_FROM_DEVICE && ioctx->cmd.data_length && in srpt_queue_response()
2996 !ioctx->queue_status_only) { in srpt_queue_response()
2997 ret = srpt_xfer_data(ch, ioctx); in srpt_queue_response()
3000 ioctx->tag); in srpt_queue_response()
3006 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->tag, in srpt_queue_response()
3011 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status, in srpt_queue_response()
3012 ioctx->tag); in srpt_queue_response()
3014 ret = srpt_post_send(ch, ioctx, resp_len); in srpt_queue_response()
3017 ioctx->tag); in srpt_queue_response()
3018 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_queue_response()
3019 srpt_set_cmd_state(ioctx, SRPT_STATE_DONE); in srpt_queue_response()
3020 target_put_sess_cmd(&ioctx->cmd); in srpt_queue_response()
3037 struct srpt_send_ioctx *ioctx = container_of(cmd, in srpt_aborted_task() local
3040 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); in srpt_aborted_task()
3045 struct srpt_send_ioctx *ioctx; in srpt_queue_status() local
3047 ioctx = container_of(cmd, struct srpt_send_ioctx, cmd); in srpt_queue_status()
3048 BUG_ON(ioctx->sense_data != cmd->sense_buffer); in srpt_queue_status()
3052 ioctx->queue_status_only = true; in srpt_queue_status()
3427 struct srpt_send_ioctx *ioctx = container_of(se_cmd, in srpt_release_cmd() local
3429 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_release_cmd()
3432 WARN_ON(ioctx->state != SRPT_STATE_DONE); in srpt_release_cmd()
3433 WARN_ON(ioctx->mapped_sg_count != 0); in srpt_release_cmd()
3435 if (ioctx->n_rbuf > 1) { in srpt_release_cmd()
3436 kfree(ioctx->rbufs); in srpt_release_cmd()
3437 ioctx->rbufs = NULL; in srpt_release_cmd()
3438 ioctx->n_rbuf = 0; in srpt_release_cmd()
3442 list_add(&ioctx->free_list, &ch->free_list); in srpt_release_cmd()
3495 struct srpt_send_ioctx *ioctx; in srpt_get_task_tag() local
3497 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); in srpt_get_task_tag()
3498 return ioctx->tag; in srpt_get_task_tag()
3504 struct srpt_send_ioctx *ioctx; in srpt_get_tcm_cmd_state() local
3506 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); in srpt_get_tcm_cmd_state()
3507 return srpt_get_cmd_state(ioctx); in srpt_get_tcm_cmd_state()