Lines Matching refs:ch
97 static void srpt_release_channel(struct srpt_rdma_ch *ch);
123 static enum rdma_ch_state srpt_get_ch_state(struct srpt_rdma_ch *ch) in srpt_get_ch_state() argument
128 spin_lock_irqsave(&ch->spinlock, flags); in srpt_get_ch_state()
129 state = ch->state; in srpt_get_ch_state()
130 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_get_ch_state()
135 srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new_state) in srpt_set_ch_state() argument
140 spin_lock_irqsave(&ch->spinlock, flags); in srpt_set_ch_state()
141 prev = ch->state; in srpt_set_ch_state()
142 ch->state = new_state; in srpt_set_ch_state()
143 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_set_ch_state()
153 srpt_test_and_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state old, in srpt_test_and_set_ch_state() argument
159 spin_lock_irqsave(&ch->spinlock, flags); in srpt_test_and_set_ch_state()
160 prev = ch->state; in srpt_test_and_set_ch_state()
162 ch->state = new; in srpt_test_and_set_ch_state()
163 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_test_and_set_ch_state()
227 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch) in srpt_qp_event() argument
230 event->event, ch->cm_id, ch->sess_name, srpt_get_ch_state(ch)); in srpt_qp_event()
234 ib_cm_notify(ch->cm_id, event->event); in srpt_qp_event()
237 if (srpt_test_and_set_ch_state(ch, CH_DRAINING, in srpt_qp_event()
239 srpt_release_channel(ch); in srpt_qp_event()
242 ch->sess_name, srpt_get_ch_state(ch)); in srpt_qp_event()
800 static int srpt_post_send(struct srpt_rdma_ch *ch, in srpt_post_send() argument
805 struct srpt_device *sdev = ch->sport->sdev; in srpt_post_send()
808 atomic_inc(&ch->req_lim); in srpt_post_send()
811 if (unlikely(atomic_dec_return(&ch->sq_wr_avail) < 0)) { in srpt_post_send()
830 ret = ib_post_send(ch->qp, &wr, &bad_wr); in srpt_post_send()
834 atomic_inc(&ch->sq_wr_avail); in srpt_post_send()
835 atomic_dec(&ch->req_lim); in srpt_post_send()
951 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_init_ch_qp() argument
963 attr->port_num = ch->sport->port; in srpt_init_ch_qp()
985 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_ch_qp_rtr() argument
992 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask); in srpt_ch_qp_rtr()
1015 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_ch_qp_rts() argument
1022 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask); in srpt_ch_qp_rts()
1037 static int srpt_ch_qp_err(struct srpt_rdma_ch *ch) in srpt_ch_qp_err() argument
1042 return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE); in srpt_ch_qp_err()
1048 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch, in srpt_unmap_sg_to_ib_sge() argument
1054 BUG_ON(!ch); in srpt_unmap_sg_to_ib_sge()
1069 ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt, in srpt_unmap_sg_to_ib_sge()
1078 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch, in srpt_map_sg_to_ib_sge() argument
1081 struct ib_device *dev = ch->sport->sdev->device; in srpt_map_sg_to_ib_sge()
1097 BUG_ON(!ch); in srpt_map_sg_to_ib_sge()
1106 count = ib_dma_map_sg(ch->sport->sdev->device, sg, sg_cnt, in srpt_map_sg_to_ib_sge()
1209 sge->lkey = ch->sport->sdev->mr->lkey; in srpt_map_sg_to_ib_sge()
1248 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_map_sg_to_ib_sge()
1256 static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch) in srpt_get_send_ioctx() argument
1261 BUG_ON(!ch); in srpt_get_send_ioctx()
1264 spin_lock_irqsave(&ch->spinlock, flags); in srpt_get_send_ioctx()
1265 if (!list_empty(&ch->free_list)) { in srpt_get_send_ioctx()
1266 ioctx = list_first_entry(&ch->free_list, in srpt_get_send_ioctx()
1270 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_get_send_ioctx()
1275 BUG_ON(ioctx->ch != ch); in srpt_get_send_ioctx()
1333 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_abort_cmd() local
1335 BUG_ON(ch->sess == NULL); in srpt_abort_cmd()
1367 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); in srpt_abort_cmd()
1386 static void srpt_handle_send_err_comp(struct srpt_rdma_ch *ch, u64 wr_id) in srpt_handle_send_err_comp() argument
1393 atomic_inc(&ch->sq_wr_avail); in srpt_handle_send_err_comp()
1396 ioctx = ch->ioctx_ring[index]; in srpt_handle_send_err_comp()
1408 atomic_dec(&ch->req_lim); in srpt_handle_send_err_comp()
1416 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch, in srpt_handle_send_comp() argument
1421 atomic_inc(&ch->sq_wr_avail); in srpt_handle_send_comp()
1431 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_handle_send_comp()
1447 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch, in srpt_handle_rdma_comp() argument
1452 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_handle_rdma_comp()
1471 static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch, in srpt_handle_rdma_err_comp() argument
1488 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_handle_rdma_err_comp()
1518 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch, in srpt_build_cmd_rsp() argument
1542 __constant_cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0)); in srpt_build_cmd_rsp()
1548 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp); in srpt_build_cmd_rsp()
1576 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch, in srpt_build_tskmgmt_rsp() argument
1593 + atomic_xchg(&ch->req_lim_delta, 0)); in srpt_build_tskmgmt_rsp()
1688 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, in srpt_handle_cmd() argument
1731 rc = target_submit_cmd(cmd, ch->sess, srp_cmd->cdb, in srpt_handle_cmd()
1771 static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch, in srpt_handle_tsk_mgmt() argument
1777 struct se_session *sess = ch->sess; in srpt_handle_tsk_mgmt()
1789 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess); in srpt_handle_tsk_mgmt()
1813 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch, in srpt_handle_new_iu() argument
1820 BUG_ON(!ch); in srpt_handle_new_iu()
1823 ib_dma_sync_single_for_cpu(ch->sport->sdev->device, in srpt_handle_new_iu()
1827 ch_state = srpt_get_ch_state(ch); in srpt_handle_new_iu()
1829 list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list); in srpt_handle_new_iu()
1839 send_ioctx = srpt_get_send_ioctx(ch); in srpt_handle_new_iu()
1842 &ch->cmd_wait_list); in srpt_handle_new_iu()
1849 srpt_handle_cmd(ch, recv_ioctx, send_ioctx); in srpt_handle_new_iu()
1852 srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx); in srpt_handle_new_iu()
1872 srpt_post_recv(ch->sport->sdev, recv_ioctx); in srpt_handle_new_iu()
1878 struct srpt_rdma_ch *ch, in srpt_process_rcv_completion() argument
1881 struct srpt_device *sdev = ch->sport->sdev; in srpt_process_rcv_completion()
1889 req_lim = atomic_dec_return(&ch->req_lim); in srpt_process_rcv_completion()
1893 srpt_handle_new_iu(ch, ioctx, NULL); in srpt_process_rcv_completion()
1916 struct srpt_rdma_ch *ch, in srpt_process_send_completion() argument
1925 send_ioctx = ch->ioctx_ring[index]; in srpt_process_send_completion()
1928 srpt_handle_send_comp(ch, send_ioctx); in srpt_process_send_completion()
1932 srpt_handle_rdma_comp(ch, send_ioctx, opcode); in srpt_process_send_completion()
1938 srpt_handle_send_err_comp(ch, wc->wr_id); in srpt_process_send_completion()
1942 srpt_handle_rdma_err_comp(ch, send_ioctx, opcode); in srpt_process_send_completion()
1947 && !list_empty(&ch->cmd_wait_list) in srpt_process_send_completion()
1948 && srpt_get_ch_state(ch) == CH_LIVE in srpt_process_send_completion()
1949 && (send_ioctx = srpt_get_send_ioctx(ch)) != NULL)) { in srpt_process_send_completion()
1952 recv_ioctx = list_first_entry(&ch->cmd_wait_list, in srpt_process_send_completion()
1956 srpt_handle_new_iu(ch, recv_ioctx, send_ioctx); in srpt_process_send_completion()
1960 static void srpt_process_completion(struct ib_cq *cq, struct srpt_rdma_ch *ch) in srpt_process_completion() argument
1962 struct ib_wc *const wc = ch->wc; in srpt_process_completion()
1965 WARN_ON(cq != ch->cq); in srpt_process_completion()
1968 while ((n = ib_poll_cq(cq, ARRAY_SIZE(ch->wc), wc)) > 0) { in srpt_process_completion()
1971 srpt_process_rcv_completion(cq, ch, &wc[i]); in srpt_process_completion()
1973 srpt_process_send_completion(cq, ch, &wc[i]); in srpt_process_completion()
1991 struct srpt_rdma_ch *ch = ctx; in srpt_completion() local
1993 wake_up_interruptible(&ch->wait_queue); in srpt_completion()
1998 struct srpt_rdma_ch *ch; in srpt_compl_thread() local
2003 ch = arg; in srpt_compl_thread()
2004 BUG_ON(!ch); in srpt_compl_thread()
2006 ch->sess_name, ch->thread->comm, current->pid); in srpt_compl_thread()
2008 wait_event_interruptible(ch->wait_queue, in srpt_compl_thread()
2009 (srpt_process_completion(ch->cq, ch), in srpt_compl_thread()
2013 ch->sess_name, ch->thread->comm, current->pid); in srpt_compl_thread()
2020 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) in srpt_create_ch_ib() argument
2023 struct srpt_port *sport = ch->sport; in srpt_create_ch_ib()
2028 WARN_ON(ch->rq_size < 1); in srpt_create_ch_ib()
2036 ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, in srpt_create_ch_ib()
2037 ch->rq_size + srp_sq_size, 0); in srpt_create_ch_ib()
2038 if (IS_ERR(ch->cq)) { in srpt_create_ch_ib()
2039 ret = PTR_ERR(ch->cq); in srpt_create_ch_ib()
2041 ch->rq_size + srp_sq_size, ret); in srpt_create_ch_ib()
2045 qp_init->qp_context = (void *)ch; in srpt_create_ch_ib()
2048 qp_init->send_cq = ch->cq; in srpt_create_ch_ib()
2049 qp_init->recv_cq = ch->cq; in srpt_create_ch_ib()
2056 ch->qp = ib_create_qp(sdev->pd, qp_init); in srpt_create_ch_ib()
2057 if (IS_ERR(ch->qp)) { in srpt_create_ch_ib()
2058 ret = PTR_ERR(ch->qp); in srpt_create_ch_ib()
2062 ib_destroy_cq(ch->cq); in srpt_create_ch_ib()
2070 atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr); in srpt_create_ch_ib()
2073 __func__, ch->cq->cqe, qp_init->cap.max_send_sge, in srpt_create_ch_ib()
2074 qp_init->cap.max_send_wr, ch->cm_id); in srpt_create_ch_ib()
2076 ret = srpt_init_ch_qp(ch, ch->qp); in srpt_create_ch_ib()
2080 init_waitqueue_head(&ch->wait_queue); in srpt_create_ch_ib()
2082 pr_debug("creating thread for session %s\n", ch->sess_name); in srpt_create_ch_ib()
2084 ch->thread = kthread_run(srpt_compl_thread, ch, "ib_srpt_compl"); in srpt_create_ch_ib()
2085 if (IS_ERR(ch->thread)) { in srpt_create_ch_ib()
2087 PTR_ERR(ch->thread)); in srpt_create_ch_ib()
2088 ch->thread = NULL; in srpt_create_ch_ib()
2097 ib_destroy_qp(ch->qp); in srpt_create_ch_ib()
2099 ib_destroy_cq(ch->cq); in srpt_create_ch_ib()
2103 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch) in srpt_destroy_ch_ib() argument
2105 if (ch->thread) in srpt_destroy_ch_ib()
2106 kthread_stop(ch->thread); in srpt_destroy_ch_ib()
2108 ib_destroy_qp(ch->qp); in srpt_destroy_ch_ib()
2109 ib_destroy_cq(ch->cq); in srpt_destroy_ch_ib()
2120 static void __srpt_close_ch(struct srpt_rdma_ch *ch) in __srpt_close_ch() argument
2126 sdev = ch->sport->sdev; in __srpt_close_ch()
2128 spin_lock_irqsave(&ch->spinlock, flags); in __srpt_close_ch()
2129 prev_state = ch->state; in __srpt_close_ch()
2133 ch->state = CH_DISCONNECTING; in __srpt_close_ch()
2138 spin_unlock_irqrestore(&ch->spinlock, flags); in __srpt_close_ch()
2142 ib_send_cm_rej(ch->cm_id, IB_CM_REJ_NO_RESOURCES, NULL, 0, in __srpt_close_ch()
2146 if (ib_send_cm_dreq(ch->cm_id, NULL, 0) < 0) in __srpt_close_ch()
2160 static void srpt_close_ch(struct srpt_rdma_ch *ch) in srpt_close_ch() argument
2164 sdev = ch->sport->sdev; in srpt_close_ch()
2166 __srpt_close_ch(ch); in srpt_close_ch()
2175 struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr; in srpt_shutdown_session() local
2178 spin_lock_irqsave(&ch->spinlock, flags); in srpt_shutdown_session()
2179 if (ch->in_shutdown) { in srpt_shutdown_session()
2180 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_shutdown_session()
2184 ch->in_shutdown = true; in srpt_shutdown_session()
2186 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_shutdown_session()
2206 struct srpt_rdma_ch *ch; in srpt_drain_channel() local
2215 list_for_each_entry(ch, &sdev->rch_list, list) { in srpt_drain_channel()
2216 if (ch->cm_id == cm_id) { in srpt_drain_channel()
2217 do_reset = srpt_test_and_set_ch_state(ch, in srpt_drain_channel()
2219 srpt_test_and_set_ch_state(ch, in srpt_drain_channel()
2221 srpt_test_and_set_ch_state(ch, in srpt_drain_channel()
2229 if (ch->sess) in srpt_drain_channel()
2230 srpt_shutdown_session(ch->sess); in srpt_drain_channel()
2232 ret = srpt_ch_qp_err(ch); in srpt_drain_channel()
2248 struct srpt_rdma_ch *ch; in srpt_find_channel() local
2256 list_for_each_entry(ch, &sdev->rch_list, list) { in srpt_find_channel()
2257 if (ch->cm_id == cm_id) { in srpt_find_channel()
2264 return found ? ch : NULL; in srpt_find_channel()
2275 static void srpt_release_channel(struct srpt_rdma_ch *ch) in srpt_release_channel() argument
2277 schedule_work(&ch->release_work); in srpt_release_channel()
2282 struct srpt_rdma_ch *ch; in srpt_release_channel_work() local
2286 ch = container_of(w, struct srpt_rdma_ch, release_work); in srpt_release_channel_work()
2287 pr_debug("ch = %p; ch->sess = %p; release_done = %p\n", ch, ch->sess, in srpt_release_channel_work()
2288 ch->release_done); in srpt_release_channel_work()
2290 sdev = ch->sport->sdev; in srpt_release_channel_work()
2293 se_sess = ch->sess; in srpt_release_channel_work()
2300 ch->sess = NULL; in srpt_release_channel_work()
2302 ib_destroy_cm_id(ch->cm_id); in srpt_release_channel_work()
2304 srpt_destroy_ch_ib(ch); in srpt_release_channel_work()
2306 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, in srpt_release_channel_work()
2307 ch->sport->sdev, ch->rq_size, in srpt_release_channel_work()
2308 ch->rsp_size, DMA_TO_DEVICE); in srpt_release_channel_work()
2311 list_del(&ch->list); in srpt_release_channel_work()
2314 if (ch->release_done) in srpt_release_channel_work()
2315 complete(ch->release_done); in srpt_release_channel_work()
2319 kfree(ch); in srpt_release_channel_work()
2363 struct srpt_rdma_ch *ch, *tmp_ch; in srpt_cm_req_recv() local
2423 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) { in srpt_cm_req_recv()
2424 if (!memcmp(ch->i_port_id, req->initiator_port_id, 16) in srpt_cm_req_recv()
2425 && !memcmp(ch->t_port_id, req->target_port_id, 16) in srpt_cm_req_recv()
2426 && param->port == ch->sport->port in srpt_cm_req_recv()
2427 && param->listen_id == ch->sport->sdev->cm_id in srpt_cm_req_recv()
2428 && ch->cm_id) { in srpt_cm_req_recv()
2431 ch_state = srpt_get_ch_state(ch); in srpt_cm_req_recv()
2439 ch->sess_name, ch->cm_id, ch_state); in srpt_cm_req_recv()
2441 __srpt_close_ch(ch); in srpt_cm_req_recv()
2464 ch = kzalloc(sizeof *ch, GFP_KERNEL); in srpt_cm_req_recv()
2465 if (!ch) { in srpt_cm_req_recv()
2473 INIT_WORK(&ch->release_work, srpt_release_channel_work); in srpt_cm_req_recv()
2474 memcpy(ch->i_port_id, req->initiator_port_id, 16); in srpt_cm_req_recv()
2475 memcpy(ch->t_port_id, req->target_port_id, 16); in srpt_cm_req_recv()
2476 ch->sport = &sdev->port[param->port - 1]; in srpt_cm_req_recv()
2477 ch->cm_id = cm_id; in srpt_cm_req_recv()
2482 ch->rq_size = SRPT_RQ_SIZE; in srpt_cm_req_recv()
2483 spin_lock_init(&ch->spinlock); in srpt_cm_req_recv()
2484 ch->state = CH_CONNECTING; in srpt_cm_req_recv()
2485 INIT_LIST_HEAD(&ch->cmd_wait_list); in srpt_cm_req_recv()
2486 ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size; in srpt_cm_req_recv()
2488 ch->ioctx_ring = (struct srpt_send_ioctx **) in srpt_cm_req_recv()
2489 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2490 sizeof(*ch->ioctx_ring[0]), in srpt_cm_req_recv()
2491 ch->rsp_size, DMA_TO_DEVICE); in srpt_cm_req_recv()
2492 if (!ch->ioctx_ring) in srpt_cm_req_recv()
2495 INIT_LIST_HEAD(&ch->free_list); in srpt_cm_req_recv()
2496 for (i = 0; i < ch->rq_size; i++) { in srpt_cm_req_recv()
2497 ch->ioctx_ring[i]->ch = ch; in srpt_cm_req_recv()
2498 list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list); in srpt_cm_req_recv()
2501 ret = srpt_create_ch_ib(ch); in srpt_cm_req_recv()
2510 ret = srpt_ch_qp_rtr(ch, ch->qp); in srpt_cm_req_recv()
2521 snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx", in srpt_cm_req_recv()
2522 be64_to_cpu(*(__be64 *)ch->i_port_id), in srpt_cm_req_recv()
2523 be64_to_cpu(*(__be64 *)(ch->i_port_id + 8))); in srpt_cm_req_recv()
2525 pr_debug("registering session %s\n", ch->sess_name); in srpt_cm_req_recv()
2527 nacl = srpt_lookup_acl(sport, ch->i_port_id); in srpt_cm_req_recv()
2530 " configured yet for initiator %s.\n", ch->sess_name); in srpt_cm_req_recv()
2536 ch->sess = transport_init_session(TARGET_PROT_NORMAL); in srpt_cm_req_recv()
2537 if (IS_ERR(ch->sess)) { in srpt_cm_req_recv()
2543 ch->sess->se_node_acl = &nacl->nacl; in srpt_cm_req_recv()
2544 transport_register_session(&sport->port_tpg_1, &nacl->nacl, ch->sess, ch); in srpt_cm_req_recv()
2546 pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess, in srpt_cm_req_recv()
2547 ch->sess_name, ch->cm_id); in srpt_cm_req_recv()
2554 ch->max_ti_iu_len = it_iu_len; in srpt_cm_req_recv()
2557 rsp->req_lim_delta = cpu_to_be32(ch->rq_size); in srpt_cm_req_recv()
2558 atomic_set(&ch->req_lim, ch->rq_size); in srpt_cm_req_recv()
2559 atomic_set(&ch->req_lim_delta, 0); in srpt_cm_req_recv()
2562 rep_param->qp_num = ch->qp->qp_num; in srpt_cm_req_recv()
2580 list_add_tail(&ch->list, &sdev->rch_list); in srpt_cm_req_recv()
2586 srpt_set_ch_state(ch, CH_RELEASING); in srpt_cm_req_recv()
2587 transport_deregister_session_configfs(ch->sess); in srpt_cm_req_recv()
2590 transport_deregister_session(ch->sess); in srpt_cm_req_recv()
2591 ch->sess = NULL; in srpt_cm_req_recv()
2594 srpt_destroy_ch_ib(ch); in srpt_cm_req_recv()
2597 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, in srpt_cm_req_recv()
2598 ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2599 ch->rsp_size, DMA_TO_DEVICE); in srpt_cm_req_recv()
2601 kfree(ch); in srpt_cm_req_recv()
2634 struct srpt_rdma_ch *ch; in srpt_cm_rtu_recv() local
2637 ch = srpt_find_channel(cm_id->context, cm_id); in srpt_cm_rtu_recv()
2638 BUG_ON(!ch); in srpt_cm_rtu_recv()
2640 if (srpt_test_and_set_ch_state(ch, CH_CONNECTING, CH_LIVE)) { in srpt_cm_rtu_recv()
2643 ret = srpt_ch_qp_rts(ch, ch->qp); in srpt_cm_rtu_recv()
2645 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list, in srpt_cm_rtu_recv()
2648 srpt_handle_new_iu(ch, ioctx, NULL); in srpt_cm_rtu_recv()
2651 srpt_close_ch(ch); in srpt_cm_rtu_recv()
2672 struct srpt_rdma_ch *ch; in srpt_cm_dreq_recv() local
2676 ch = srpt_find_channel(cm_id->context, cm_id); in srpt_cm_dreq_recv()
2677 BUG_ON(!ch); in srpt_cm_dreq_recv()
2679 pr_debug("cm_id= %p ch->state= %d\n", cm_id, srpt_get_ch_state(ch)); in srpt_cm_dreq_recv()
2681 spin_lock_irqsave(&ch->spinlock, flags); in srpt_cm_dreq_recv()
2682 switch (ch->state) { in srpt_cm_dreq_recv()
2686 ch->state = CH_DISCONNECTING; in srpt_cm_dreq_recv()
2691 WARN(true, "unexpected channel state %d\n", ch->state); in srpt_cm_dreq_recv()
2694 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_cm_dreq_recv()
2697 if (ib_send_cm_drep(ch->cm_id, NULL, 0) < 0) in srpt_cm_dreq_recv()
2700 ch->sess_name); in srpt_cm_dreq_recv()
2771 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, in srpt_perform_rdmas() argument
2787 sq_wr_avail = atomic_sub_return(n_rdma, &ch->sq_wr_avail); in srpt_perform_rdmas()
2824 ret = ib_post_send(ch->qp, &wr, &bad_wr); in srpt_perform_rdmas()
2836 while (ch->state == CH_LIVE && in srpt_perform_rdmas()
2837 ib_post_send(ch->qp, &wr, &bad_wr) != 0) { in srpt_perform_rdmas()
2842 while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) { in srpt_perform_rdmas()
2850 atomic_add(n_rdma, &ch->sq_wr_avail); in srpt_perform_rdmas()
2857 static int srpt_xfer_data(struct srpt_rdma_ch *ch, in srpt_xfer_data() argument
2862 ret = srpt_map_sg_to_ib_sge(ch, ioctx); in srpt_xfer_data()
2868 ret = srpt_perform_rdmas(ch, ioctx); in srpt_xfer_data()
2882 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_xfer_data()
2899 struct srpt_rdma_ch *ch; in srpt_write_pending() local
2910 ch = ioctx->ch; in srpt_write_pending()
2911 BUG_ON(!ch); in srpt_write_pending()
2913 ch_state = srpt_get_ch_state(ch); in srpt_write_pending()
2930 ret = srpt_xfer_data(ch, ioctx); in srpt_write_pending()
2955 struct srpt_rdma_ch *ch; in srpt_queue_response() local
2965 ch = ioctx->ch; in srpt_queue_response()
2966 BUG_ON(!ch); in srpt_queue_response()
2980 ch, ioctx->ioctx.index, ioctx->state); in srpt_queue_response()
2987 atomic_inc(&ch->req_lim_delta); in srpt_queue_response()
2997 ret = srpt_xfer_data(ch, ioctx); in srpt_queue_response()
3006 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->tag, in srpt_queue_response()
3011 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status, in srpt_queue_response()
3014 ret = srpt_post_send(ch, ioctx, resp_len); in srpt_queue_response()
3018 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_queue_response()
3040 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); in srpt_aborted_task()
3080 struct srpt_rdma_ch *ch, *tmp_ch; in srpt_release_sdev() local
3088 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) in srpt_release_sdev()
3089 __srpt_close_ch(ch); in srpt_release_sdev()
3429 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_release_cmd() local
3441 spin_lock_irqsave(&ch->spinlock, flags); in srpt_release_cmd()
3442 list_add(&ioctx->free_list, &ch->free_list); in srpt_release_cmd()
3443 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_release_cmd()
3456 struct srpt_rdma_ch *ch; in srpt_close_session() local
3460 ch = se_sess->fabric_sess_ptr; in srpt_close_session()
3461 WARN_ON(ch->sess != se_sess); in srpt_close_session()
3463 pr_debug("ch %p state %d\n", ch, srpt_get_ch_state(ch)); in srpt_close_session()
3465 sdev = ch->sport->sdev; in srpt_close_session()
3467 BUG_ON(ch->release_done); in srpt_close_session()
3468 ch->release_done = &release_done; in srpt_close_session()
3469 __srpt_close_ch(ch); in srpt_close_session()