Lines Matching refs:ch
94 static void srpt_release_channel(struct srpt_rdma_ch *ch);
120 static enum rdma_ch_state srpt_get_ch_state(struct srpt_rdma_ch *ch) in srpt_get_ch_state() argument
125 spin_lock_irqsave(&ch->spinlock, flags); in srpt_get_ch_state()
126 state = ch->state; in srpt_get_ch_state()
127 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_get_ch_state()
132 srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new_state) in srpt_set_ch_state() argument
137 spin_lock_irqsave(&ch->spinlock, flags); in srpt_set_ch_state()
138 prev = ch->state; in srpt_set_ch_state()
139 ch->state = new_state; in srpt_set_ch_state()
140 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_set_ch_state()
150 srpt_test_and_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state old, in srpt_test_and_set_ch_state() argument
156 spin_lock_irqsave(&ch->spinlock, flags); in srpt_test_and_set_ch_state()
157 prev = ch->state; in srpt_test_and_set_ch_state()
159 ch->state = new; in srpt_test_and_set_ch_state()
160 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_test_and_set_ch_state()
224 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch) in srpt_qp_event() argument
227 event->event, ch->cm_id, ch->sess_name, srpt_get_ch_state(ch)); in srpt_qp_event()
231 ib_cm_notify(ch->cm_id, event->event); in srpt_qp_event()
234 if (srpt_test_and_set_ch_state(ch, CH_DRAINING, in srpt_qp_event()
236 srpt_release_channel(ch); in srpt_qp_event()
239 ch->sess_name, srpt_get_ch_state(ch)); in srpt_qp_event()
799 static int srpt_post_send(struct srpt_rdma_ch *ch, in srpt_post_send() argument
804 struct srpt_device *sdev = ch->sport->sdev; in srpt_post_send()
807 atomic_inc(&ch->req_lim); in srpt_post_send()
810 if (unlikely(atomic_dec_return(&ch->sq_wr_avail) < 0)) { in srpt_post_send()
829 ret = ib_post_send(ch->qp, &wr, &bad_wr); in srpt_post_send()
833 atomic_inc(&ch->sq_wr_avail); in srpt_post_send()
834 atomic_dec(&ch->req_lim); in srpt_post_send()
950 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_init_ch_qp() argument
962 attr->port_num = ch->sport->port; in srpt_init_ch_qp()
984 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_ch_qp_rtr() argument
991 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask); in srpt_ch_qp_rtr()
1014 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_ch_qp_rts() argument
1021 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask); in srpt_ch_qp_rts()
1036 static int srpt_ch_qp_err(struct srpt_rdma_ch *ch) in srpt_ch_qp_err() argument
1041 return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE); in srpt_ch_qp_err()
1047 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch, in srpt_unmap_sg_to_ib_sge() argument
1053 BUG_ON(!ch); in srpt_unmap_sg_to_ib_sge()
1068 ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt, in srpt_unmap_sg_to_ib_sge()
1077 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch, in srpt_map_sg_to_ib_sge() argument
1080 struct ib_device *dev = ch->sport->sdev->device; in srpt_map_sg_to_ib_sge()
1096 BUG_ON(!ch); in srpt_map_sg_to_ib_sge()
1105 count = ib_dma_map_sg(ch->sport->sdev->device, sg, sg_cnt, in srpt_map_sg_to_ib_sge()
1208 sge->lkey = ch->sport->sdev->pd->local_dma_lkey; in srpt_map_sg_to_ib_sge()
1247 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_map_sg_to_ib_sge()
1255 static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch) in srpt_get_send_ioctx() argument
1260 BUG_ON(!ch); in srpt_get_send_ioctx()
1263 spin_lock_irqsave(&ch->spinlock, flags); in srpt_get_send_ioctx()
1264 if (!list_empty(&ch->free_list)) { in srpt_get_send_ioctx()
1265 ioctx = list_first_entry(&ch->free_list, in srpt_get_send_ioctx()
1269 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_get_send_ioctx()
1274 BUG_ON(ioctx->ch != ch); in srpt_get_send_ioctx()
1332 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_abort_cmd() local
1334 BUG_ON(ch->sess == NULL); in srpt_abort_cmd()
1366 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); in srpt_abort_cmd()
1385 static void srpt_handle_send_err_comp(struct srpt_rdma_ch *ch, u64 wr_id) in srpt_handle_send_err_comp() argument
1391 atomic_inc(&ch->sq_wr_avail); in srpt_handle_send_err_comp()
1394 ioctx = ch->ioctx_ring[index]; in srpt_handle_send_err_comp()
1405 atomic_dec(&ch->req_lim); in srpt_handle_send_err_comp()
1413 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch, in srpt_handle_send_comp() argument
1418 atomic_inc(&ch->sq_wr_avail); in srpt_handle_send_comp()
1428 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_handle_send_comp()
1444 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch, in srpt_handle_rdma_comp() argument
1449 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_handle_rdma_comp()
1468 static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch, in srpt_handle_rdma_err_comp() argument
1483 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_handle_rdma_err_comp()
1513 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch, in srpt_build_cmd_rsp() argument
1537 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0)); in srpt_build_cmd_rsp()
1543 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp); in srpt_build_cmd_rsp()
1571 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch, in srpt_build_tskmgmt_rsp() argument
1588 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0)); in srpt_build_tskmgmt_rsp()
1683 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, in srpt_handle_cmd() argument
1726 rc = target_submit_cmd(cmd, ch->sess, srp_cmd->cdb, in srpt_handle_cmd()
1766 static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch, in srpt_handle_tsk_mgmt() argument
1772 struct se_session *sess = ch->sess; in srpt_handle_tsk_mgmt()
1784 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess); in srpt_handle_tsk_mgmt()
1808 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch, in srpt_handle_new_iu() argument
1815 BUG_ON(!ch); in srpt_handle_new_iu()
1818 ib_dma_sync_single_for_cpu(ch->sport->sdev->device, in srpt_handle_new_iu()
1822 ch_state = srpt_get_ch_state(ch); in srpt_handle_new_iu()
1824 list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list); in srpt_handle_new_iu()
1834 send_ioctx = srpt_get_send_ioctx(ch); in srpt_handle_new_iu()
1837 &ch->cmd_wait_list); in srpt_handle_new_iu()
1844 srpt_handle_cmd(ch, recv_ioctx, send_ioctx); in srpt_handle_new_iu()
1847 srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx); in srpt_handle_new_iu()
1867 srpt_post_recv(ch->sport->sdev, recv_ioctx); in srpt_handle_new_iu()
1873 struct srpt_rdma_ch *ch, in srpt_process_rcv_completion() argument
1876 struct srpt_device *sdev = ch->sport->sdev; in srpt_process_rcv_completion()
1884 req_lim = atomic_dec_return(&ch->req_lim); in srpt_process_rcv_completion()
1888 srpt_handle_new_iu(ch, ioctx, NULL); in srpt_process_rcv_completion()
1911 struct srpt_rdma_ch *ch, in srpt_process_send_completion() argument
1920 send_ioctx = ch->ioctx_ring[index]; in srpt_process_send_completion()
1923 srpt_handle_send_comp(ch, send_ioctx); in srpt_process_send_completion()
1927 srpt_handle_rdma_comp(ch, send_ioctx, opcode); in srpt_process_send_completion()
1933 srpt_handle_send_err_comp(ch, wc->wr_id); in srpt_process_send_completion()
1937 srpt_handle_rdma_err_comp(ch, send_ioctx, opcode); in srpt_process_send_completion()
1942 && !list_empty(&ch->cmd_wait_list) in srpt_process_send_completion()
1943 && srpt_get_ch_state(ch) == CH_LIVE in srpt_process_send_completion()
1944 && (send_ioctx = srpt_get_send_ioctx(ch)) != NULL)) { in srpt_process_send_completion()
1947 recv_ioctx = list_first_entry(&ch->cmd_wait_list, in srpt_process_send_completion()
1951 srpt_handle_new_iu(ch, recv_ioctx, send_ioctx); in srpt_process_send_completion()
1955 static void srpt_process_completion(struct ib_cq *cq, struct srpt_rdma_ch *ch) in srpt_process_completion() argument
1957 struct ib_wc *const wc = ch->wc; in srpt_process_completion()
1960 WARN_ON(cq != ch->cq); in srpt_process_completion()
1963 while ((n = ib_poll_cq(cq, ARRAY_SIZE(ch->wc), wc)) > 0) { in srpt_process_completion()
1966 srpt_process_rcv_completion(cq, ch, &wc[i]); in srpt_process_completion()
1968 srpt_process_send_completion(cq, ch, &wc[i]); in srpt_process_completion()
1986 struct srpt_rdma_ch *ch = ctx; in srpt_completion() local
1988 wake_up_interruptible(&ch->wait_queue); in srpt_completion()
1993 struct srpt_rdma_ch *ch; in srpt_compl_thread() local
1998 ch = arg; in srpt_compl_thread()
1999 BUG_ON(!ch); in srpt_compl_thread()
2001 ch->sess_name, ch->thread->comm, current->pid); in srpt_compl_thread()
2003 wait_event_interruptible(ch->wait_queue, in srpt_compl_thread()
2004 (srpt_process_completion(ch->cq, ch), in srpt_compl_thread()
2008 ch->sess_name, ch->thread->comm, current->pid); in srpt_compl_thread()
2015 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) in srpt_create_ch_ib() argument
2018 struct srpt_port *sport = ch->sport; in srpt_create_ch_ib()
2024 WARN_ON(ch->rq_size < 1); in srpt_create_ch_ib()
2032 cq_attr.cqe = ch->rq_size + srp_sq_size; in srpt_create_ch_ib()
2033 ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, in srpt_create_ch_ib()
2035 if (IS_ERR(ch->cq)) { in srpt_create_ch_ib()
2036 ret = PTR_ERR(ch->cq); in srpt_create_ch_ib()
2038 ch->rq_size + srp_sq_size, ret); in srpt_create_ch_ib()
2042 qp_init->qp_context = (void *)ch; in srpt_create_ch_ib()
2045 qp_init->send_cq = ch->cq; in srpt_create_ch_ib()
2046 qp_init->recv_cq = ch->cq; in srpt_create_ch_ib()
2053 ch->qp = ib_create_qp(sdev->pd, qp_init); in srpt_create_ch_ib()
2054 if (IS_ERR(ch->qp)) { in srpt_create_ch_ib()
2055 ret = PTR_ERR(ch->qp); in srpt_create_ch_ib()
2059 ib_destroy_cq(ch->cq); in srpt_create_ch_ib()
2067 atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr); in srpt_create_ch_ib()
2070 __func__, ch->cq->cqe, qp_init->cap.max_send_sge, in srpt_create_ch_ib()
2071 qp_init->cap.max_send_wr, ch->cm_id); in srpt_create_ch_ib()
2073 ret = srpt_init_ch_qp(ch, ch->qp); in srpt_create_ch_ib()
2077 init_waitqueue_head(&ch->wait_queue); in srpt_create_ch_ib()
2079 pr_debug("creating thread for session %s\n", ch->sess_name); in srpt_create_ch_ib()
2081 ch->thread = kthread_run(srpt_compl_thread, ch, "ib_srpt_compl"); in srpt_create_ch_ib()
2082 if (IS_ERR(ch->thread)) { in srpt_create_ch_ib()
2084 PTR_ERR(ch->thread)); in srpt_create_ch_ib()
2085 ch->thread = NULL; in srpt_create_ch_ib()
2094 ib_destroy_qp(ch->qp); in srpt_create_ch_ib()
2096 ib_destroy_cq(ch->cq); in srpt_create_ch_ib()
2100 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch) in srpt_destroy_ch_ib() argument
2102 if (ch->thread) in srpt_destroy_ch_ib()
2103 kthread_stop(ch->thread); in srpt_destroy_ch_ib()
2105 ib_destroy_qp(ch->qp); in srpt_destroy_ch_ib()
2106 ib_destroy_cq(ch->cq); in srpt_destroy_ch_ib()
2117 static void __srpt_close_ch(struct srpt_rdma_ch *ch) in __srpt_close_ch() argument
2122 spin_lock_irqsave(&ch->spinlock, flags); in __srpt_close_ch()
2123 prev_state = ch->state; in __srpt_close_ch()
2127 ch->state = CH_DISCONNECTING; in __srpt_close_ch()
2132 spin_unlock_irqrestore(&ch->spinlock, flags); in __srpt_close_ch()
2136 ib_send_cm_rej(ch->cm_id, IB_CM_REJ_NO_RESOURCES, NULL, 0, in __srpt_close_ch()
2140 if (ib_send_cm_dreq(ch->cm_id, NULL, 0) < 0) in __srpt_close_ch()
2154 static void srpt_close_ch(struct srpt_rdma_ch *ch) in srpt_close_ch() argument
2158 sdev = ch->sport->sdev; in srpt_close_ch()
2160 __srpt_close_ch(ch); in srpt_close_ch()
2169 struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr; in srpt_shutdown_session() local
2172 spin_lock_irqsave(&ch->spinlock, flags); in srpt_shutdown_session()
2173 if (ch->in_shutdown) { in srpt_shutdown_session()
2174 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_shutdown_session()
2178 ch->in_shutdown = true; in srpt_shutdown_session()
2180 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_shutdown_session()
2200 struct srpt_rdma_ch *ch; in srpt_drain_channel() local
2209 list_for_each_entry(ch, &sdev->rch_list, list) { in srpt_drain_channel()
2210 if (ch->cm_id == cm_id) { in srpt_drain_channel()
2211 do_reset = srpt_test_and_set_ch_state(ch, in srpt_drain_channel()
2213 srpt_test_and_set_ch_state(ch, in srpt_drain_channel()
2215 srpt_test_and_set_ch_state(ch, in srpt_drain_channel()
2223 if (ch->sess) in srpt_drain_channel()
2224 srpt_shutdown_session(ch->sess); in srpt_drain_channel()
2226 ret = srpt_ch_qp_err(ch); in srpt_drain_channel()
2242 struct srpt_rdma_ch *ch; in srpt_find_channel() local
2250 list_for_each_entry(ch, &sdev->rch_list, list) { in srpt_find_channel()
2251 if (ch->cm_id == cm_id) { in srpt_find_channel()
2258 return found ? ch : NULL; in srpt_find_channel()
2269 static void srpt_release_channel(struct srpt_rdma_ch *ch) in srpt_release_channel() argument
2271 schedule_work(&ch->release_work); in srpt_release_channel()
2276 struct srpt_rdma_ch *ch; in srpt_release_channel_work() local
2280 ch = container_of(w, struct srpt_rdma_ch, release_work); in srpt_release_channel_work()
2281 pr_debug("ch = %p; ch->sess = %p; release_done = %p\n", ch, ch->sess, in srpt_release_channel_work()
2282 ch->release_done); in srpt_release_channel_work()
2284 sdev = ch->sport->sdev; in srpt_release_channel_work()
2287 se_sess = ch->sess; in srpt_release_channel_work()
2294 ch->sess = NULL; in srpt_release_channel_work()
2296 ib_destroy_cm_id(ch->cm_id); in srpt_release_channel_work()
2298 srpt_destroy_ch_ib(ch); in srpt_release_channel_work()
2300 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, in srpt_release_channel_work()
2301 ch->sport->sdev, ch->rq_size, in srpt_release_channel_work()
2302 ch->rsp_size, DMA_TO_DEVICE); in srpt_release_channel_work()
2305 list_del(&ch->list); in srpt_release_channel_work()
2308 if (ch->release_done) in srpt_release_channel_work()
2309 complete(ch->release_done); in srpt_release_channel_work()
2313 kfree(ch); in srpt_release_channel_work()
2357 struct srpt_rdma_ch *ch, *tmp_ch; in srpt_cm_req_recv() local
2417 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) { in srpt_cm_req_recv()
2418 if (!memcmp(ch->i_port_id, req->initiator_port_id, 16) in srpt_cm_req_recv()
2419 && !memcmp(ch->t_port_id, req->target_port_id, 16) in srpt_cm_req_recv()
2420 && param->port == ch->sport->port in srpt_cm_req_recv()
2421 && param->listen_id == ch->sport->sdev->cm_id in srpt_cm_req_recv()
2422 && ch->cm_id) { in srpt_cm_req_recv()
2425 ch_state = srpt_get_ch_state(ch); in srpt_cm_req_recv()
2433 ch->sess_name, ch->cm_id, ch_state); in srpt_cm_req_recv()
2435 __srpt_close_ch(ch); in srpt_cm_req_recv()
2458 ch = kzalloc(sizeof *ch, GFP_KERNEL); in srpt_cm_req_recv()
2459 if (!ch) { in srpt_cm_req_recv()
2467 INIT_WORK(&ch->release_work, srpt_release_channel_work); in srpt_cm_req_recv()
2468 memcpy(ch->i_port_id, req->initiator_port_id, 16); in srpt_cm_req_recv()
2469 memcpy(ch->t_port_id, req->target_port_id, 16); in srpt_cm_req_recv()
2470 ch->sport = &sdev->port[param->port - 1]; in srpt_cm_req_recv()
2471 ch->cm_id = cm_id; in srpt_cm_req_recv()
2476 ch->rq_size = SRPT_RQ_SIZE; in srpt_cm_req_recv()
2477 spin_lock_init(&ch->spinlock); in srpt_cm_req_recv()
2478 ch->state = CH_CONNECTING; in srpt_cm_req_recv()
2479 INIT_LIST_HEAD(&ch->cmd_wait_list); in srpt_cm_req_recv()
2480 ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size; in srpt_cm_req_recv()
2482 ch->ioctx_ring = (struct srpt_send_ioctx **) in srpt_cm_req_recv()
2483 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2484 sizeof(*ch->ioctx_ring[0]), in srpt_cm_req_recv()
2485 ch->rsp_size, DMA_TO_DEVICE); in srpt_cm_req_recv()
2486 if (!ch->ioctx_ring) in srpt_cm_req_recv()
2489 INIT_LIST_HEAD(&ch->free_list); in srpt_cm_req_recv()
2490 for (i = 0; i < ch->rq_size; i++) { in srpt_cm_req_recv()
2491 ch->ioctx_ring[i]->ch = ch; in srpt_cm_req_recv()
2492 list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list); in srpt_cm_req_recv()
2495 ret = srpt_create_ch_ib(ch); in srpt_cm_req_recv()
2504 ret = srpt_ch_qp_rtr(ch, ch->qp); in srpt_cm_req_recv()
2514 snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx", in srpt_cm_req_recv()
2515 be64_to_cpu(*(__be64 *)ch->i_port_id), in srpt_cm_req_recv()
2516 be64_to_cpu(*(__be64 *)(ch->i_port_id + 8))); in srpt_cm_req_recv()
2518 pr_debug("registering session %s\n", ch->sess_name); in srpt_cm_req_recv()
2520 nacl = srpt_lookup_acl(sport, ch->i_port_id); in srpt_cm_req_recv()
2523 " configured yet for initiator %s.\n", ch->sess_name); in srpt_cm_req_recv()
2529 ch->sess = transport_init_session(TARGET_PROT_NORMAL); in srpt_cm_req_recv()
2530 if (IS_ERR(ch->sess)) { in srpt_cm_req_recv()
2536 ch->sess->se_node_acl = &nacl->nacl; in srpt_cm_req_recv()
2537 transport_register_session(&sport->port_tpg_1, &nacl->nacl, ch->sess, ch); in srpt_cm_req_recv()
2539 pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess, in srpt_cm_req_recv()
2540 ch->sess_name, ch->cm_id); in srpt_cm_req_recv()
2547 ch->max_ti_iu_len = it_iu_len; in srpt_cm_req_recv()
2550 rsp->req_lim_delta = cpu_to_be32(ch->rq_size); in srpt_cm_req_recv()
2551 atomic_set(&ch->req_lim, ch->rq_size); in srpt_cm_req_recv()
2552 atomic_set(&ch->req_lim_delta, 0); in srpt_cm_req_recv()
2555 rep_param->qp_num = ch->qp->qp_num; in srpt_cm_req_recv()
2573 list_add_tail(&ch->list, &sdev->rch_list); in srpt_cm_req_recv()
2579 srpt_set_ch_state(ch, CH_RELEASING); in srpt_cm_req_recv()
2580 transport_deregister_session_configfs(ch->sess); in srpt_cm_req_recv()
2583 transport_deregister_session(ch->sess); in srpt_cm_req_recv()
2584 ch->sess = NULL; in srpt_cm_req_recv()
2587 srpt_destroy_ch_ib(ch); in srpt_cm_req_recv()
2590 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, in srpt_cm_req_recv()
2591 ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2592 ch->rsp_size, DMA_TO_DEVICE); in srpt_cm_req_recv()
2594 kfree(ch); in srpt_cm_req_recv()
2627 struct srpt_rdma_ch *ch; in srpt_cm_rtu_recv() local
2630 ch = srpt_find_channel(cm_id->context, cm_id); in srpt_cm_rtu_recv()
2631 BUG_ON(!ch); in srpt_cm_rtu_recv()
2633 if (srpt_test_and_set_ch_state(ch, CH_CONNECTING, CH_LIVE)) { in srpt_cm_rtu_recv()
2636 ret = srpt_ch_qp_rts(ch, ch->qp); in srpt_cm_rtu_recv()
2638 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list, in srpt_cm_rtu_recv()
2641 srpt_handle_new_iu(ch, ioctx, NULL); in srpt_cm_rtu_recv()
2644 srpt_close_ch(ch); in srpt_cm_rtu_recv()
2665 struct srpt_rdma_ch *ch; in srpt_cm_dreq_recv() local
2669 ch = srpt_find_channel(cm_id->context, cm_id); in srpt_cm_dreq_recv()
2670 BUG_ON(!ch); in srpt_cm_dreq_recv()
2672 pr_debug("cm_id= %p ch->state= %d\n", cm_id, srpt_get_ch_state(ch)); in srpt_cm_dreq_recv()
2674 spin_lock_irqsave(&ch->spinlock, flags); in srpt_cm_dreq_recv()
2675 switch (ch->state) { in srpt_cm_dreq_recv()
2679 ch->state = CH_DISCONNECTING; in srpt_cm_dreq_recv()
2684 WARN(true, "unexpected channel state %d\n", ch->state); in srpt_cm_dreq_recv()
2687 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_cm_dreq_recv()
2690 if (ib_send_cm_drep(ch->cm_id, NULL, 0) < 0) in srpt_cm_dreq_recv()
2693 ch->sess_name); in srpt_cm_dreq_recv()
2764 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, in srpt_perform_rdmas() argument
2780 sq_wr_avail = atomic_sub_return(n_rdma, &ch->sq_wr_avail); in srpt_perform_rdmas()
2817 ret = ib_post_send(ch->qp, &wr.wr, &bad_wr); in srpt_perform_rdmas()
2829 while (ch->state == CH_LIVE && in srpt_perform_rdmas()
2830 ib_post_send(ch->qp, &wr.wr, &bad_wr) != 0) { in srpt_perform_rdmas()
2835 while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) { in srpt_perform_rdmas()
2843 atomic_add(n_rdma, &ch->sq_wr_avail); in srpt_perform_rdmas()
2850 static int srpt_xfer_data(struct srpt_rdma_ch *ch, in srpt_xfer_data() argument
2855 ret = srpt_map_sg_to_ib_sge(ch, ioctx); in srpt_xfer_data()
2861 ret = srpt_perform_rdmas(ch, ioctx); in srpt_xfer_data()
2875 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_xfer_data()
2892 struct srpt_rdma_ch *ch; in srpt_write_pending() local
2903 ch = ioctx->ch; in srpt_write_pending()
2904 BUG_ON(!ch); in srpt_write_pending()
2906 ch_state = srpt_get_ch_state(ch); in srpt_write_pending()
2923 ret = srpt_xfer_data(ch, ioctx); in srpt_write_pending()
2948 struct srpt_rdma_ch *ch; in srpt_queue_response() local
2958 ch = ioctx->ch; in srpt_queue_response()
2959 BUG_ON(!ch); in srpt_queue_response()
2973 ch, ioctx->ioctx.index, ioctx->state); in srpt_queue_response()
2980 atomic_inc(&ch->req_lim_delta); in srpt_queue_response()
2990 ret = srpt_xfer_data(ch, ioctx); in srpt_queue_response()
2999 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag, in srpt_queue_response()
3004 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status, in srpt_queue_response()
3007 ret = srpt_post_send(ch, ioctx, resp_len); in srpt_queue_response()
3011 srpt_unmap_sg_to_ib_sge(ch, ioctx); in srpt_queue_response()
3033 srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); in srpt_aborted_task()
3073 struct srpt_rdma_ch *ch, *tmp_ch; in srpt_release_sdev() local
3081 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) in srpt_release_sdev()
3082 __srpt_close_ch(ch); in srpt_release_sdev()
3346 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_release_cmd() local
3358 spin_lock_irqsave(&ch->spinlock, flags); in srpt_release_cmd()
3359 list_add(&ioctx->free_list, &ch->free_list); in srpt_release_cmd()
3360 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_release_cmd()
3373 struct srpt_rdma_ch *ch; in srpt_close_session() local
3377 ch = se_sess->fabric_sess_ptr; in srpt_close_session()
3378 WARN_ON(ch->sess != se_sess); in srpt_close_session()
3380 pr_debug("ch %p state %d\n", ch, srpt_get_ch_state(ch)); in srpt_close_session()
3382 sdev = ch->sport->sdev; in srpt_close_session()
3384 BUG_ON(ch->release_done); in srpt_close_session()
3385 ch->release_done = &release_done; in srpt_close_session()
3386 __srpt_close_ch(ch); in srpt_close_session()