115 static inline const char *srpt_sdev_name(struct srpt_device *sdev)  in srpt_sdev_name()  argument
117 	return sdev->device->name;  in srpt_sdev_name()
175 	struct srpt_device *sdev;  in srpt_event_handler()  local
178 	sdev = ib_get_client_data(event->device, &srpt_client);  in srpt_event_handler()
179 	if (!sdev || sdev->device != event->device)  in srpt_event_handler()
183 		 srpt_sdev_name(sdev));  in srpt_event_handler()
187 		if (event->element.port_num <= sdev->device->phys_port_cnt) {  in srpt_event_handler()
188 			sport = &sdev->port[event->element.port_num - 1];  in srpt_event_handler()
200 		if (event->element.port_num <= sdev->device->phys_port_cnt) {  in srpt_event_handler()
201 			sport = &sdev->port[event->element.port_num - 1];  in srpt_event_handler()
324 	struct srpt_device *sdev = sport->sdev;  in srpt_get_ioc()  local
344 	iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);  in srpt_get_ioc()
345 	iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);  in srpt_get_ioc()
346 	iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);  in srpt_get_ioc()
347 	iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);  in srpt_get_ioc()
353 	iocp->send_queue_depth = cpu_to_be16(sdev->srq_size);  in srpt_get_ioc()
536 	ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);  in srpt_refresh_port()
540 	ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);  in srpt_refresh_port()
547 	ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid,  in srpt_refresh_port()
559 		sport->mad_agent = ib_register_mad_agent(sport->sdev->device,  in srpt_refresh_port()
579 	ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);  in srpt_refresh_port()
591 static void srpt_unregister_mad_agent(struct srpt_device *sdev)  in srpt_unregister_mad_agent()  argument
599 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {  in srpt_unregister_mad_agent()
600 		sport = &sdev->port[i - 1];  in srpt_unregister_mad_agent()
602 		if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)  in srpt_unregister_mad_agent()
614 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,  in srpt_alloc_ioctx()  argument
628 	ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);  in srpt_alloc_ioctx()
629 	if (ib_dma_mapping_error(sdev->device, ioctx->dma))  in srpt_alloc_ioctx()
645 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,  in srpt_free_ioctx()  argument
651 	ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);  in srpt_free_ioctx()
664 static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,  in srpt_alloc_ioctx_ring()  argument
678 		ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);  in srpt_alloc_ioctx_ring()
687 		srpt_free_ioctx(sdev, ring[i], dma_size, dir);  in srpt_alloc_ioctx_ring()
698 				 struct srpt_device *sdev, int ring_size,  in srpt_free_ioctx_ring()  argument
704 		srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);  in srpt_free_ioctx_ring()
774 static int srpt_post_recv(struct srpt_device *sdev,  in srpt_post_recv()  argument
780 	BUG_ON(!sdev);  in srpt_post_recv()
785 	list.lkey = sdev->pd->local_dma_lkey;  in srpt_post_recv()
791 	return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);  in srpt_post_recv()
804 	struct srpt_device *sdev = ch->sport->sdev;  in srpt_post_send()  local
815 	ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, len,  in srpt_post_send()
820 	list.lkey = sdev->pd->local_dma_lkey;  in srpt_post_send()
1068 		ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt,  in srpt_unmap_sg_to_ib_sge()
1080 	struct ib_device *dev = ch->sport->sdev->device;  in srpt_map_sg_to_ib_sge()
1105 	count = ib_dma_map_sg(ch->sport->sdev->device, sg, sg_cnt,  in srpt_map_sg_to_ib_sge()
1208 			sge->lkey = ch->sport->sdev->pd->local_dma_lkey;  in srpt_map_sg_to_ib_sge()
1818 	ib_dma_sync_single_for_cpu(ch->sport->sdev->device,  in srpt_handle_new_iu()
1867 	srpt_post_recv(ch->sport->sdev, recv_ioctx);  in srpt_handle_new_iu()
1876 	struct srpt_device *sdev = ch->sport->sdev;  in srpt_process_rcv_completion()  local
1887 		ioctx = sdev->ioctx_ring[index];  in srpt_process_rcv_completion()
2019 	struct srpt_device *sdev = sport->sdev;  in srpt_create_ch_ib()  local
2033 	ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch,  in srpt_create_ch_ib()
2047 	qp_init->srq = sdev->srq;  in srpt_create_ch_ib()
2053 	ch->qp = ib_create_qp(sdev->pd, qp_init);  in srpt_create_ch_ib()
2156 	struct srpt_device *sdev;  in srpt_close_ch()  local
2158 	sdev = ch->sport->sdev;  in srpt_close_ch()
2159 	spin_lock_irq(&sdev->spinlock);  in srpt_close_ch()
2161 	spin_unlock_irq(&sdev->spinlock);  in srpt_close_ch()
2199 	struct srpt_device *sdev;  in srpt_drain_channel()  local
2206 	sdev = cm_id->context;  in srpt_drain_channel()
2207 	BUG_ON(!sdev);  in srpt_drain_channel()
2208 	spin_lock_irq(&sdev->spinlock);  in srpt_drain_channel()
2209 	list_for_each_entry(ch, &sdev->rch_list, list) {  in srpt_drain_channel()
2220 	spin_unlock_irq(&sdev->spinlock);  in srpt_drain_channel()
2239 static struct srpt_rdma_ch *srpt_find_channel(struct srpt_device *sdev,  in srpt_find_channel()  argument
2246 	BUG_ON(!sdev);  in srpt_find_channel()
2249 	spin_lock_irq(&sdev->spinlock);  in srpt_find_channel()
2250 	list_for_each_entry(ch, &sdev->rch_list, list) {  in srpt_find_channel()
2256 	spin_unlock_irq(&sdev->spinlock);  in srpt_find_channel()
2277 	struct srpt_device *sdev;  in srpt_release_channel_work()  local
2284 	sdev = ch->sport->sdev;  in srpt_release_channel_work()
2285 	BUG_ON(!sdev);  in srpt_release_channel_work()
2301 			     ch->sport->sdev, ch->rq_size,  in srpt_release_channel_work()
2304 	spin_lock_irq(&sdev->spinlock);  in srpt_release_channel_work()
2306 	spin_unlock_irq(&sdev->spinlock);  in srpt_release_channel_work()
2311 	wake_up(&sdev->ch_releaseQ);  in srpt_release_channel_work()
2351 	struct srpt_device *sdev = cm_id->context;  in srpt_cm_req_recv()  local
2352 	struct srpt_port *sport = &sdev->port[param->port - 1];  in srpt_cm_req_recv()
2365 	if (WARN_ON(!sdev || !private_data))  in srpt_cm_req_recv()
2381 		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),  in srpt_cm_req_recv()
2382 		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));  in srpt_cm_req_recv()
2415 		spin_lock_irq(&sdev->spinlock);  in srpt_cm_req_recv()
2417 		list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {  in srpt_cm_req_recv()
2421 			    && param->listen_id == ch->sport->sdev->cm_id  in srpt_cm_req_recv()
2442 		spin_unlock_irq(&sdev->spinlock);  in srpt_cm_req_recv()
2470 	ch->sport = &sdev->port[param->port - 1];  in srpt_cm_req_recv()
2483 		srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,  in srpt_cm_req_recv()
2572 	spin_lock_irq(&sdev->spinlock);  in srpt_cm_req_recv()
2573 	list_add_tail(&ch->list, &sdev->rch_list);  in srpt_cm_req_recv()
2574 	spin_unlock_irq(&sdev->spinlock);  in srpt_cm_req_recv()
2591 			     ch->sport->sdev, ch->rq_size,  in srpt_cm_req_recv()
3057 static int srpt_ch_list_empty(struct srpt_device *sdev)  in srpt_ch_list_empty()  argument
3061 	spin_lock_irq(&sdev->spinlock);  in srpt_ch_list_empty()
3062 	res = list_empty(&sdev->rch_list);  in srpt_ch_list_empty()
3063 	spin_unlock_irq(&sdev->spinlock);  in srpt_ch_list_empty()
3071 static int srpt_release_sdev(struct srpt_device *sdev)  in srpt_release_sdev()  argument
3078 	BUG_ON(!sdev);  in srpt_release_sdev()
3080 	spin_lock_irq(&sdev->spinlock);  in srpt_release_sdev()
3081 	list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list)  in srpt_release_sdev()
3083 	spin_unlock_irq(&sdev->spinlock);  in srpt_release_sdev()
3085 	res = wait_event_interruptible(sdev->ch_releaseQ,  in srpt_release_sdev()
3086 				       srpt_ch_list_empty(sdev));  in srpt_release_sdev()
3096 	struct srpt_device *sdev;  in __srpt_lookup_port()  local
3100 	list_for_each_entry(sdev, &srpt_dev_list, list) {  in __srpt_lookup_port()
3101 		dev = sdev->device;  in __srpt_lookup_port()
3106 			sport = &sdev->port[i];  in __srpt_lookup_port()
3132 	struct srpt_device *sdev;  in srpt_add_one()  local
3140 	sdev = kzalloc(sizeof *sdev, GFP_KERNEL);  in srpt_add_one()
3141 	if (!sdev)  in srpt_add_one()
3144 	sdev->device = device;  in srpt_add_one()
3145 	INIT_LIST_HEAD(&sdev->rch_list);  in srpt_add_one()
3146 	init_waitqueue_head(&sdev->ch_releaseQ);  in srpt_add_one()
3147 	spin_lock_init(&sdev->spinlock);  in srpt_add_one()
3149 	if (ib_query_device(device, &sdev->dev_attr))  in srpt_add_one()
3152 	sdev->pd = ib_alloc_pd(device);  in srpt_add_one()
3153 	if (IS_ERR(sdev->pd))  in srpt_add_one()
3156 	sdev->srq_size = min(srpt_srq_size, sdev->dev_attr.max_srq_wr);  in srpt_add_one()
3159 	srq_attr.srq_context = (void *)sdev;  in srpt_add_one()
3160 	srq_attr.attr.max_wr = sdev->srq_size;  in srpt_add_one()
3165 	sdev->srq = ib_create_srq(sdev->pd, &srq_attr);  in srpt_add_one()
3166 	if (IS_ERR(sdev->srq))  in srpt_add_one()
3170 		 __func__, sdev->srq_size, sdev->dev_attr.max_srq_wr,  in srpt_add_one()
3176 	sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);  in srpt_add_one()
3177 	if (IS_ERR(sdev->cm_id))  in srpt_add_one()
3191 	if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0))  in srpt_add_one()
3194 	INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,  in srpt_add_one()
3196 	if (ib_register_event_handler(&sdev->event_handler))  in srpt_add_one()
3199 	sdev->ioctx_ring = (struct srpt_recv_ioctx **)  in srpt_add_one()
3200 		srpt_alloc_ioctx_ring(sdev, sdev->srq_size,  in srpt_add_one()
3201 				      sizeof(*sdev->ioctx_ring[0]),  in srpt_add_one()
3203 	if (!sdev->ioctx_ring)  in srpt_add_one()
3206 	for (i = 0; i < sdev->srq_size; ++i)  in srpt_add_one()
3207 		srpt_post_recv(sdev, sdev->ioctx_ring[i]);  in srpt_add_one()
3209 	WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port));  in srpt_add_one()
3211 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {  in srpt_add_one()
3212 		sport = &sdev->port[i - 1];  in srpt_add_one()
3213 		sport->sdev = sdev;  in srpt_add_one()
3224 			       srpt_sdev_name(sdev), i);  in srpt_add_one()
3234 	list_add_tail(&sdev->list, &srpt_dev_list);  in srpt_add_one()
3238 	ib_set_client_data(device, &srpt_client, sdev);  in srpt_add_one()
3243 	srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,  in srpt_add_one()
3244 			     sdev->srq_size, srp_max_req_size,  in srpt_add_one()
3247 	ib_unregister_event_handler(&sdev->event_handler);  in srpt_add_one()
3249 	ib_destroy_cm_id(sdev->cm_id);  in srpt_add_one()
3251 	ib_destroy_srq(sdev->srq);  in srpt_add_one()
3253 	ib_dealloc_pd(sdev->pd);  in srpt_add_one()
3255 	kfree(sdev);  in srpt_add_one()
3257 	sdev = NULL;  in srpt_add_one()
3267 	struct srpt_device *sdev = client_data;  in srpt_remove_one()  local
3270 	if (!sdev) {  in srpt_remove_one()
3275 	srpt_unregister_mad_agent(sdev);  in srpt_remove_one()
3277 	ib_unregister_event_handler(&sdev->event_handler);  in srpt_remove_one()
3280 	for (i = 0; i < sdev->device->phys_port_cnt; i++)  in srpt_remove_one()
3281 		cancel_work_sync(&sdev->port[i].work);  in srpt_remove_one()
3283 	ib_destroy_cm_id(sdev->cm_id);  in srpt_remove_one()
3291 	list_del(&sdev->list);  in srpt_remove_one()
3293 	srpt_release_sdev(sdev);  in srpt_remove_one()
3295 	ib_destroy_srq(sdev->srq);  in srpt_remove_one()
3296 	ib_dealloc_pd(sdev->pd);  in srpt_remove_one()
3298 	srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,  in srpt_remove_one()
3299 			     sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);  in srpt_remove_one()
3300 	sdev->ioctx_ring = NULL;  in srpt_remove_one()
3301 	kfree(sdev);  in srpt_remove_one()
3374 	struct srpt_device *sdev;  in srpt_close_session()  local
3382 	sdev = ch->sport->sdev;  in srpt_close_session()
3383 	spin_lock_irq(&sdev->spinlock);  in srpt_close_session()
3387 	spin_unlock_irq(&sdev->spinlock);  in srpt_close_session()