251 static int ntb_bus_init(struct ntb_transport *nt)  in ntb_bus_init()  argument
259 	list_add(&nt->entry, &ntb_transport_list);  in ntb_bus_init()
264 static void ntb_bus_remove(struct ntb_transport *nt)  in ntb_bus_remove()  argument
268 	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {  in ntb_bus_remove()
275 	list_del(&nt->entry);  in ntb_bus_remove()
298 	struct ntb_transport *nt;  in ntb_unregister_client_dev()  local
300 	list_for_each_entry(nt, &ntb_transport_list, entry)  in ntb_unregister_client_dev()
301 		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)  in ntb_unregister_client_dev()
319 	struct ntb_transport *nt;  in ntb_register_client_dev()  local
325 	list_for_each_entry(nt, &ntb_transport_list, entry) {  in ntb_register_client_dev()
341 		dev->parent = &ntb_query_pdev(nt->ndev)->dev;  in ntb_register_client_dev()
349 		list_add_tail(&client_dev->entry, &nt->client_devs);  in ntb_register_client_dev()
500 static void ntb_transport_setup_qp_mw(struct ntb_transport *nt,  in ntb_transport_setup_qp_mw()  argument
503 	struct ntb_transport_qp *qp = &nt->qps[qp_num];  in ntb_transport_setup_qp_mw()
508 	mw_max = ntb_max_mw(nt->ndev);  in ntb_transport_setup_qp_mw()
509 	mw_num = QP_TO_MW(nt->ndev, qp_num);  in ntb_transport_setup_qp_mw()
511 	WARN_ON(nt->mw[mw_num].virt_addr == NULL);  in ntb_transport_setup_qp_mw()
513 	if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max)  in ntb_transport_setup_qp_mw()
514 		num_qps_mw = nt->max_qps / mw_max + 1;  in ntb_transport_setup_qp_mw()
516 		num_qps_mw = nt->max_qps / mw_max;  in ntb_transport_setup_qp_mw()
518 	rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw;  in ntb_transport_setup_qp_mw()
519 	qp->rx_buff = nt->mw[mw_num].virt_addr + qp_num / mw_max * rx_size;  in ntb_transport_setup_qp_mw()
543 static void ntb_free_mw(struct ntb_transport *nt, int num_mw)  in ntb_free_mw()  argument
545 	struct ntb_transport_mw *mw = &nt->mw[num_mw];  in ntb_free_mw()
546 	struct pci_dev *pdev = ntb_query_pdev(nt->ndev);  in ntb_free_mw()
555 static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size)  in ntb_set_mw()  argument
557 	struct ntb_transport_mw *mw = &nt->mw[num_mw];  in ntb_set_mw()
558 	struct pci_dev *pdev = ntb_query_pdev(nt->ndev);  in ntb_set_mw()
565 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
588 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
593 	ntb_set_mw_addr(nt->ndev, num_mw, mw->dma_addr);  in ntb_set_mw()
600 	struct ntb_transport *nt = qp->transport;  in ntb_qp_link_cleanup()  local
601 	struct pci_dev *pdev = ntb_query_pdev(nt->ndev);  in ntb_qp_link_cleanup()
620 	struct ntb_transport *nt = qp->transport;  in ntb_qp_link_cleanup_work()  local
624 	if (nt->transport_link == NTB_LINK_UP)  in ntb_qp_link_cleanup_work()
634 static void ntb_transport_link_cleanup(struct ntb_transport *nt)  in ntb_transport_link_cleanup()  argument
639 	for (i = 0; i < nt->max_qps; i++)  in ntb_transport_link_cleanup()
640 		if (!test_bit(i, &nt->qp_bitmap))  in ntb_transport_link_cleanup()
641 			ntb_qp_link_cleanup(&nt->qps[i]);  in ntb_transport_link_cleanup()
643 	if (nt->transport_link == NTB_LINK_DOWN)  in ntb_transport_link_cleanup()
644 		cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_link_cleanup()
646 		nt->transport_link = NTB_LINK_DOWN;  in ntb_transport_link_cleanup()
653 		ntb_write_local_spad(nt->ndev, i, 0);  in ntb_transport_link_cleanup()
658 	struct ntb_transport *nt = container_of(work, struct ntb_transport,  in ntb_transport_link_cleanup_work()  local
661 	ntb_transport_link_cleanup(nt);  in ntb_transport_link_cleanup_work()
666 	struct ntb_transport *nt = data;  in ntb_transport_event_callback()  local
670 		schedule_delayed_work(&nt->link_work, 0);  in ntb_transport_event_callback()
673 		schedule_work(&nt->link_cleanup);  in ntb_transport_event_callback()
682 	struct ntb_transport *nt = container_of(work, struct ntb_transport,  in ntb_transport_link_work()  local
684 	struct ntb_device *ndev = nt->ndev;  in ntb_transport_link_work()
717 	rc = ntb_write_remote_spad(ndev, NUM_QPS, nt->max_qps);  in ntb_transport_link_work()
720 			nt->max_qps, NUM_QPS);  in ntb_transport_link_work()
748 	if (val != nt->max_qps)  in ntb_transport_link_work()
785 		rc = ntb_set_mw(nt, i, val64);  in ntb_transport_link_work()
790 	nt->transport_link = NTB_LINK_UP;  in ntb_transport_link_work()
792 	for (i = 0; i < nt->max_qps; i++) {  in ntb_transport_link_work()
793 		struct ntb_transport_qp *qp = &nt->qps[i];  in ntb_transport_link_work()
795 		ntb_transport_setup_qp_mw(nt, i);  in ntb_transport_link_work()
805 		ntb_free_mw(nt, i);  in ntb_transport_link_work()
808 		schedule_delayed_work(&nt->link_work,  in ntb_transport_link_work()
818 	struct ntb_transport *nt = qp->transport;  in ntb_qp_link_work()  local
821 	WARN_ON(nt->transport_link != NTB_LINK_UP);  in ntb_qp_link_work()
823 	rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val);  in ntb_qp_link_work()
829 	rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num);  in ntb_qp_link_work()
835 	rc = ntb_read_remote_spad(nt->ndev, QP_LINKS, &val);  in ntb_qp_link_work()
848 	} else if (nt->transport_link == NTB_LINK_UP)  in ntb_qp_link_work()
853 static int ntb_transport_init_queue(struct ntb_transport *nt,  in ntb_transport_init_queue()  argument
861 	mw_max = ntb_max_mw(nt->ndev);  in ntb_transport_init_queue()
862 	mw_num = QP_TO_MW(nt->ndev, qp_num);  in ntb_transport_init_queue()
864 	qp = &nt->qps[qp_num];  in ntb_transport_init_queue()
866 	qp->transport = nt;  in ntb_transport_init_queue()
867 	qp->ndev = nt->ndev;  in ntb_transport_init_queue()
872 	if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max)  in ntb_transport_init_queue()
873 		num_qps_mw = nt->max_qps / mw_max + 1;  in ntb_transport_init_queue()
875 		num_qps_mw = nt->max_qps / mw_max;  in ntb_transport_init_queue()
879 	qp->tx_mw = ntb_get_mw_vbase(nt->ndev, mw_num) + qp_offset;  in ntb_transport_init_queue()
894 	if (ntb_query_debugfs(nt->ndev)) {  in ntb_transport_init_queue()
899 						 ntb_query_debugfs(nt->ndev));  in ntb_transport_init_queue()
922 	struct ntb_transport *nt;  in ntb_transport_init()  local
925 	nt = kzalloc(sizeof(struct ntb_transport), GFP_KERNEL);  in ntb_transport_init()
926 	if (!nt)  in ntb_transport_init()
929 	nt->ndev = ntb_register_transport(pdev, nt);  in ntb_transport_init()
930 	if (!nt->ndev) {  in ntb_transport_init()
935 	nt->mw = kcalloc(ntb_max_mw(nt->ndev), sizeof(struct ntb_transport_mw),  in ntb_transport_init()
937 	if (!nt->mw) {  in ntb_transport_init()
943 		nt->max_qps = min(ntb_max_cbs(nt->ndev), max_num_clients);  in ntb_transport_init()
945 		nt->max_qps = min(ntb_max_cbs(nt->ndev), ntb_max_mw(nt->ndev));  in ntb_transport_init()
947 	nt->qps = kcalloc(nt->max_qps, sizeof(struct ntb_transport_qp),  in ntb_transport_init()
949 	if (!nt->qps) {  in ntb_transport_init()
954 	nt->qp_bitmap = ((u64) 1 << nt->max_qps) - 1;  in ntb_transport_init()
956 	for (i = 0; i < nt->max_qps; i++) {  in ntb_transport_init()
957 		rc = ntb_transport_init_queue(nt, i);  in ntb_transport_init()
962 	INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);  in ntb_transport_init()
963 	INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);  in ntb_transport_init()
965 	rc = ntb_register_event_callback(nt->ndev,  in ntb_transport_init()
970 	INIT_LIST_HEAD(&nt->client_devs);  in ntb_transport_init()
971 	rc = ntb_bus_init(nt);  in ntb_transport_init()
975 	if (ntb_hw_link_status(nt->ndev))  in ntb_transport_init()
976 		schedule_delayed_work(&nt->link_work, 0);  in ntb_transport_init()
981 	ntb_unregister_event_callback(nt->ndev);  in ntb_transport_init()
983 	kfree(nt->qps);  in ntb_transport_init()
985 	kfree(nt->mw);  in ntb_transport_init()
987 	ntb_unregister_transport(nt->ndev);  in ntb_transport_init()
989 	kfree(nt);  in ntb_transport_init()
995 	struct ntb_transport *nt = transport;  in ntb_transport_free()  local
996 	struct ntb_device *ndev = nt->ndev;  in ntb_transport_free()
999 	ntb_transport_link_cleanup(nt);  in ntb_transport_free()
1002 	for (i = 0; i < nt->max_qps; i++) {  in ntb_transport_free()
1003 		if (!test_bit(i, &nt->qp_bitmap))  in ntb_transport_free()
1004 			ntb_transport_free_queue(&nt->qps[i]);  in ntb_transport_free()
1005 		debugfs_remove_recursive(nt->qps[i].debugfs_dir);  in ntb_transport_free()
1008 	ntb_bus_remove(nt);  in ntb_transport_free()
1010 	cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_free()
1015 		ntb_free_mw(nt, i);  in ntb_transport_free()
1017 	kfree(nt->qps);  in ntb_transport_free()
1018 	kfree(nt->mw);  in ntb_transport_free()
1020 	kfree(nt);  in ntb_transport_free()
1430 	struct ntb_transport *nt;  in ntb_transport_create_queue()  local
1434 	nt = ntb_find_transport(pdev);  in ntb_transport_create_queue()
1435 	if (!nt)  in ntb_transport_create_queue()
1438 	free_queue = ffs(nt->qp_bitmap);  in ntb_transport_create_queue()
1445 	clear_bit(free_queue, &nt->qp_bitmap);  in ntb_transport_create_queue()
1447 	qp = &nt->qps[free_queue];  in ntb_transport_create_queue()
1497 	set_bit(free_queue, &nt->qp_bitmap);  in ntb_transport_create_queue()