196 	struct ntb_transport_ctx *nt;  member
249 #define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)  argument
299 static int ntb_bus_init(struct ntb_transport_ctx *nt)  in ntb_bus_init()  argument
301 	list_add_tail(&nt->entry, &ntb_transport_list);  in ntb_bus_init()
305 static void ntb_bus_remove(struct ntb_transport_ctx *nt)  in ntb_bus_remove()  argument
309 	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {  in ntb_bus_remove()
316 	list_del(&nt->entry);  in ntb_bus_remove()
336 	struct ntb_transport_ctx *nt;  in ntb_transport_unregister_client_dev()  local
338 	list_for_each_entry(nt, &ntb_transport_list, entry)  in ntb_transport_unregister_client_dev()
339 		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)  in ntb_transport_unregister_client_dev()
357 	struct ntb_transport_ctx *nt;  in ntb_transport_register_client_dev()  local
364 	list_for_each_entry(nt, &ntb_transport_list, entry) {  in ntb_transport_register_client_dev()
367 		node = dev_to_node(&nt->ndev->dev);  in ntb_transport_register_client_dev()
382 		dev->parent = &nt->ndev->dev;  in ntb_transport_register_client_dev()
390 		list_add_tail(&client_dev->entry, &nt->client_devs);  in ntb_transport_register_client_dev()
584 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,  in ntb_transport_setup_qp_mw()  argument
587 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_mw()
593 	mw_count = nt->mw_count;  in ntb_transport_setup_qp_mw()
594 	qp_count = nt->qp_count;  in ntb_transport_setup_qp_mw()
596 	mw_num = QP_TO_MW(nt, qp_num);  in ntb_transport_setup_qp_mw()
597 	mw = &nt->mw_vec[mw_num];  in ntb_transport_setup_qp_mw()
634 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)  in ntb_free_mw()  argument
636 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_free_mw()
637 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_free_mw()
642 	ntb_mw_clear_trans(nt->ndev, num_mw);  in ntb_free_mw()
650 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,  in ntb_set_mw()  argument
653 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_set_mw()
654 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_set_mw()
669 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
694 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
699 	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);  in ntb_set_mw()
702 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
733 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup()  local
734 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_qp_link_cleanup()
750 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup_work()  local
754 	if (nt->link_is_up)  in ntb_qp_link_cleanup_work()
764 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)  in ntb_transport_link_cleanup()  argument
770 	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;  in ntb_transport_link_cleanup()
773 	for (i = 0; i < nt->qp_count; i++)  in ntb_transport_link_cleanup()
775 			qp = &nt->qp_vec[i];  in ntb_transport_link_cleanup()
781 	if (!nt->link_is_up)  in ntb_transport_link_cleanup()
782 		cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_link_cleanup()
789 		ntb_spad_write(nt->ndev, i, 0);  in ntb_transport_link_cleanup()
794 	struct ntb_transport_ctx *nt =  in ntb_transport_link_cleanup_work()  local
797 	ntb_transport_link_cleanup(nt);  in ntb_transport_link_cleanup_work()
802 	struct ntb_transport_ctx *nt = data;  in ntb_transport_event_callback()  local
804 	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)  in ntb_transport_event_callback()
805 		schedule_delayed_work(&nt->link_work, 0);  in ntb_transport_event_callback()
807 		schedule_work(&nt->link_cleanup);  in ntb_transport_event_callback()
812 	struct ntb_transport_ctx *nt =  in ntb_transport_link_work()  local
814 	struct ntb_dev *ndev = nt->ndev;  in ntb_transport_link_work()
821 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
822 		size = nt->mw_vec[i].phys_size;  in ntb_transport_link_work()
834 	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);  in ntb_transport_link_work()
836 	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);  in ntb_transport_link_work()
848 	if (val != nt->qp_count)  in ntb_transport_link_work()
853 	if (val != nt->mw_count)  in ntb_transport_link_work()
856 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
867 		rc = ntb_set_mw(nt, i, val64);  in ntb_transport_link_work()
872 	nt->link_is_up = true;  in ntb_transport_link_work()
874 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_link_work()
875 		struct ntb_transport_qp *qp = &nt->qp_vec[i];  in ntb_transport_link_work()
877 		ntb_transport_setup_qp_mw(nt, i);  in ntb_transport_link_work()
886 	for (i = 0; i < nt->mw_count; i++)  in ntb_transport_link_work()
887 		ntb_free_mw(nt, i);  in ntb_transport_link_work()
890 		schedule_delayed_work(&nt->link_work,  in ntb_transport_link_work()
900 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_work()  local
903 	WARN_ON(!nt->link_is_up);  in ntb_qp_link_work()
905 	val = ntb_spad_read(nt->ndev, QP_LINKS);  in ntb_qp_link_work()
907 	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));  in ntb_qp_link_work()
910 	ntb_peer_spad_read(nt->ndev, QP_LINKS);  in ntb_qp_link_work()
922 	} else if (nt->link_is_up)  in ntb_qp_link_work()
927 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,  in ntb_transport_init_queue()  argument
937 	mw_count = nt->mw_count;  in ntb_transport_init_queue()
938 	qp_count = nt->qp_count;  in ntb_transport_init_queue()
940 	mw_num = QP_TO_MW(nt, qp_num);  in ntb_transport_init_queue()
942 	qp = &nt->qp_vec[qp_num];  in ntb_transport_init_queue()
944 	qp->transport = nt;  in ntb_transport_init_queue()
945 	qp->ndev = nt->ndev;  in ntb_transport_init_queue()
955 	mw_base = nt->mw_vec[mw_num].phys_addr;  in ntb_transport_init_queue()
956 	mw_size = nt->mw_vec[mw_num].phys_size;  in ntb_transport_init_queue()
961 	qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;  in ntb_transport_init_queue()
976 	if (nt->debugfs_node_dir) {  in ntb_transport_init_queue()
981 						     nt->debugfs_node_dir);  in ntb_transport_init_queue()
1010 	struct ntb_transport_ctx *nt;  in ntb_transport_probe()  local
1026 	nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);  in ntb_transport_probe()
1027 	if (!nt)  in ntb_transport_probe()
1030 	nt->ndev = ndev;  in ntb_transport_probe()
1034 	nt->mw_count = mw_count;  in ntb_transport_probe()
1036 	nt->mw_vec = kzalloc_node(mw_count * sizeof(*nt->mw_vec),  in ntb_transport_probe()
1038 	if (!nt->mw_vec) {  in ntb_transport_probe()
1044 		mw = &nt->mw_vec[i];  in ntb_transport_probe()
1073 	nt->qp_count = qp_count;  in ntb_transport_probe()
1074 	nt->qp_bitmap = qp_bitmap;  in ntb_transport_probe()
1075 	nt->qp_bitmap_free = qp_bitmap;  in ntb_transport_probe()
1077 	nt->qp_vec = kzalloc_node(qp_count * sizeof(*nt->qp_vec),  in ntb_transport_probe()
1079 	if (!nt->qp_vec) {  in ntb_transport_probe()
1085 		nt->debugfs_node_dir =  in ntb_transport_probe()
1091 		rc = ntb_transport_init_queue(nt, i);  in ntb_transport_probe()
1096 	INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);  in ntb_transport_probe()
1097 	INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);  in ntb_transport_probe()
1099 	rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops);  in ntb_transport_probe()
1103 	INIT_LIST_HEAD(&nt->client_devs);  in ntb_transport_probe()
1104 	rc = ntb_bus_init(nt);  in ntb_transport_probe()
1108 	nt->link_is_up = false;  in ntb_transport_probe()
1117 	kfree(nt->qp_vec);  in ntb_transport_probe()
1120 		mw = &nt->mw_vec[i];  in ntb_transport_probe()
1123 	kfree(nt->mw_vec);  in ntb_transport_probe()
1125 	kfree(nt);  in ntb_transport_probe()
1131 	struct ntb_transport_ctx *nt = ndev->ctx;  in ntb_transport_free()  local
1136 	ntb_transport_link_cleanup(nt);  in ntb_transport_free()
1137 	cancel_work_sync(&nt->link_cleanup);  in ntb_transport_free()
1138 	cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_free()
1140 	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;  in ntb_transport_free()
1143 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_free()
1144 		qp = &nt->qp_vec[i];  in ntb_transport_free()
1153 	ntb_bus_remove(nt);  in ntb_transport_free()
1155 	for (i = nt->mw_count; i--; ) {  in ntb_transport_free()
1156 		ntb_free_mw(nt, i);  in ntb_transport_free()
1157 		iounmap(nt->mw_vec[i].vbase);  in ntb_transport_free()
1160 	kfree(nt->qp_vec);  in ntb_transport_free()
1161 	kfree(nt->mw_vec);  in ntb_transport_free()
1162 	kfree(nt);  in ntb_transport_free()
1611 	struct ntb_transport_ctx *nt;  in ntb_transport_create_queue()  local
1622 	nt = ndev->ctx;  in ntb_transport_create_queue()
1626 	free_queue = ffs(nt->qp_bitmap);  in ntb_transport_create_queue()
1633 	qp = &nt->qp_vec[free_queue];  in ntb_transport_create_queue()
1636 	nt->qp_bitmap_free &= ~qp_bit;  in ntb_transport_create_queue()
1706 	nt->qp_bitmap_free |= qp_bit;  in ntb_transport_create_queue()
2026 	struct ntb_transport_ctx *nt = data;  in ntb_transport_doorbell_callback()  local
2031 	db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free &  in ntb_transport_doorbell_callback()
2032 		   ntb_db_vector_mask(nt->ndev, vector));  in ntb_transport_doorbell_callback()
2036 		qp = &nt->qp_vec[qp_num];  in ntb_transport_doorbell_callback()