Lines Matching refs:qp
84 struct ntb_transport_qp *qp; member
109 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
119 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
203 #define QP_TO_MW(ndev, qp) ((qp) % ntb_max_mw(ndev)) argument
398 struct ntb_transport_qp *qp; in debugfs_read() local
408 qp = filp->private_data; in debugfs_read()
413 "rx_bytes - \t%llu\n", qp->rx_bytes); in debugfs_read()
415 "rx_pkts - \t%llu\n", qp->rx_pkts); in debugfs_read()
417 "rx_memcpy - \t%llu\n", qp->rx_memcpy); in debugfs_read()
419 "rx_async - \t%llu\n", qp->rx_async); in debugfs_read()
421 "rx_ring_empty - %llu\n", qp->rx_ring_empty); in debugfs_read()
423 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); in debugfs_read()
425 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); in debugfs_read()
427 "rx_err_ver - \t%llu\n", qp->rx_err_ver); in debugfs_read()
429 "rx_buff - \t%p\n", qp->rx_buff); in debugfs_read()
431 "rx_index - \t%u\n", qp->rx_index); in debugfs_read()
433 "rx_max_entry - \t%u\n", qp->rx_max_entry); in debugfs_read()
436 "tx_bytes - \t%llu\n", qp->tx_bytes); in debugfs_read()
438 "tx_pkts - \t%llu\n", qp->tx_pkts); in debugfs_read()
440 "tx_memcpy - \t%llu\n", qp->tx_memcpy); in debugfs_read()
442 "tx_async - \t%llu\n", qp->tx_async); in debugfs_read()
444 "tx_ring_full - \t%llu\n", qp->tx_ring_full); in debugfs_read()
446 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); in debugfs_read()
448 "tx_mw - \t%p\n", qp->tx_mw); in debugfs_read()
450 "tx_index - \t%u\n", qp->tx_index); in debugfs_read()
452 "tx_max_entry - \t%u\n", qp->tx_max_entry); in debugfs_read()
455 "\nQP Link %s\n", (qp->qp_link == NTB_LINK_UP) ? in debugfs_read()
503 struct ntb_transport_qp *qp = &nt->qps[qp_num]; in ntb_transport_setup_qp_mw() local
519 qp->rx_buff = nt->mw[mw_num].virt_addr + qp_num / mw_max * rx_size; in ntb_transport_setup_qp_mw()
522 qp->remote_rx_info = qp->rx_buff + rx_size; in ntb_transport_setup_qp_mw()
525 qp->rx_max_frame = min(transport_mtu, rx_size / 2); in ntb_transport_setup_qp_mw()
526 qp->rx_max_entry = rx_size / qp->rx_max_frame; in ntb_transport_setup_qp_mw()
527 qp->rx_index = 0; in ntb_transport_setup_qp_mw()
529 qp->remote_rx_info->entry = qp->rx_max_entry - 1; in ntb_transport_setup_qp_mw()
532 for (i = 0; i < qp->rx_max_entry; i++) { in ntb_transport_setup_qp_mw()
533 void *offset = qp->rx_buff + qp->rx_max_frame * (i + 1) - in ntb_transport_setup_qp_mw()
538 qp->rx_pkts = 0; in ntb_transport_setup_qp_mw()
539 qp->tx_pkts = 0; in ntb_transport_setup_qp_mw()
540 qp->tx_index = 0; in ntb_transport_setup_qp_mw()
598 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) in ntb_qp_link_cleanup() argument
600 struct ntb_transport *nt = qp->transport; in ntb_qp_link_cleanup()
603 if (qp->qp_link == NTB_LINK_DOWN) { in ntb_qp_link_cleanup()
604 cancel_delayed_work_sync(&qp->link_work); in ntb_qp_link_cleanup()
608 if (qp->event_handler) in ntb_qp_link_cleanup()
609 qp->event_handler(qp->cb_data, NTB_LINK_DOWN); in ntb_qp_link_cleanup()
611 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); in ntb_qp_link_cleanup()
612 qp->qp_link = NTB_LINK_DOWN; in ntb_qp_link_cleanup()
617 struct ntb_transport_qp *qp = container_of(work, in ntb_qp_link_cleanup_work() local
620 struct ntb_transport *nt = qp->transport; in ntb_qp_link_cleanup_work()
622 ntb_qp_link_cleanup(qp); in ntb_qp_link_cleanup_work()
625 schedule_delayed_work(&qp->link_work, in ntb_qp_link_cleanup_work()
629 static void ntb_qp_link_down(struct ntb_transport_qp *qp) in ntb_qp_link_down() argument
631 schedule_work(&qp->link_cleanup); in ntb_qp_link_down()
793 struct ntb_transport_qp *qp = &nt->qps[i]; in ntb_transport_link_work() local
797 if (qp->client_ready == NTB_LINK_UP) in ntb_transport_link_work()
798 schedule_delayed_work(&qp->link_work, 0); in ntb_transport_link_work()
814 struct ntb_transport_qp *qp = container_of(work, in ntb_qp_link_work() local
817 struct pci_dev *pdev = ntb_query_pdev(qp->ndev); in ntb_qp_link_work()
818 struct ntb_transport *nt = qp->transport; in ntb_qp_link_work()
829 rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num); in ntb_qp_link_work()
832 val | 1 << qp->qp_num, QP_LINKS); in ntb_qp_link_work()
842 if (1 << qp->qp_num & val) { in ntb_qp_link_work()
843 qp->qp_link = NTB_LINK_UP; in ntb_qp_link_work()
845 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); in ntb_qp_link_work()
846 if (qp->event_handler) in ntb_qp_link_work()
847 qp->event_handler(qp->cb_data, NTB_LINK_UP); in ntb_qp_link_work()
849 schedule_delayed_work(&qp->link_work, in ntb_qp_link_work()
856 struct ntb_transport_qp *qp; in ntb_transport_init_queue() local
864 qp = &nt->qps[qp_num]; in ntb_transport_init_queue()
865 qp->qp_num = qp_num; in ntb_transport_init_queue()
866 qp->transport = nt; in ntb_transport_init_queue()
867 qp->ndev = nt->ndev; in ntb_transport_init_queue()
868 qp->qp_link = NTB_LINK_DOWN; in ntb_transport_init_queue()
869 qp->client_ready = NTB_LINK_DOWN; in ntb_transport_init_queue()
870 qp->event_handler = NULL; in ntb_transport_init_queue()
877 tx_size = (unsigned int) ntb_get_mw_size(qp->ndev, mw_num) / num_qps_mw; in ntb_transport_init_queue()
879 qp->tx_mw = ntb_get_mw_vbase(nt->ndev, mw_num) + qp_offset; in ntb_transport_init_queue()
880 if (!qp->tx_mw) in ntb_transport_init_queue()
883 qp->tx_mw_phys = ntb_get_mw_base(qp->ndev, mw_num) + qp_offset; in ntb_transport_init_queue()
884 if (!qp->tx_mw_phys) in ntb_transport_init_queue()
888 qp->rx_info = qp->tx_mw + tx_size; in ntb_transport_init_queue()
891 qp->tx_max_frame = min(transport_mtu, tx_size / 2); in ntb_transport_init_queue()
892 qp->tx_max_entry = tx_size / qp->tx_max_frame; in ntb_transport_init_queue()
898 qp->debugfs_dir = debugfs_create_dir(debugfs_name, in ntb_transport_init_queue()
901 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, in ntb_transport_init_queue()
902 qp->debugfs_dir, qp, in ntb_transport_init_queue()
906 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); in ntb_transport_init_queue()
907 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); in ntb_transport_init_queue()
909 spin_lock_init(&qp->ntb_rx_pend_q_lock); in ntb_transport_init_queue()
910 spin_lock_init(&qp->ntb_rx_free_q_lock); in ntb_transport_init_queue()
911 spin_lock_init(&qp->ntb_tx_free_q_lock); in ntb_transport_init_queue()
913 INIT_LIST_HEAD(&qp->rx_pend_q); in ntb_transport_init_queue()
914 INIT_LIST_HEAD(&qp->rx_free_q); in ntb_transport_init_queue()
915 INIT_LIST_HEAD(&qp->tx_free_q); in ntb_transport_init_queue()
1026 struct ntb_transport_qp *qp = entry->qp; in ntb_rx_copy_callback() local
1035 iowrite32(entry->index, &qp->rx_info->entry); in ntb_rx_copy_callback()
1037 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); in ntb_rx_copy_callback()
1039 if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) in ntb_rx_copy_callback()
1040 qp->rx_handler(qp, qp->cb_data, cb_data, len); in ntb_rx_copy_callback()
1057 struct ntb_transport_qp *qp = entry->qp; in ntb_async_rx() local
1058 struct dma_chan *chan = qp->dma_chan; in ntb_async_rx()
1115 qp->last_cookie = cookie; in ntb_async_rx()
1117 qp->rx_async++; in ntb_async_rx()
1130 dma_sync_wait(chan, qp->last_cookie); in ntb_async_rx()
1133 qp->rx_memcpy++; in ntb_async_rx()
1136 static int ntb_process_rxc(struct ntb_transport_qp *qp) in ntb_process_rxc() argument
1142 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; in ntb_process_rxc()
1143 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); in ntb_process_rxc()
1145 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); in ntb_process_rxc()
1147 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, in ntb_process_rxc()
1150 qp->rx_err_no_buf++; in ntb_process_rxc()
1155 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, in ntb_process_rxc()
1156 &qp->rx_pend_q); in ntb_process_rxc()
1157 qp->rx_ring_empty++; in ntb_process_rxc()
1161 if (hdr->ver != (u32) qp->rx_pkts) { in ntb_process_rxc()
1162 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, in ntb_process_rxc()
1164 qp->qp_num, qp->rx_pkts, hdr->ver); in ntb_process_rxc()
1165 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, in ntb_process_rxc()
1166 &qp->rx_pend_q); in ntb_process_rxc()
1167 qp->rx_err_ver++; in ntb_process_rxc()
1172 ntb_qp_link_down(qp); in ntb_process_rxc()
1177 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, in ntb_process_rxc()
1179 qp->rx_index, hdr->ver, hdr->len, entry->len); in ntb_process_rxc()
1181 qp->rx_bytes += hdr->len; in ntb_process_rxc()
1182 qp->rx_pkts++; in ntb_process_rxc()
1185 qp->rx_err_oflow++; in ntb_process_rxc()
1186 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, in ntb_process_rxc()
1193 entry->index = qp->rx_index; in ntb_process_rxc()
1199 qp->rx_index++; in ntb_process_rxc()
1200 qp->rx_index %= qp->rx_max_entry; in ntb_process_rxc()
1205 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); in ntb_process_rxc()
1209 iowrite32(qp->rx_index, &qp->rx_info->entry); in ntb_process_rxc()
1216 struct ntb_transport_qp *qp = data; in ntb_transport_rxc_db() local
1219 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n", in ntb_transport_rxc_db()
1225 for (i = 0; i < qp->rx_max_entry; i++) { in ntb_transport_rxc_db()
1226 rc = ntb_process_rxc(qp); in ntb_transport_rxc_db()
1231 if (qp->dma_chan) in ntb_transport_rxc_db()
1232 dma_async_issue_pending(qp->dma_chan); in ntb_transport_rxc_db()
1240 struct ntb_transport_qp *qp = entry->qp; in ntb_tx_copy_callback() local
1247 ntb_ring_doorbell(qp->ndev, qp->qp_num); in ntb_tx_copy_callback()
1254 qp->tx_bytes += entry->len; in ntb_tx_copy_callback()
1256 if (qp->tx_handler) in ntb_tx_copy_callback()
1257 qp->tx_handler(qp, qp->cb_data, entry->cb_data, in ntb_tx_copy_callback()
1261 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); in ntb_tx_copy_callback()
1271 static void ntb_async_tx(struct ntb_transport_qp *qp, in ntb_async_tx() argument
1276 struct dma_chan *chan = qp->dma_chan; in ntb_async_tx()
1286 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; in ntb_async_tx()
1287 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); in ntb_async_tx()
1291 iowrite32((u32) qp->tx_pkts, &hdr->ver); in ntb_async_tx()
1300 dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; in ntb_async_tx()
1335 qp->tx_async++; in ntb_async_tx()
1344 qp->tx_memcpy++; in ntb_async_tx()
1347 static int ntb_process_tx(struct ntb_transport_qp *qp, in ntb_process_tx() argument
1350 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%lld - tx %u, entry len %d flags %x buff %p\n", in ntb_process_tx()
1351 qp->tx_pkts, qp->tx_index, entry->len, entry->flags, in ntb_process_tx()
1353 if (qp->tx_index == qp->remote_rx_info->entry) { in ntb_process_tx()
1354 qp->tx_ring_full++; in ntb_process_tx()
1358 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { in ntb_process_tx()
1359 if (qp->tx_handler) in ntb_process_tx()
1360 qp->tx_handler(qp->cb_data, qp, NULL, -EIO); in ntb_process_tx()
1362 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, in ntb_process_tx()
1363 &qp->tx_free_q); in ntb_process_tx()
1367 ntb_async_tx(qp, entry); in ntb_process_tx()
1369 qp->tx_index++; in ntb_process_tx()
1370 qp->tx_index %= qp->tx_max_entry; in ntb_process_tx()
1372 qp->tx_pkts++; in ntb_process_tx()
1377 static void ntb_send_link_down(struct ntb_transport_qp *qp) in ntb_send_link_down() argument
1379 struct pci_dev *pdev = ntb_query_pdev(qp->ndev); in ntb_send_link_down()
1383 if (qp->qp_link == NTB_LINK_DOWN) in ntb_send_link_down()
1386 qp->qp_link = NTB_LINK_DOWN; in ntb_send_link_down()
1387 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); in ntb_send_link_down()
1390 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); in ntb_send_link_down()
1404 rc = ntb_process_tx(qp, entry); in ntb_send_link_down()
1407 qp->qp_num); in ntb_send_link_down()
1429 struct ntb_transport_qp *qp; in ntb_transport_create_queue() local
1447 qp = &nt->qps[free_queue]; in ntb_transport_create_queue()
1448 qp->cb_data = data; in ntb_transport_create_queue()
1449 qp->rx_handler = handlers->rx_handler; in ntb_transport_create_queue()
1450 qp->tx_handler = handlers->tx_handler; in ntb_transport_create_queue()
1451 qp->event_handler = handlers->event_handler; in ntb_transport_create_queue()
1454 qp->dma_chan = dma_find_channel(DMA_MEMCPY); in ntb_transport_create_queue()
1455 if (!qp->dma_chan) { in ntb_transport_create_queue()
1465 entry->qp = qp; in ntb_transport_create_queue()
1466 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, in ntb_transport_create_queue()
1467 &qp->rx_free_q); in ntb_transport_create_queue()
1475 entry->qp = qp; in ntb_transport_create_queue()
1476 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, in ntb_transport_create_queue()
1477 &qp->tx_free_q); in ntb_transport_create_queue()
1480 rc = ntb_register_db_callback(qp->ndev, free_queue, qp, in ntb_transport_create_queue()
1485 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); in ntb_transport_create_queue()
1487 return qp; in ntb_transport_create_queue()
1490 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) in ntb_transport_create_queue()
1493 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) in ntb_transport_create_queue()
1495 if (qp->dma_chan) in ntb_transport_create_queue()
1509 void ntb_transport_free_queue(struct ntb_transport_qp *qp) in ntb_transport_free_queue() argument
1514 if (!qp) in ntb_transport_free_queue()
1517 pdev = ntb_query_pdev(qp->ndev); in ntb_transport_free_queue()
1519 if (qp->dma_chan) { in ntb_transport_free_queue()
1520 struct dma_chan *chan = qp->dma_chan; in ntb_transport_free_queue()
1524 qp->dma_chan = NULL; in ntb_transport_free_queue()
1529 dma_sync_wait(chan, qp->last_cookie); in ntb_transport_free_queue()
1534 ntb_unregister_db_callback(qp->ndev, qp->qp_num); in ntb_transport_free_queue()
1536 cancel_delayed_work_sync(&qp->link_work); in ntb_transport_free_queue()
1538 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) in ntb_transport_free_queue()
1541 while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) { in ntb_transport_free_queue()
1546 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) in ntb_transport_free_queue()
1549 set_bit(qp->qp_num, &qp->transport->qp_bitmap); in ntb_transport_free_queue()
1551 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); in ntb_transport_free_queue()
1565 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) in ntb_transport_rx_remove() argument
1570 if (!qp || qp->client_ready == NTB_LINK_UP) in ntb_transport_rx_remove()
1573 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); in ntb_transport_rx_remove()
1580 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); in ntb_transport_rx_remove()
1598 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, in ntb_transport_rx_enqueue() argument
1603 if (!qp) in ntb_transport_rx_enqueue()
1606 entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q); in ntb_transport_rx_enqueue()
1614 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); in ntb_transport_rx_enqueue()
1633 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, in ntb_transport_tx_enqueue() argument
1639 if (!qp || qp->qp_link != NTB_LINK_UP || !len) in ntb_transport_tx_enqueue()
1642 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); in ntb_transport_tx_enqueue()
1644 qp->tx_err_no_buf++; in ntb_transport_tx_enqueue()
1653 rc = ntb_process_tx(qp, entry); in ntb_transport_tx_enqueue()
1655 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, in ntb_transport_tx_enqueue()
1656 &qp->tx_free_q); in ntb_transport_tx_enqueue()
1668 void ntb_transport_link_up(struct ntb_transport_qp *qp) in ntb_transport_link_up() argument
1670 if (!qp) in ntb_transport_link_up()
1673 qp->client_ready = NTB_LINK_UP; in ntb_transport_link_up()
1675 if (qp->transport->transport_link == NTB_LINK_UP) in ntb_transport_link_up()
1676 schedule_delayed_work(&qp->link_work, 0); in ntb_transport_link_up()
1688 void ntb_transport_link_down(struct ntb_transport_qp *qp) in ntb_transport_link_down() argument
1693 if (!qp) in ntb_transport_link_down()
1696 pdev = ntb_query_pdev(qp->ndev); in ntb_transport_link_down()
1697 qp->client_ready = NTB_LINK_DOWN; in ntb_transport_link_down()
1699 rc = ntb_read_local_spad(qp->ndev, QP_LINKS, &val); in ntb_transport_link_down()
1705 rc = ntb_write_remote_spad(qp->ndev, QP_LINKS, in ntb_transport_link_down()
1706 val & ~(1 << qp->qp_num)); in ntb_transport_link_down()
1709 val & ~(1 << qp->qp_num), QP_LINKS); in ntb_transport_link_down()
1711 if (qp->qp_link == NTB_LINK_UP) in ntb_transport_link_down()
1712 ntb_send_link_down(qp); in ntb_transport_link_down()
1714 cancel_delayed_work_sync(&qp->link_work); in ntb_transport_link_down()
1726 bool ntb_transport_link_query(struct ntb_transport_qp *qp) in ntb_transport_link_query() argument
1728 if (!qp) in ntb_transport_link_query()
1731 return qp->qp_link == NTB_LINK_UP; in ntb_transport_link_query()
1743 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) in ntb_transport_qp_num() argument
1745 if (!qp) in ntb_transport_qp_num()
1748 return qp->qp_num; in ntb_transport_qp_num()
1760 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) in ntb_transport_max_size() argument
1764 if (!qp) in ntb_transport_max_size()
1767 if (!qp->dma_chan) in ntb_transport_max_size()
1768 return qp->tx_max_frame - sizeof(struct ntb_payload_header); in ntb_transport_max_size()
1771 max = qp->tx_max_frame - sizeof(struct ntb_payload_header); in ntb_transport_max_size()
1772 max -= max % (1 << qp->dma_chan->device->copy_align); in ntb_transport_max_size()