Lines Matching refs:ndev
108 static int is_ntb_xeon(struct ntb_device *ndev) in is_ntb_xeon() argument
110 switch (ndev->pdev->device) { in is_ntb_xeon()
131 static int is_ntb_atom(struct ntb_device *ndev) in is_ntb_atom() argument
133 switch (ndev->pdev->device) { in is_ntb_atom()
143 static void ntb_set_errata_flags(struct ntb_device *ndev) in ntb_set_errata_flags() argument
145 switch (ndev->pdev->device) { in ntb_set_errata_flags()
162 ndev->wa_flags |= WA_SNB_ERR; in ntb_set_errata_flags()
177 int ntb_register_event_callback(struct ntb_device *ndev, in ntb_register_event_callback() argument
181 if (ndev->event_cb) in ntb_register_event_callback()
184 ndev->event_cb = func; in ntb_register_event_callback()
195 void ntb_unregister_event_callback(struct ntb_device *ndev) in ntb_unregister_event_callback() argument
197 ndev->event_cb = NULL; in ntb_unregister_event_callback()
209 struct ntb_device *ndev = db_cb->ndev; in ntb_irq_work() local
212 mask = readw(ndev->reg_ofs.ldb_mask); in ntb_irq_work()
213 clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask); in ntb_irq_work()
214 writew(mask, ndev->reg_ofs.ldb_mask); in ntb_irq_work()
231 int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, in ntb_register_db_callback() argument
236 if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) { in ntb_register_db_callback()
237 dev_warn(&ndev->pdev->dev, "Invalid Index.\n"); in ntb_register_db_callback()
241 ndev->db_cb[idx].callback = func; in ntb_register_db_callback()
242 ndev->db_cb[idx].data = data; in ntb_register_db_callback()
243 ndev->db_cb[idx].ndev = ndev; in ntb_register_db_callback()
245 tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work, in ntb_register_db_callback()
246 (unsigned long) &ndev->db_cb[idx]); in ntb_register_db_callback()
249 mask = readw(ndev->reg_ofs.ldb_mask); in ntb_register_db_callback()
250 clear_bit(idx * ndev->bits_per_vector, &mask); in ntb_register_db_callback()
251 writew(mask, ndev->reg_ofs.ldb_mask); in ntb_register_db_callback()
264 void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx) in ntb_unregister_db_callback() argument
268 if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback) in ntb_unregister_db_callback()
271 mask = readw(ndev->reg_ofs.ldb_mask); in ntb_unregister_db_callback()
272 set_bit(idx * ndev->bits_per_vector, &mask); in ntb_unregister_db_callback()
273 writew(mask, ndev->reg_ofs.ldb_mask); in ntb_unregister_db_callback()
275 tasklet_disable(&ndev->db_cb[idx].irq_work); in ntb_unregister_db_callback()
277 ndev->db_cb[idx].callback = NULL; in ntb_unregister_db_callback()
291 struct ntb_device *ndev = pci_get_drvdata(pdev); in ntb_find_transport() local
292 return ndev->ntb_transport; in ntb_find_transport()
306 struct ntb_device *ndev = pci_get_drvdata(pdev); in ntb_register_transport() local
308 if (ndev->ntb_transport) in ntb_register_transport()
311 ndev->ntb_transport = transport; in ntb_register_transport()
312 return ndev; in ntb_register_transport()
322 void ntb_unregister_transport(struct ntb_device *ndev) in ntb_unregister_transport() argument
326 if (!ndev->ntb_transport) in ntb_unregister_transport()
329 for (i = 0; i < ndev->max_cbs; i++) in ntb_unregister_transport()
330 ntb_unregister_db_callback(ndev, i); in ntb_unregister_transport()
332 ntb_unregister_event_callback(ndev); in ntb_unregister_transport()
333 ndev->ntb_transport = NULL; in ntb_unregister_transport()
348 int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val) in ntb_write_local_spad() argument
350 if (idx >= ndev->limits.max_spads) in ntb_write_local_spad()
353 dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n", in ntb_write_local_spad()
355 writel(val, ndev->reg_ofs.spad_read + idx * 4); in ntb_write_local_spad()
372 int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) in ntb_read_local_spad() argument
374 if (idx >= ndev->limits.max_spads) in ntb_read_local_spad()
377 *val = readl(ndev->reg_ofs.spad_write + idx * 4); in ntb_read_local_spad()
378 dev_dbg(&ndev->pdev->dev, in ntb_read_local_spad()
397 int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val) in ntb_write_remote_spad() argument
399 if (idx >= ndev->limits.max_spads) in ntb_write_remote_spad()
402 dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n", in ntb_write_remote_spad()
404 writel(val, ndev->reg_ofs.spad_write + idx * 4); in ntb_write_remote_spad()
421 int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) in ntb_read_remote_spad() argument
423 if (idx >= ndev->limits.max_spads) in ntb_read_remote_spad()
426 *val = readl(ndev->reg_ofs.spad_read + idx * 4); in ntb_read_remote_spad()
427 dev_dbg(&ndev->pdev->dev, in ntb_read_remote_spad()
442 resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw) in ntb_get_mw_base() argument
444 if (mw >= ntb_max_mw(ndev)) in ntb_get_mw_base()
447 return pci_resource_start(ndev->pdev, MW_TO_BAR(mw)); in ntb_get_mw_base()
460 void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw) in ntb_get_mw_vbase() argument
462 if (mw >= ntb_max_mw(ndev)) in ntb_get_mw_vbase()
465 return ndev->mw[mw].vbase; in ntb_get_mw_vbase()
477 u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw) in ntb_get_mw_size() argument
479 if (mw >= ntb_max_mw(ndev)) in ntb_get_mw_size()
482 return ndev->mw[mw].bar_sz; in ntb_get_mw_size()
495 void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr) in ntb_set_mw_addr() argument
497 if (mw >= ntb_max_mw(ndev)) in ntb_set_mw_addr()
500 dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr, in ntb_set_mw_addr()
503 ndev->mw[mw].phys_addr = addr; in ntb_set_mw_addr()
507 writeq(addr, ndev->reg_ofs.bar2_xlat); in ntb_set_mw_addr()
510 if (ndev->split_bar) in ntb_set_mw_addr()
511 writel(addr, ndev->reg_ofs.bar4_xlat); in ntb_set_mw_addr()
513 writeq(addr, ndev->reg_ofs.bar4_xlat); in ntb_set_mw_addr()
516 writel(addr, ndev->reg_ofs.bar5_xlat); in ntb_set_mw_addr()
531 void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db) in ntb_ring_doorbell() argument
533 dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db); in ntb_ring_doorbell()
535 if (ndev->hw_type == BWD_HW) in ntb_ring_doorbell()
536 writeq((u64) 1 << db, ndev->reg_ofs.rdb); in ntb_ring_doorbell()
538 writew(((1 << ndev->bits_per_vector) - 1) << in ntb_ring_doorbell()
539 (db * ndev->bits_per_vector), ndev->reg_ofs.rdb); in ntb_ring_doorbell()
542 static void bwd_recover_link(struct ntb_device *ndev) in bwd_recover_link() argument
547 writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6); in bwd_recover_link()
548 writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4); in bwd_recover_link()
549 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4); in bwd_recover_link()
550 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6); in bwd_recover_link()
556 status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET); in bwd_recover_link()
557 dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status); in bwd_recover_link()
559 writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET); in bwd_recover_link()
562 status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); in bwd_recover_link()
563 dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status); in bwd_recover_link()
565 writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); in bwd_recover_link()
568 status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET); in bwd_recover_link()
569 dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status); in bwd_recover_link()
571 writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET); in bwd_recover_link()
573 status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); in bwd_recover_link()
574 dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status); in bwd_recover_link()
576 writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); in bwd_recover_link()
579 status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); in bwd_recover_link()
580 dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status); in bwd_recover_link()
582 writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); in bwd_recover_link()
585 static void ntb_link_event(struct ntb_device *ndev, int link_state) in ntb_link_event() argument
589 if (ndev->link_status == link_state) in ntb_link_event()
595 dev_info(&ndev->pdev->dev, "Link Up\n"); in ntb_link_event()
596 ndev->link_status = NTB_LINK_UP; in ntb_link_event()
599 if (is_ntb_atom(ndev) || in ntb_link_event()
600 ndev->conn_type == NTB_CONN_TRANSPARENT) in ntb_link_event()
601 status = readw(ndev->reg_ofs.lnk_stat); in ntb_link_event()
603 int rc = pci_read_config_word(ndev->pdev, in ntb_link_event()
610 ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; in ntb_link_event()
611 ndev->link_speed = (status & NTB_LINK_SPEED_MASK); in ntb_link_event()
612 dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n", in ntb_link_event()
613 ndev->link_width, ndev->link_speed); in ntb_link_event()
615 dev_info(&ndev->pdev->dev, "Link Down\n"); in ntb_link_event()
616 ndev->link_status = NTB_LINK_DOWN; in ntb_link_event()
622 if (ndev->event_cb) in ntb_link_event()
623 ndev->event_cb(ndev->ntb_transport, event); in ntb_link_event()
626 static int ntb_link_status(struct ntb_device *ndev) in ntb_link_status() argument
630 if (is_ntb_atom(ndev)) { in ntb_link_status()
633 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); in ntb_link_status()
642 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, in ntb_link_status()
653 ntb_link_event(ndev, link_state); in ntb_link_status()
660 struct ntb_device *ndev = container_of(work, struct ntb_device, in bwd_link_recovery() local
664 bwd_recover_link(ndev); in bwd_link_recovery()
672 status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); in bwd_link_recovery()
676 status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); in bwd_link_recovery()
680 status32 = readl(ndev->reg_ofs.lnk_cntl); in bwd_link_recovery()
685 status16 = readw(ndev->reg_ofs.lnk_stat); in bwd_link_recovery()
688 if (ndev->link_width != width || ndev->link_speed != speed) in bwd_link_recovery()
692 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); in bwd_link_recovery()
696 schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT); in bwd_link_recovery()
702 struct ntb_device *ndev = container_of(work, struct ntb_device, in bwd_link_poll() local
709 if (ts > ndev->last_ts + NTB_HB_TIMEOUT) { in bwd_link_poll()
710 int rc = ntb_link_status(ndev); in bwd_link_poll()
712 dev_err(&ndev->pdev->dev, in bwd_link_poll()
716 if (ndev->link_status == NTB_LINK_DOWN) { in bwd_link_poll()
717 u32 status32 = readl(ndev->reg_base + in bwd_link_poll()
720 schedule_delayed_work(&ndev->lr_timer, 0); in bwd_link_poll()
726 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); in bwd_link_poll()
729 static int ntb_xeon_setup(struct ntb_device *ndev) in ntb_xeon_setup() argument
731 switch (ndev->conn_type) { in ntb_xeon_setup()
733 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; in ntb_xeon_setup()
734 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; in ntb_xeon_setup()
735 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; in ntb_xeon_setup()
736 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; in ntb_xeon_setup()
737 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; in ntb_xeon_setup()
738 if (ndev->split_bar) in ntb_xeon_setup()
739 ndev->reg_ofs.bar5_xlat = in ntb_xeon_setup()
740 ndev->reg_base + SNB_SBAR5XLAT_OFFSET; in ntb_xeon_setup()
741 ndev->limits.max_spads = SNB_MAX_B2B_SPADS; in ntb_xeon_setup()
749 if (ndev->wa_flags & WA_SNB_ERR) { in ntb_xeon_setup()
750 if (!ndev->mw[ndev->limits.max_mw - 1].bar_sz) in ntb_xeon_setup()
753 ndev->limits.max_db_bits = SNB_MAX_DB_BITS; in ntb_xeon_setup()
754 ndev->reg_ofs.spad_write = in ntb_xeon_setup()
755 ndev->mw[ndev->limits.max_mw - 1].vbase + in ntb_xeon_setup()
757 ndev->reg_ofs.rdb = in ntb_xeon_setup()
758 ndev->mw[ndev->limits.max_mw - 1].vbase + in ntb_xeon_setup()
764 writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base + in ntb_xeon_setup()
773 ndev->limits.max_mw = SNB_ERRATA_MAX_MW; in ntb_xeon_setup()
780 ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1; in ntb_xeon_setup()
781 ndev->reg_ofs.spad_write = ndev->reg_base + in ntb_xeon_setup()
783 ndev->reg_ofs.rdb = ndev->reg_base + in ntb_xeon_setup()
790 writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET); in ntb_xeon_setup()
797 if (ndev->split_bar) in ntb_xeon_setup()
798 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; in ntb_xeon_setup()
800 ndev->limits.max_mw = SNB_MAX_MW; in ntb_xeon_setup()
807 if (ndev->dev_type == NTB_DEV_USD) { in ntb_xeon_setup()
808 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + in ntb_xeon_setup()
810 if (ndev->wa_flags & WA_SNB_ERR) in ntb_xeon_setup()
811 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + in ntb_xeon_setup()
814 if (ndev->split_bar) { in ntb_xeon_setup()
816 ndev->reg_base + in ntb_xeon_setup()
819 ndev->reg_base + in ntb_xeon_setup()
823 ndev->reg_base + in ntb_xeon_setup()
830 ndev->reg_base + SNB_B2B_XLAT_OFFSETL); in ntb_xeon_setup()
832 ndev->reg_base + SNB_B2B_XLAT_OFFSETU); in ntb_xeon_setup()
835 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + in ntb_xeon_setup()
837 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + in ntb_xeon_setup()
839 if (ndev->split_bar) { in ntb_xeon_setup()
840 writel(SNB_MBAR4_USD_ADDR, ndev->reg_base + in ntb_xeon_setup()
842 writel(SNB_MBAR5_USD_ADDR, ndev->reg_base + in ntb_xeon_setup()
845 writeq(SNB_MBAR4_USD_ADDR, ndev->reg_base + in ntb_xeon_setup()
848 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + in ntb_xeon_setup()
850 if (ndev->wa_flags & WA_SNB_ERR) in ntb_xeon_setup()
851 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + in ntb_xeon_setup()
854 if (ndev->split_bar) { in ntb_xeon_setup()
856 ndev->reg_base + in ntb_xeon_setup()
859 ndev->reg_base + in ntb_xeon_setup()
863 ndev->reg_base + in ntb_xeon_setup()
871 ndev->reg_base + SNB_B2B_XLAT_OFFSETL); in ntb_xeon_setup()
873 ndev->reg_base + SNB_B2B_XLAT_OFFSETU); in ntb_xeon_setup()
875 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + in ntb_xeon_setup()
877 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + in ntb_xeon_setup()
879 if (ndev->split_bar) { in ntb_xeon_setup()
880 writel(SNB_MBAR4_DSD_ADDR, ndev->reg_base + in ntb_xeon_setup()
882 writel(SNB_MBAR5_DSD_ADDR, ndev->reg_base + in ntb_xeon_setup()
885 writeq(SNB_MBAR4_DSD_ADDR, ndev->reg_base + in ntb_xeon_setup()
891 if (ndev->wa_flags & WA_SNB_ERR) { in ntb_xeon_setup()
892 dev_err(&ndev->pdev->dev, in ntb_xeon_setup()
901 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; in ntb_xeon_setup()
902 ndev->limits.max_db_bits = SNB_MAX_DB_BITS; in ntb_xeon_setup()
906 ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET; in ntb_xeon_setup()
907 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; in ntb_xeon_setup()
908 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; in ntb_xeon_setup()
912 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET + in ntb_xeon_setup()
913 ndev->limits.max_spads * 4; in ntb_xeon_setup()
914 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; in ntb_xeon_setup()
915 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; in ntb_xeon_setup()
916 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; in ntb_xeon_setup()
917 if (ndev->split_bar) { in ntb_xeon_setup()
918 ndev->reg_ofs.bar5_xlat = in ntb_xeon_setup()
919 ndev->reg_base + SNB_SBAR5XLAT_OFFSET; in ntb_xeon_setup()
920 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; in ntb_xeon_setup()
922 ndev->limits.max_mw = SNB_MAX_MW; in ntb_xeon_setup()
925 if (ndev->wa_flags & WA_SNB_ERR) { in ntb_xeon_setup()
926 dev_err(&ndev->pdev->dev, in ntb_xeon_setup()
935 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; in ntb_xeon_setup()
936 ndev->limits.max_db_bits = SNB_MAX_DB_BITS; in ntb_xeon_setup()
937 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET; in ntb_xeon_setup()
938 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET; in ntb_xeon_setup()
939 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET; in ntb_xeon_setup()
940 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET; in ntb_xeon_setup()
944 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET + in ntb_xeon_setup()
945 ndev->limits.max_spads * 4; in ntb_xeon_setup()
946 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET; in ntb_xeon_setup()
947 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET; in ntb_xeon_setup()
949 if (ndev->split_bar) { in ntb_xeon_setup()
950 ndev->reg_ofs.bar5_xlat = in ntb_xeon_setup()
951 ndev->reg_base + SNB_PBAR5XLAT_OFFSET; in ntb_xeon_setup()
952 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; in ntb_xeon_setup()
954 ndev->limits.max_mw = SNB_MAX_MW; in ntb_xeon_setup()
964 ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET; in ntb_xeon_setup()
965 ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET; in ntb_xeon_setup()
966 ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET; in ntb_xeon_setup()
968 ndev->limits.msix_cnt = SNB_MSIX_CNT; in ntb_xeon_setup()
969 ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; in ntb_xeon_setup()
974 static int ntb_bwd_setup(struct ntb_device *ndev) in ntb_bwd_setup() argument
979 ndev->hw_type = BWD_HW; in ntb_bwd_setup()
981 rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val); in ntb_bwd_setup()
987 ndev->conn_type = NTB_CONN_B2B; in ntb_bwd_setup()
991 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); in ntb_bwd_setup()
996 ndev->dev_type = NTB_DEV_DSD; in ntb_bwd_setup()
998 ndev->dev_type = NTB_DEV_USD; in ntb_bwd_setup()
1001 rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET, in ntb_bwd_setup()
1006 ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET; in ntb_bwd_setup()
1007 ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET; in ntb_bwd_setup()
1008 ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET; in ntb_bwd_setup()
1009 ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET; in ntb_bwd_setup()
1010 ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET; in ntb_bwd_setup()
1011 ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET; in ntb_bwd_setup()
1012 ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET; in ntb_bwd_setup()
1013 ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET; in ntb_bwd_setup()
1014 ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET; in ntb_bwd_setup()
1015 ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET; in ntb_bwd_setup()
1016 ndev->limits.max_mw = BWD_MAX_MW; in ntb_bwd_setup()
1017 ndev->limits.max_spads = BWD_MAX_SPADS; in ntb_bwd_setup()
1018 ndev->limits.max_db_bits = BWD_MAX_DB_BITS; in ntb_bwd_setup()
1019 ndev->limits.msix_cnt = BWD_MSIX_CNT; in ntb_bwd_setup()
1020 ndev->bits_per_vector = BWD_DB_BITS_PER_VEC; in ntb_bwd_setup()
1023 INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll); in ntb_bwd_setup()
1024 INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery); in ntb_bwd_setup()
1025 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); in ntb_bwd_setup()
1030 static int ntb_device_setup(struct ntb_device *ndev) in ntb_device_setup() argument
1034 if (is_ntb_xeon(ndev)) in ntb_device_setup()
1035 rc = ntb_xeon_setup(ndev); in ntb_device_setup()
1036 else if (is_ntb_atom(ndev)) in ntb_device_setup()
1037 rc = ntb_bwd_setup(ndev); in ntb_device_setup()
1044 if (ndev->conn_type == NTB_CONN_B2B) in ntb_device_setup()
1047 ndev->reg_ofs.spci_cmd); in ntb_device_setup()
1052 static void ntb_device_free(struct ntb_device *ndev) in ntb_device_free() argument
1054 if (is_ntb_atom(ndev)) { in ntb_device_free()
1055 cancel_delayed_work_sync(&ndev->hb_timer); in ntb_device_free()
1056 cancel_delayed_work_sync(&ndev->lr_timer); in ntb_device_free()
1063 struct ntb_device *ndev = db_cb->ndev; in bwd_callback_msix_irq() local
1066 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, in bwd_callback_msix_irq()
1069 mask = readw(ndev->reg_ofs.ldb_mask); in bwd_callback_msix_irq()
1070 set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); in bwd_callback_msix_irq()
1071 writew(mask, ndev->reg_ofs.ldb_mask); in bwd_callback_msix_irq()
1078 ndev->last_ts = jiffies; in bwd_callback_msix_irq()
1080 writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb); in bwd_callback_msix_irq()
1088 struct ntb_device *ndev = db_cb->ndev; in xeon_callback_msix_irq() local
1091 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, in xeon_callback_msix_irq()
1094 mask = readw(ndev->reg_ofs.ldb_mask); in xeon_callback_msix_irq()
1095 set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); in xeon_callback_msix_irq()
1096 writew(mask, ndev->reg_ofs.ldb_mask); in xeon_callback_msix_irq()
1105 writew(((1 << ndev->bits_per_vector) - 1) << in xeon_callback_msix_irq()
1106 (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb); in xeon_callback_msix_irq()
1114 struct ntb_device *ndev = dev; in xeon_event_msix_irq() local
1117 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq); in xeon_event_msix_irq()
1119 rc = ntb_link_status(ndev); in xeon_event_msix_irq()
1121 dev_err(&ndev->pdev->dev, "Error determining link status\n"); in xeon_event_msix_irq()
1124 writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb); in xeon_event_msix_irq()
1131 struct ntb_device *ndev = dev; in ntb_interrupt() local
1134 if (is_ntb_atom(ndev)) { in ntb_interrupt()
1135 u64 ldb = readq(ndev->reg_ofs.ldb); in ntb_interrupt()
1137 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb); in ntb_interrupt()
1142 bwd_callback_msix_irq(irq, &ndev->db_cb[i]); in ntb_interrupt()
1145 u16 ldb = readw(ndev->reg_ofs.ldb); in ntb_interrupt()
1147 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb); in ntb_interrupt()
1157 xeon_callback_msix_irq(irq, &ndev->db_cb[i]); in ntb_interrupt()
1164 static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries) in ntb_setup_snb_msix() argument
1166 struct pci_dev *pdev = ndev->pdev; in ntb_setup_snb_msix()
1170 if (msix_entries < ndev->limits.msix_cnt) in ntb_setup_snb_msix()
1173 rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries); in ntb_setup_snb_msix()
1178 msix = &ndev->msix_entries[i]; in ntb_setup_snb_msix()
1184 "ntb-event-msix", ndev); in ntb_setup_snb_msix()
1191 &ndev->db_cb[i]); in ntb_setup_snb_msix()
1197 ndev->num_msix = msix_entries; in ntb_setup_snb_msix()
1198 ndev->max_cbs = msix_entries - 1; in ntb_setup_snb_msix()
1205 msix = &ndev->msix_entries[i]; in ntb_setup_snb_msix()
1206 free_irq(msix->vector, &ndev->db_cb[i]); in ntb_setup_snb_msix()
1210 ndev->num_msix = 0; in ntb_setup_snb_msix()
1215 static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries) in ntb_setup_bwd_msix() argument
1217 struct pci_dev *pdev = ndev->pdev; in ntb_setup_bwd_msix()
1221 msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries, in ntb_setup_bwd_msix()
1227 msix = &ndev->msix_entries[i]; in ntb_setup_bwd_msix()
1231 "ntb-callback-msix", &ndev->db_cb[i]); in ntb_setup_bwd_msix()
1236 ndev->num_msix = msix_entries; in ntb_setup_bwd_msix()
1237 ndev->max_cbs = msix_entries; in ntb_setup_bwd_msix()
1243 free_irq(msix->vector, &ndev->db_cb[i]); in ntb_setup_bwd_msix()
1246 ndev->num_msix = 0; in ntb_setup_bwd_msix()
1251 static int ntb_setup_msix(struct ntb_device *ndev) in ntb_setup_msix() argument
1253 struct pci_dev *pdev = ndev->pdev; in ntb_setup_msix()
1261 } else if (msix_entries > ndev->limits.msix_cnt) { in ntb_setup_msix()
1266 ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries, in ntb_setup_msix()
1268 if (!ndev->msix_entries) { in ntb_setup_msix()
1274 ndev->msix_entries[i].entry = i; in ntb_setup_msix()
1276 if (is_ntb_atom(ndev)) in ntb_setup_msix()
1277 rc = ntb_setup_bwd_msix(ndev, msix_entries); in ntb_setup_msix()
1279 rc = ntb_setup_snb_msix(ndev, msix_entries); in ntb_setup_msix()
1286 kfree(ndev->msix_entries); in ntb_setup_msix()
1292 static int ntb_setup_msi(struct ntb_device *ndev) in ntb_setup_msi() argument
1294 struct pci_dev *pdev = ndev->pdev; in ntb_setup_msi()
1301 rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev); in ntb_setup_msi()
1311 static int ntb_setup_intx(struct ntb_device *ndev) in ntb_setup_intx() argument
1313 struct pci_dev *pdev = ndev->pdev; in ntb_setup_intx()
1322 ndev); in ntb_setup_intx()
1329 static int ntb_setup_interrupts(struct ntb_device *ndev) in ntb_setup_interrupts() argument
1336 if (is_ntb_atom(ndev)) in ntb_setup_interrupts()
1337 writeq(~0, ndev->reg_ofs.ldb_mask); in ntb_setup_interrupts()
1340 writew(~var, ndev->reg_ofs.ldb_mask); in ntb_setup_interrupts()
1343 rc = ntb_setup_msix(ndev); in ntb_setup_interrupts()
1347 ndev->bits_per_vector = 1; in ntb_setup_interrupts()
1348 ndev->max_cbs = ndev->limits.max_db_bits; in ntb_setup_interrupts()
1350 rc = ntb_setup_msi(ndev); in ntb_setup_interrupts()
1354 rc = ntb_setup_intx(ndev); in ntb_setup_interrupts()
1356 dev_err(&ndev->pdev->dev, "no usable interrupts\n"); in ntb_setup_interrupts()
1364 static void ntb_free_interrupts(struct ntb_device *ndev) in ntb_free_interrupts() argument
1366 struct pci_dev *pdev = ndev->pdev; in ntb_free_interrupts()
1369 if (is_ntb_atom(ndev)) in ntb_free_interrupts()
1370 writeq(~0, ndev->reg_ofs.ldb_mask); in ntb_free_interrupts()
1372 writew(~0, ndev->reg_ofs.ldb_mask); in ntb_free_interrupts()
1374 if (ndev->num_msix) { in ntb_free_interrupts()
1378 for (i = 0; i < ndev->num_msix; i++) { in ntb_free_interrupts()
1379 msix = &ndev->msix_entries[i]; in ntb_free_interrupts()
1380 if (is_ntb_xeon(ndev) && i == ndev->num_msix - 1) in ntb_free_interrupts()
1381 free_irq(msix->vector, ndev); in ntb_free_interrupts()
1383 free_irq(msix->vector, &ndev->db_cb[i]); in ntb_free_interrupts()
1386 kfree(ndev->msix_entries); in ntb_free_interrupts()
1388 free_irq(pdev->irq, ndev); in ntb_free_interrupts()
1395 static int ntb_create_callbacks(struct ntb_device *ndev) in ntb_create_callbacks() argument
1404 ndev->db_cb = kcalloc(ndev->limits.max_db_bits, in ntb_create_callbacks()
1407 if (!ndev->db_cb) in ntb_create_callbacks()
1410 for (i = 0; i < ndev->limits.max_db_bits; i++) { in ntb_create_callbacks()
1411 ndev->db_cb[i].db_num = i; in ntb_create_callbacks()
1412 ndev->db_cb[i].ndev = ndev; in ntb_create_callbacks()
1418 static void ntb_free_callbacks(struct ntb_device *ndev) in ntb_free_callbacks() argument
1422 for (i = 0; i < ndev->limits.max_db_bits; i++) in ntb_free_callbacks()
1423 ntb_unregister_db_callback(ndev, i); in ntb_free_callbacks()
1425 kfree(ndev->db_cb); in ntb_free_callbacks()
1431 struct ntb_device *ndev; in ntb_debugfs_read() local
1441 ndev = filp->private_data; in ntb_debugfs_read()
1447 ndev->conn_type == NTB_CONN_TRANSPARENT ? in ntb_debugfs_read()
1448 "Transparent" : (ndev->conn_type == NTB_CONN_B2B) ? in ntb_debugfs_read()
1452 ndev->dev_type == NTB_DEV_USD ? in ntb_debugfs_read()
1456 ntb_max_cbs(ndev)); in ntb_debugfs_read()
1459 ntb_hw_link_status(ndev) ? "Up" : "Down"); in ntb_debugfs_read()
1460 if (ntb_hw_link_status(ndev)) { in ntb_debugfs_read()
1463 ndev->link_speed); in ntb_debugfs_read()
1466 ndev->link_width); in ntb_debugfs_read()
1469 if (is_ntb_xeon(ndev)) { in ntb_debugfs_read()
1478 readw(ndev->reg_base + in ntb_debugfs_read()
1484 rc = pci_read_config_word(ndev->pdev, SNB_DEVSTS_OFFSET, in ntb_debugfs_read()
1490 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, in ntb_debugfs_read()
1496 rc = pci_read_config_dword(ndev->pdev, SNB_UNCERRSTS_OFFSET, in ntb_debugfs_read()
1502 rc = pci_read_config_dword(ndev->pdev, SNB_CORERRSTS_OFFSET, in ntb_debugfs_read()
1523 static void ntb_setup_debugfs(struct ntb_device *ndev) in ntb_setup_debugfs() argument
1531 ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev), in ntb_setup_debugfs()
1533 if (ndev->debugfs_dir) in ntb_setup_debugfs()
1534 ndev->debugfs_info = debugfs_create_file("info", S_IRUSR, in ntb_setup_debugfs()
1535 ndev->debugfs_dir, in ntb_setup_debugfs()
1536 ndev, in ntb_setup_debugfs()
1540 static void ntb_free_debugfs(struct ntb_device *ndev) in ntb_free_debugfs() argument
1542 debugfs_remove_recursive(ndev->debugfs_dir); in ntb_free_debugfs()
1550 static void ntb_hw_link_up(struct ntb_device *ndev) in ntb_hw_link_up() argument
1552 if (ndev->conn_type == NTB_CONN_TRANSPARENT) in ntb_hw_link_up()
1553 ntb_link_event(ndev, NTB_LINK_UP); in ntb_hw_link_up()
1558 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); in ntb_hw_link_up()
1562 if (ndev->split_bar) in ntb_hw_link_up()
1566 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); in ntb_hw_link_up()
1570 static void ntb_hw_link_down(struct ntb_device *ndev) in ntb_hw_link_down() argument
1574 if (ndev->conn_type == NTB_CONN_TRANSPARENT) { in ntb_hw_link_down()
1575 ntb_link_event(ndev, NTB_LINK_DOWN); in ntb_hw_link_down()
1580 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); in ntb_hw_link_down()
1583 if (ndev->split_bar) in ntb_hw_link_down()
1587 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); in ntb_hw_link_down()
1590 static void ntb_max_mw_detect(struct ntb_device *ndev) in ntb_max_mw_detect() argument
1592 if (ndev->split_bar) in ntb_max_mw_detect()
1593 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; in ntb_max_mw_detect()
1595 ndev->limits.max_mw = SNB_MAX_MW; in ntb_max_mw_detect()
1598 static int ntb_xeon_detect(struct ntb_device *ndev) in ntb_xeon_detect() argument
1604 ndev->hw_type = SNB_HW; in ntb_xeon_detect()
1606 rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &ppd); in ntb_xeon_detect()
1611 ndev->dev_type = NTB_DEV_USD; in ntb_xeon_detect()
1613 ndev->dev_type = NTB_DEV_DSD; in ntb_xeon_detect()
1615 ndev->split_bar = (ppd & SNB_PPD_SPLIT_BAR) ? 1 : 0; in ntb_xeon_detect()
1619 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); in ntb_xeon_detect()
1620 ndev->conn_type = NTB_CONN_B2B; in ntb_xeon_detect()
1623 dev_info(&ndev->pdev->dev, "Conn Type = RP\n"); in ntb_xeon_detect()
1624 ndev->conn_type = NTB_CONN_RP; in ntb_xeon_detect()
1627 dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n"); in ntb_xeon_detect()
1628 ndev->conn_type = NTB_CONN_TRANSPARENT; in ntb_xeon_detect()
1634 ndev->dev_type = NTB_DEV_USD; in ntb_xeon_detect()
1641 bars_mask = pci_select_bars(ndev->pdev, IORESOURCE_MEM); in ntb_xeon_detect()
1644 ndev->split_bar = 1; in ntb_xeon_detect()
1648 dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", ppd); in ntb_xeon_detect()
1652 ntb_max_mw_detect(ndev); in ntb_xeon_detect()
1657 static int ntb_atom_detect(struct ntb_device *ndev) in ntb_atom_detect() argument
1662 ndev->hw_type = BWD_HW; in ntb_atom_detect()
1663 ndev->limits.max_mw = BWD_MAX_MW; in ntb_atom_detect()
1665 rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &ppd); in ntb_atom_detect()
1671 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); in ntb_atom_detect()
1672 ndev->conn_type = NTB_CONN_B2B; in ntb_atom_detect()
1676 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); in ntb_atom_detect()
1681 ndev->dev_type = NTB_DEV_DSD; in ntb_atom_detect()
1683 ndev->dev_type = NTB_DEV_USD; in ntb_atom_detect()
1688 static int ntb_device_detect(struct ntb_device *ndev) in ntb_device_detect() argument
1692 if (is_ntb_xeon(ndev)) in ntb_device_detect()
1693 rc = ntb_xeon_detect(ndev); in ntb_device_detect()
1694 else if (is_ntb_atom(ndev)) in ntb_device_detect()
1695 rc = ntb_atom_detect(ndev); in ntb_device_detect()
1699 dev_info(&ndev->pdev->dev, "Device Type = %s\n", in ntb_device_detect()
1700 ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP"); in ntb_device_detect()
1707 struct ntb_device *ndev; in ntb_pci_probe() local
1710 ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL); in ntb_pci_probe()
1711 if (!ndev) in ntb_pci_probe()
1714 ndev->pdev = pdev; in ntb_pci_probe()
1716 ntb_set_errata_flags(ndev); in ntb_pci_probe()
1718 ndev->link_status = NTB_LINK_DOWN; in ntb_pci_probe()
1719 pci_set_drvdata(pdev, ndev); in ntb_pci_probe()
1720 ntb_setup_debugfs(ndev); in ntb_pci_probe()
1726 pci_set_master(ndev->pdev); in ntb_pci_probe()
1728 rc = ntb_device_detect(ndev); in ntb_pci_probe()
1732 ndev->mw = kcalloc(ndev->limits.max_mw, sizeof(struct ntb_mw), in ntb_pci_probe()
1734 if (!ndev->mw) { in ntb_pci_probe()
1739 if (ndev->split_bar) in ntb_pci_probe()
1749 ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO); in ntb_pci_probe()
1750 if (!ndev->reg_base) { in ntb_pci_probe()
1756 for (i = 0; i < ndev->limits.max_mw; i++) { in ntb_pci_probe()
1757 ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i)); in ntb_pci_probe()
1763 if ((ndev->wa_flags & WA_SNB_ERR) && in ntb_pci_probe()
1764 (i == (ndev->limits.max_mw - 1))) { in ntb_pci_probe()
1765 ndev->mw[i].vbase = in ntb_pci_probe()
1768 ndev->mw[i].bar_sz); in ntb_pci_probe()
1770 ndev->mw[i].vbase = in ntb_pci_probe()
1773 ndev->mw[i].bar_sz); in ntb_pci_probe()
1777 (unsigned long long) ndev->mw[i].bar_sz); in ntb_pci_probe()
1778 if (!ndev->mw[i].vbase) { in ntb_pci_probe()
1804 rc = ntb_device_setup(ndev); in ntb_pci_probe()
1808 rc = ntb_create_callbacks(ndev); in ntb_pci_probe()
1812 rc = ntb_setup_interrupts(ndev); in ntb_pci_probe()
1819 for (i = 0; i < ndev->limits.max_spads; i++) { in ntb_pci_probe()
1820 ntb_write_local_spad(ndev, i, 0); in ntb_pci_probe()
1821 ntb_write_remote_spad(ndev, i, 0); in ntb_pci_probe()
1828 ntb_hw_link_up(ndev); in ntb_pci_probe()
1833 ntb_free_interrupts(ndev); in ntb_pci_probe()
1835 ntb_free_callbacks(ndev); in ntb_pci_probe()
1837 ntb_device_free(ndev); in ntb_pci_probe()
1840 iounmap(ndev->mw[i].vbase); in ntb_pci_probe()
1841 iounmap(ndev->reg_base); in ntb_pci_probe()
1843 if (ndev->split_bar) in ntb_pci_probe()
1848 kfree(ndev->mw); in ntb_pci_probe()
1852 ntb_free_debugfs(ndev); in ntb_pci_probe()
1853 kfree(ndev); in ntb_pci_probe()
1861 struct ntb_device *ndev = pci_get_drvdata(pdev); in ntb_pci_remove() local
1864 ntb_hw_link_down(ndev); in ntb_pci_remove()
1866 ntb_transport_free(ndev->ntb_transport); in ntb_pci_remove()
1868 ntb_free_interrupts(ndev); in ntb_pci_remove()
1869 ntb_free_callbacks(ndev); in ntb_pci_remove()
1870 ntb_device_free(ndev); in ntb_pci_remove()
1873 if (ndev->hw_type == SNB_HW) in ntb_pci_remove()
1874 ntb_max_mw_detect(ndev); in ntb_pci_remove()
1876 for (i = 0; i < ndev->limits.max_mw; i++) in ntb_pci_remove()
1877 iounmap(ndev->mw[i].vbase); in ntb_pci_remove()
1879 kfree(ndev->mw); in ntb_pci_remove()
1880 iounmap(ndev->reg_base); in ntb_pci_remove()
1881 if (ndev->split_bar) in ntb_pci_remove()
1886 ntb_free_debugfs(ndev); in ntb_pci_remove()
1887 kfree(ndev); in ntb_pci_remove()