41 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
42 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
58 	struct tsi721_device *priv = mport->priv;  in tsi721_lcread()  local
63 	*data = ioread32(priv->regs + offset);  in tsi721_lcread()
82 	struct tsi721_device *priv = mport->priv;  in tsi721_lcwrite()  local
87 	iowrite32(data, priv->regs + offset);  in tsi721_lcwrite()
107 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,  in tsi721_maint_dma()  argument
111 	void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);  in tsi721_maint_dma()
120 	bd_ptr = priv->mdma.bd_base;  in tsi721_maint_dma()
146 			dev_dbg(&priv->pdev->dev,  in tsi721_maint_dma()
148 				__func__, priv->mdma.ch_id, ch_stat);  in tsi721_maint_dma()
160 		dev_dbg(&priv->pdev->dev, "%s : DMA ABORT ch_stat=%x\n",  in tsi721_maint_dma()
162 		dev_dbg(&priv->pdev->dev, "OP=%d : destid=%x hc=%x off=%x\n",  in tsi721_maint_dma()
207 	struct tsi721_device *priv = mport->priv;  in tsi721_cread_dma()  local
209 	return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,  in tsi721_cread_dma()
230 	struct tsi721_device *priv = mport->priv;  in tsi721_cwrite_dma()  local
233 	return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,  in tsi721_cwrite_dma()
248 	struct tsi721_device *priv = mport->priv;  in tsi721_pw_handler()  local
253 	pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);  in tsi721_pw_handler()
256 		pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));  in tsi721_pw_handler()
257 		pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));  in tsi721_pw_handler()
258 		pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));  in tsi721_pw_handler()
259 		pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));  in tsi721_pw_handler()
264 		spin_lock(&priv->pw_fifo_lock);  in tsi721_pw_handler()
265 		if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)  in tsi721_pw_handler()
266 			kfifo_in(&priv->pw_fifo, pw_buf,  in tsi721_pw_handler()
269 			priv->pw_discard_count++;  in tsi721_pw_handler()
270 		spin_unlock(&priv->pw_fifo_lock);  in tsi721_pw_handler()
275 		  priv->regs + TSI721_RIO_PW_RX_STAT);  in tsi721_pw_handler()
277 	schedule_work(&priv->pw_work);  in tsi721_pw_handler()
284 	struct tsi721_device *priv = container_of(work, struct tsi721_device,  in tsi721_pw_dpc()  local
292 	while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)msg_buffer,  in tsi721_pw_dpc()
293 			 TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {  in tsi721_pw_dpc()
320 	struct tsi721_device *priv = mport->priv;  in tsi721_pw_enable()  local
323 	rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);  in tsi721_pw_enable()
332 		  priv->regs + TSI721_RIO_PW_RX_STAT);  in tsi721_pw_enable()
334 	iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);  in tsi721_pw_enable()
351 	struct tsi721_device *priv = mport->priv;  in tsi721_dsend()  local
357 	dev_dbg(&priv->pdev->dev,  in tsi721_dsend()
359 	iowrite16be(data, priv->odb_base + offset);  in tsi721_dsend()
375 	struct tsi721_device *priv = mport->priv;  in tsi721_dbell_handler()  local
379 	regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));  in tsi721_dbell_handler()
382 		priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));  in tsi721_dbell_handler()
384 	schedule_work(&priv->idb_work);  in tsi721_dbell_handler()
391 	struct tsi721_device *priv = container_of(work, struct tsi721_device,  in tsi721_db_dpc()  local
407 	mport = priv->mport;  in tsi721_db_dpc()
409 	wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;  in tsi721_db_dpc()
410 	rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;  in tsi721_db_dpc()
413 		idb_entry = (u64 *)(priv->idb_base +  in tsi721_db_dpc()
433 			dev_dbg(&priv->pdev->dev,  in tsi721_db_dpc()
439 		wr_ptr = ioread32(priv->regs +  in tsi721_db_dpc()
444 		priv->regs + TSI721_IDQ_RP(IDB_QUEUE));  in tsi721_db_dpc()
447 	regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));  in tsi721_db_dpc()
450 		priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));  in tsi721_db_dpc()
452 	wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;  in tsi721_db_dpc()
454 		schedule_work(&priv->idb_work);  in tsi721_db_dpc()
468 	struct tsi721_device *priv = mport->priv;  in tsi721_irqhandler()  local
475 	if (priv->flags & TSI721_USING_MSI)  in tsi721_irqhandler()
476 		iowrite32(0, priv->regs + TSI721_DEV_INTE);  in tsi721_irqhandler()
478 	dev_int = ioread32(priv->regs + TSI721_DEV_INT);  in tsi721_irqhandler()
482 	dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);  in tsi721_irqhandler()
488 			intval = ioread32(priv->regs +  in tsi721_irqhandler()
493 				dev_info(&priv->pdev->dev,  in tsi721_irqhandler()
498 				priv->regs + TSI721_SR_CHINT(IDB_QUEUE));  in tsi721_irqhandler()
499 			ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));  in tsi721_irqhandler()
512 			ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_irqhandler()
514 			iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_irqhandler()
522 				tsi721_imsg_handler(priv, ch);  in tsi721_irqhandler()
528 			ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_irqhandler()
530 			iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_irqhandler()
539 				tsi721_omsg_handler(priv, ch);  in tsi721_irqhandler()
546 		intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);  in tsi721_irqhandler()
556 			dev_dbg(&priv->pdev->dev,  in tsi721_irqhandler()
562 				tsi721_bdma_handler(&priv->bdma[ch]);  in tsi721_irqhandler()
569 	if (priv->flags & TSI721_USING_MSI) {  in tsi721_irqhandler()
572 		iowrite32(dev_int, priv->regs + TSI721_DEV_INTE);  in tsi721_irqhandler()
578 static void tsi721_interrupts_init(struct tsi721_device *priv)  in tsi721_interrupts_init()  argument
584 		priv->regs + TSI721_SR_CHINT(IDB_QUEUE));  in tsi721_interrupts_init()
586 		priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));  in tsi721_interrupts_init()
590 		priv->regs + TSI721_RIO_EM_DEV_INT_EN);  in tsi721_interrupts_init()
600 	iowrite32(intr,	priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_interrupts_init()
602 	if (priv->flags & TSI721_USING_MSIX)  in tsi721_interrupts_init()
608 	iowrite32(intr, priv->regs + TSI721_DEV_INTE);  in tsi721_interrupts_init()
609 	ioread32(priv->regs + TSI721_DEV_INTE);  in tsi721_interrupts_init()
622 	struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;  in tsi721_omsg_msix()  local
625 	mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;  in tsi721_omsg_msix()
626 	tsi721_omsg_handler(priv, mbox);  in tsi721_omsg_msix()
639 	struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;  in tsi721_imsg_msix()  local
642 	mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;  in tsi721_imsg_msix()
643 	tsi721_imsg_handler(priv, mbox + 4);  in tsi721_imsg_msix()
656 	struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;  in tsi721_srio_msix()  local
660 	srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);  in tsi721_srio_msix()
678 	struct tsi721_device *priv = ((struct rio_mport *)ptr)->priv;  in tsi721_sr2pc_ch_msix()  local
682 	sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));  in tsi721_sr2pc_ch_msix()
687 	iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));  in tsi721_sr2pc_ch_msix()
689 	sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));  in tsi721_sr2pc_ch_msix()
704 	struct tsi721_device *priv = mport->priv;  in tsi721_request_msix()  local
707 	err = request_irq(priv->msix[TSI721_VECT_IDB].vector,  in tsi721_request_msix()
709 			priv->msix[TSI721_VECT_IDB].irq_name, (void *)mport);  in tsi721_request_msix()
713 	err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,  in tsi721_request_msix()
715 			priv->msix[TSI721_VECT_PWRX].irq_name, (void *)mport);  in tsi721_request_msix()
718 			priv->msix[TSI721_VECT_IDB].vector,  in tsi721_request_msix()
731 static int tsi721_enable_msix(struct tsi721_device *priv)  in tsi721_enable_msix()  argument
771 	err = pci_enable_msix_exact(priv->pdev, entries, ARRAY_SIZE(entries));  in tsi721_enable_msix()
773 		dev_err(&priv->pdev->dev,  in tsi721_enable_msix()
781 	priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;  in tsi721_enable_msix()
782 	snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,  in tsi721_enable_msix()
783 		 DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));  in tsi721_enable_msix()
784 	priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;  in tsi721_enable_msix()
785 	snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,  in tsi721_enable_msix()
786 		 DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));  in tsi721_enable_msix()
789 		priv->msix[TSI721_VECT_IMB0_RCV + i].vector =  in tsi721_enable_msix()
791 		snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,  in tsi721_enable_msix()
793 			 i, pci_name(priv->pdev));  in tsi721_enable_msix()
795 		priv->msix[TSI721_VECT_IMB0_INT + i].vector =  in tsi721_enable_msix()
797 		snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,  in tsi721_enable_msix()
799 			 i, pci_name(priv->pdev));  in tsi721_enable_msix()
801 		priv->msix[TSI721_VECT_OMB0_DONE + i].vector =  in tsi721_enable_msix()
803 		snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,  in tsi721_enable_msix()
805 			 i, pci_name(priv->pdev));  in tsi721_enable_msix()
807 		priv->msix[TSI721_VECT_OMB0_INT + i].vector =  in tsi721_enable_msix()
809 		snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,  in tsi721_enable_msix()
811 			 i, pci_name(priv->pdev));  in tsi721_enable_msix()
816 		priv->msix[TSI721_VECT_DMA0_DONE + i].vector =  in tsi721_enable_msix()
818 		snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,  in tsi721_enable_msix()
820 			 i, pci_name(priv->pdev));  in tsi721_enable_msix()
822 		priv->msix[TSI721_VECT_DMA0_INT + i].vector =  in tsi721_enable_msix()
824 		snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,  in tsi721_enable_msix()
826 			 i, pci_name(priv->pdev));  in tsi721_enable_msix()
836 	struct tsi721_device *priv = mport->priv;  in tsi721_request_irq()  local
840 	if (priv->flags & TSI721_USING_MSIX)  in tsi721_request_irq()
844 		err = request_irq(priv->pdev->irq, tsi721_irqhandler,  in tsi721_request_irq()
845 			  (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,  in tsi721_request_irq()
849 		dev_err(&priv->pdev->dev,  in tsi721_request_irq()
862 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)  in tsi721_init_pc2sr_mapping()  argument
868 		iowrite32(0, priv->regs + TSI721_OBWINLB(i));  in tsi721_init_pc2sr_mapping()
887 	struct tsi721_device *priv = mport->priv;  in tsi721_rio_map_inb_mem()  local
897 		regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));  in tsi721_rio_map_inb_mem()
903 		dev_err(&priv->pdev->dev,  in tsi721_rio_map_inb_mem()
909 			priv->regs + TSI721_IBWIN_SZ(i));  in tsi721_rio_map_inb_mem()
911 	iowrite32(((u64)lstart >> 32), priv->regs + TSI721_IBWIN_TUA(i));  in tsi721_rio_map_inb_mem()
913 		  priv->regs + TSI721_IBWIN_TLA(i));  in tsi721_rio_map_inb_mem()
915 	iowrite32(rstart >> 32, priv->regs + TSI721_IBWIN_UB(i));  in tsi721_rio_map_inb_mem()
917 		priv->regs + TSI721_IBWIN_LB(i));  in tsi721_rio_map_inb_mem()
918 	dev_dbg(&priv->pdev->dev,  in tsi721_rio_map_inb_mem()
933 	struct tsi721_device *priv = mport->priv;  in tsi721_rio_unmap_inb_mem()  local
940 		regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));  in tsi721_rio_unmap_inb_mem()
942 			regval = ioread32(priv->regs + TSI721_IBWIN_TUA(i));  in tsi721_rio_unmap_inb_mem()
944 			regval = ioread32(priv->regs + TSI721_IBWIN_TLA(i));  in tsi721_rio_unmap_inb_mem()
948 				iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));  in tsi721_rio_unmap_inb_mem()
962 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)  in tsi721_init_sr2pc_mapping()  argument
968 		iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));  in tsi721_init_sr2pc_mapping()
978 static int tsi721_port_write_init(struct tsi721_device *priv)  in tsi721_port_write_init()  argument
980 	priv->pw_discard_count = 0;  in tsi721_port_write_init()
981 	INIT_WORK(&priv->pw_work, tsi721_pw_dpc);  in tsi721_port_write_init()
982 	spin_lock_init(&priv->pw_fifo_lock);  in tsi721_port_write_init()
983 	if (kfifo_alloc(&priv->pw_fifo,  in tsi721_port_write_init()
985 		dev_err(&priv->pdev->dev, "PW FIFO allocation failed\n");  in tsi721_port_write_init()
990 	iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);  in tsi721_port_write_init()
994 static int tsi721_doorbell_init(struct tsi721_device *priv)  in tsi721_doorbell_init()  argument
1002 	priv->db_discard_count = 0;  in tsi721_doorbell_init()
1003 	INIT_WORK(&priv->idb_work, tsi721_db_dpc);  in tsi721_doorbell_init()
1006 	priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,  in tsi721_doorbell_init()
1008 				&priv->idb_dma, GFP_KERNEL);  in tsi721_doorbell_init()
1009 	if (!priv->idb_base)  in tsi721_doorbell_init()
1012 	dev_dbg(&priv->pdev->dev, "Allocated IDB buffer @ %p (phys = %llx)\n",  in tsi721_doorbell_init()
1013 		priv->idb_base, (unsigned long long)priv->idb_dma);  in tsi721_doorbell_init()
1016 		priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));  in tsi721_doorbell_init()
1017 	iowrite32(((u64)priv->idb_dma >> 32),  in tsi721_doorbell_init()
1018 		priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));  in tsi721_doorbell_init()
1019 	iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),  in tsi721_doorbell_init()
1020 		priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));  in tsi721_doorbell_init()
1022 	iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));  in tsi721_doorbell_init()
1024 	iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));  in tsi721_doorbell_init()
1026 	iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));  in tsi721_doorbell_init()
1031 static void tsi721_doorbell_free(struct tsi721_device *priv)  in tsi721_doorbell_free()  argument
1033 	if (priv->idb_base == NULL)  in tsi721_doorbell_free()
1037 	dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,  in tsi721_doorbell_free()
1038 			  priv->idb_base, priv->idb_dma);  in tsi721_doorbell_free()
1039 	priv->idb_base = NULL;  in tsi721_doorbell_free()
1050 static int tsi721_bdma_maint_init(struct tsi721_device *priv)  in tsi721_bdma_maint_init()  argument
1059 	dev_dbg(&priv->pdev->dev,  in tsi721_bdma_maint_init()
1067 	priv->mdma.ch_id = TSI721_DMACH_MAINT;  in tsi721_bdma_maint_init()
1068 	regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);  in tsi721_bdma_maint_init()
1071 	bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,  in tsi721_bdma_maint_init()
1077 	priv->mdma.bd_num = bd_num;  in tsi721_bdma_maint_init()
1078 	priv->mdma.bd_phys = bd_phys;  in tsi721_bdma_maint_init()
1079 	priv->mdma.bd_base = bd_ptr;  in tsi721_bdma_maint_init()
1081 	dev_dbg(&priv->pdev->dev, "DMA descriptors @ %p (phys = %llx)\n",  in tsi721_bdma_maint_init()
1088 	sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,  in tsi721_bdma_maint_init()
1093 		dma_free_coherent(&priv->pdev->dev,  in tsi721_bdma_maint_init()
1096 		priv->mdma.bd_base = NULL;  in tsi721_bdma_maint_init()
1100 	priv->mdma.sts_phys = sts_phys;  in tsi721_bdma_maint_init()
1101 	priv->mdma.sts_base = sts_ptr;  in tsi721_bdma_maint_init()
1102 	priv->mdma.sts_size = sts_size;  in tsi721_bdma_maint_init()
1104 	dev_dbg(&priv->pdev->dev,  in tsi721_bdma_maint_init()
1139 static int tsi721_bdma_maint_free(struct tsi721_device *priv)  in tsi721_bdma_maint_free()  argument
1142 	struct tsi721_bdma_maint *mdma = &priv->mdma;  in tsi721_bdma_maint_free()
1143 	void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);  in tsi721_bdma_maint_free()
1157 	dma_free_coherent(&priv->pdev->dev,  in tsi721_bdma_maint_free()
1163 	dma_free_coherent(&priv->pdev->dev,  in tsi721_bdma_maint_free()
1172 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,  in tsi721_imsg_interrupt_enable()  argument
1181 	iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));  in tsi721_imsg_interrupt_enable()
1184 	rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));  in tsi721_imsg_interrupt_enable()
1185 	iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));  in tsi721_imsg_interrupt_enable()
1187 	if (priv->flags & TSI721_USING_MSIX)  in tsi721_imsg_interrupt_enable()
1195 	rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_imsg_interrupt_enable()
1197 		  priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_imsg_interrupt_enable()
1202 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,  in tsi721_imsg_interrupt_disable()  argument
1211 	iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));  in tsi721_imsg_interrupt_disable()
1214 	rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));  in tsi721_imsg_interrupt_disable()
1216 	iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));  in tsi721_imsg_interrupt_disable()
1218 	if (priv->flags & TSI721_USING_MSIX)  in tsi721_imsg_interrupt_disable()
1226 	rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_imsg_interrupt_disable()
1228 	iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_imsg_interrupt_disable()
1233 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,  in tsi721_omsg_interrupt_enable()  argument
1242 	iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));  in tsi721_omsg_interrupt_enable()
1245 	rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));  in tsi721_omsg_interrupt_enable()
1246 	iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));  in tsi721_omsg_interrupt_enable()
1248 	if (priv->flags & TSI721_USING_MSIX)  in tsi721_omsg_interrupt_enable()
1256 	rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_omsg_interrupt_enable()
1258 		  priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_omsg_interrupt_enable()
1263 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,  in tsi721_omsg_interrupt_disable()  argument
1272 	iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));  in tsi721_omsg_interrupt_disable()
1275 	rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));  in tsi721_omsg_interrupt_disable()
1277 	iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));  in tsi721_omsg_interrupt_disable()
1279 	if (priv->flags & TSI721_USING_MSIX)  in tsi721_omsg_interrupt_disable()
1287 	rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_omsg_interrupt_disable()
1289 	iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_omsg_interrupt_disable()
1304 	struct tsi721_device *priv = mport->priv;  in tsi721_add_outb_message()  local
1308 	if (!priv->omsg_init[mbox] ||  in tsi721_add_outb_message()
1312 	tx_slot = priv->omsg_ring[mbox].tx_slot;  in tsi721_add_outb_message()
1315 	memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);  in tsi721_add_outb_message()
1321 	desc = priv->omsg_ring[mbox].omd_base;  in tsi721_add_outb_message()
1330 		cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &  in tsi721_add_outb_message()
1333 		cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);  in tsi721_add_outb_message()
1335 	priv->omsg_ring[mbox].wr_count++;  in tsi721_add_outb_message()
1338 	if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {  in tsi721_add_outb_message()
1339 		priv->omsg_ring[mbox].tx_slot = 0;  in tsi721_add_outb_message()
1341 		priv->omsg_ring[mbox].wr_count++;  in tsi721_add_outb_message()
1347 	iowrite32(priv->omsg_ring[mbox].wr_count,  in tsi721_add_outb_message()
1348 		priv->regs + TSI721_OBDMAC_DWRCNT(mbox));  in tsi721_add_outb_message()
1349 	ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));  in tsi721_add_outb_message()
1361 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)  in tsi721_omsg_handler()  argument
1365 	spin_lock(&priv->omsg_ring[ch].lock);  in tsi721_omsg_handler()
1367 	omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));  in tsi721_omsg_handler()
1370 		dev_info(&priv->pdev->dev,  in tsi721_omsg_handler()
1384 		srd_ptr = priv->omsg_ring[ch].sts_rdptr;  in tsi721_omsg_handler()
1385 		sts_ptr = priv->omsg_ring[ch].sts_base;  in tsi721_omsg_handler()
1395 			srd_ptr %= priv->omsg_ring[ch].sts_size;  in tsi721_omsg_handler()
1402 		priv->omsg_ring[ch].sts_rdptr = srd_ptr;  in tsi721_omsg_handler()
1403 		iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));  in tsi721_omsg_handler()
1405 		if (!priv->mport->outb_msg[ch].mcback)  in tsi721_omsg_handler()
1410 		tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/  in tsi721_omsg_handler()
1418 		if (tx_slot == priv->omsg_ring[ch].size) {  in tsi721_omsg_handler()
1421 					(u64)priv->omsg_ring[ch].omd_phys)/  in tsi721_omsg_handler()
1429 		if (tx_slot == priv->omsg_ring[ch].size)  in tsi721_omsg_handler()
1431 		BUG_ON(tx_slot >= priv->omsg_ring[ch].size);  in tsi721_omsg_handler()
1432 		priv->mport->outb_msg[ch].mcback(priv->mport,  in tsi721_omsg_handler()
1433 				priv->omsg_ring[ch].dev_id, ch,  in tsi721_omsg_handler()
1445 		dev_dbg(&priv->pdev->dev, "OB MSG ABORT ch_stat=%x\n",  in tsi721_omsg_handler()
1446 			ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));  in tsi721_omsg_handler()
1449 				priv->regs + TSI721_OBDMAC_INT(ch));  in tsi721_omsg_handler()
1451 				priv->regs + TSI721_OBDMAC_CTL(ch));  in tsi721_omsg_handler()
1452 		ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));  in tsi721_omsg_handler()
1455 		if (priv->mport->outb_msg[ch].mcback)  in tsi721_omsg_handler()
1456 			priv->mport->outb_msg[ch].mcback(priv->mport,  in tsi721_omsg_handler()
1457 					priv->omsg_ring[ch].dev_id, ch,  in tsi721_omsg_handler()
1458 					priv->omsg_ring[ch].tx_slot);  in tsi721_omsg_handler()
1460 		iowrite32(priv->omsg_ring[ch].tx_slot,  in tsi721_omsg_handler()
1461 			priv->regs + TSI721_OBDMAC_DRDCNT(ch));  in tsi721_omsg_handler()
1462 		ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));  in tsi721_omsg_handler()
1463 		priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;  in tsi721_omsg_handler()
1464 		priv->omsg_ring[ch].sts_rdptr = 0;  in tsi721_omsg_handler()
1468 	iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));  in tsi721_omsg_handler()
1470 	if (!(priv->flags & TSI721_USING_MSIX)) {  in tsi721_omsg_handler()
1474 		ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_omsg_handler()
1476 		iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_omsg_handler()
1479 	spin_unlock(&priv->omsg_ring[ch].lock);  in tsi721_omsg_handler()
1492 	struct tsi721_device *priv = mport->priv;  in tsi721_open_outb_mbox()  local
1503 	priv->omsg_ring[mbox].dev_id = dev_id;  in tsi721_open_outb_mbox()
1504 	priv->omsg_ring[mbox].size = entries;  in tsi721_open_outb_mbox()
1505 	priv->omsg_ring[mbox].sts_rdptr = 0;  in tsi721_open_outb_mbox()
1506 	spin_lock_init(&priv->omsg_ring[mbox].lock);  in tsi721_open_outb_mbox()
1511 		priv->omsg_ring[mbox].omq_base[i] =  in tsi721_open_outb_mbox()
1513 				&priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,  in tsi721_open_outb_mbox()
1514 				&priv->omsg_ring[mbox].omq_phys[i],  in tsi721_open_outb_mbox()
1516 		if (priv->omsg_ring[mbox].omq_base[i] == NULL) {  in tsi721_open_outb_mbox()
1517 			dev_dbg(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1526 	priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(  in tsi721_open_outb_mbox()
1527 				&priv->pdev->dev,  in tsi721_open_outb_mbox()
1529 				&priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);  in tsi721_open_outb_mbox()
1530 	if (priv->omsg_ring[mbox].omd_base == NULL) {  in tsi721_open_outb_mbox()
1531 		dev_dbg(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1538 	priv->omsg_ring[mbox].tx_slot = 0;  in tsi721_open_outb_mbox()
1541 	priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);  in tsi721_open_outb_mbox()
1542 	priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1543 			priv->omsg_ring[mbox].sts_size *  in tsi721_open_outb_mbox()
1545 			&priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);  in tsi721_open_outb_mbox()
1546 	if (priv->omsg_ring[mbox].sts_base == NULL) {  in tsi721_open_outb_mbox()
1547 		dev_dbg(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1559 	iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),  in tsi721_open_outb_mbox()
1560 			priv->regs + TSI721_OBDMAC_DPTRH(mbox));  in tsi721_open_outb_mbox()
1561 	iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &  in tsi721_open_outb_mbox()
1563 			priv->regs + TSI721_OBDMAC_DPTRL(mbox));  in tsi721_open_outb_mbox()
1566 	iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),  in tsi721_open_outb_mbox()
1567 			priv->regs + TSI721_OBDMAC_DSBH(mbox));  in tsi721_open_outb_mbox()
1568 	iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &  in tsi721_open_outb_mbox()
1570 			priv->regs + TSI721_OBDMAC_DSBL(mbox));  in tsi721_open_outb_mbox()
1571 	iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),  in tsi721_open_outb_mbox()
1572 		priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));  in tsi721_open_outb_mbox()
1577 	if (priv->flags & TSI721_USING_MSIX) {  in tsi721_open_outb_mbox()
1580 			priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,  in tsi721_open_outb_mbox()
1582 			priv->msix[TSI721_VECT_OMB0_DONE + mbox].irq_name,  in tsi721_open_outb_mbox()
1586 			dev_dbg(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1592 		rc = request_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,  in tsi721_open_outb_mbox()
1594 			priv->msix[TSI721_VECT_OMB0_INT + mbox].irq_name,  in tsi721_open_outb_mbox()
1598 			dev_dbg(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1602 				priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,  in tsi721_open_outb_mbox()
1609 	tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);  in tsi721_open_outb_mbox()
1612 	bd_ptr = priv->omsg_ring[mbox].omd_base;  in tsi721_open_outb_mbox()
1616 		cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &  in tsi721_open_outb_mbox()
1619 		cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);  in tsi721_open_outb_mbox()
1620 	priv->omsg_ring[mbox].wr_count = 0;  in tsi721_open_outb_mbox()
1624 	iowrite32(TSI721_OBDMAC_CTL_INIT, priv->regs + TSI721_OBDMAC_CTL(mbox));  in tsi721_open_outb_mbox()
1625 	ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));  in tsi721_open_outb_mbox()
1628 	priv->omsg_init[mbox] = 1;  in tsi721_open_outb_mbox()
1634 	dma_free_coherent(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1635 		priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),  in tsi721_open_outb_mbox()
1636 		priv->omsg_ring[mbox].sts_base,  in tsi721_open_outb_mbox()
1637 		priv->omsg_ring[mbox].sts_phys);  in tsi721_open_outb_mbox()
1639 	priv->omsg_ring[mbox].sts_base = NULL;  in tsi721_open_outb_mbox()
1643 	dma_free_coherent(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1645 		priv->omsg_ring[mbox].omd_base,  in tsi721_open_outb_mbox()
1646 		priv->omsg_ring[mbox].omd_phys);  in tsi721_open_outb_mbox()
1648 	priv->omsg_ring[mbox].omd_base = NULL;  in tsi721_open_outb_mbox()
1651 	for (i = 0; i < priv->omsg_ring[mbox].size; i++) {  in tsi721_open_outb_mbox()
1652 		if (priv->omsg_ring[mbox].omq_base[i]) {  in tsi721_open_outb_mbox()
1653 			dma_free_coherent(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1655 				priv->omsg_ring[mbox].omq_base[i],  in tsi721_open_outb_mbox()
1656 				priv->omsg_ring[mbox].omq_phys[i]);  in tsi721_open_outb_mbox()
1658 			priv->omsg_ring[mbox].omq_base[i] = NULL;  in tsi721_open_outb_mbox()
1673 	struct tsi721_device *priv = mport->priv;  in tsi721_close_outb_mbox()  local
1676 	if (!priv->omsg_init[mbox])  in tsi721_close_outb_mbox()
1678 	priv->omsg_init[mbox] = 0;  in tsi721_close_outb_mbox()
1682 	tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);  in tsi721_close_outb_mbox()
1685 	if (priv->flags & TSI721_USING_MSIX) {  in tsi721_close_outb_mbox()
1686 		free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,  in tsi721_close_outb_mbox()
1688 		free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,  in tsi721_close_outb_mbox()
1694 	dma_free_coherent(&priv->pdev->dev,  in tsi721_close_outb_mbox()
1695 		priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),  in tsi721_close_outb_mbox()
1696 		priv->omsg_ring[mbox].sts_base,  in tsi721_close_outb_mbox()
1697 		priv->omsg_ring[mbox].sts_phys);  in tsi721_close_outb_mbox()
1699 	priv->omsg_ring[mbox].sts_base = NULL;  in tsi721_close_outb_mbox()
1702 	dma_free_coherent(&priv->pdev->dev,  in tsi721_close_outb_mbox()
1703 		(priv->omsg_ring[mbox].size + 1) *  in tsi721_close_outb_mbox()
1705 		priv->omsg_ring[mbox].omd_base,  in tsi721_close_outb_mbox()
1706 		priv->omsg_ring[mbox].omd_phys);  in tsi721_close_outb_mbox()
1708 	priv->omsg_ring[mbox].omd_base = NULL;  in tsi721_close_outb_mbox()
1711 	for (i = 0; i < priv->omsg_ring[mbox].size; i++) {  in tsi721_close_outb_mbox()
1712 		if (priv->omsg_ring[mbox].omq_base[i]) {  in tsi721_close_outb_mbox()
1713 			dma_free_coherent(&priv->pdev->dev,  in tsi721_close_outb_mbox()
1715 				priv->omsg_ring[mbox].omq_base[i],  in tsi721_close_outb_mbox()
1716 				priv->omsg_ring[mbox].omq_phys[i]);  in tsi721_close_outb_mbox()
1718 			priv->omsg_ring[mbox].omq_base[i] = NULL;  in tsi721_close_outb_mbox()
1730 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)  in tsi721_imsg_handler()  argument
1735 	spin_lock(&priv->imsg_ring[mbox].lock);  in tsi721_imsg_handler()
1737 	imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));  in tsi721_imsg_handler()
1740 		dev_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout\n",  in tsi721_imsg_handler()
1744 		dev_info(&priv->pdev->dev, "IB MBOX%d PCIe error\n",  in tsi721_imsg_handler()
1748 		dev_info(&priv->pdev->dev,  in tsi721_imsg_handler()
1752 	iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));  in tsi721_imsg_handler()
1756 		priv->mport->inb_msg[mbox].mcback)  in tsi721_imsg_handler()
1757 		priv->mport->inb_msg[mbox].mcback(priv->mport,  in tsi721_imsg_handler()
1758 				priv->imsg_ring[mbox].dev_id, mbox, -1);  in tsi721_imsg_handler()
1760 	if (!(priv->flags & TSI721_USING_MSIX)) {  in tsi721_imsg_handler()
1764 		ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_imsg_handler()
1766 		iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_imsg_handler()
1769 	spin_unlock(&priv->imsg_ring[mbox].lock);  in tsi721_imsg_handler()
1782 	struct tsi721_device *priv = mport->priv;  in tsi721_open_inb_mbox()  local
1796 	priv->imsg_ring[mbox].dev_id = dev_id;  in tsi721_open_inb_mbox()
1797 	priv->imsg_ring[mbox].size = entries;  in tsi721_open_inb_mbox()
1798 	priv->imsg_ring[mbox].rx_slot = 0;  in tsi721_open_inb_mbox()
1799 	priv->imsg_ring[mbox].desc_rdptr = 0;  in tsi721_open_inb_mbox()
1800 	priv->imsg_ring[mbox].fq_wrptr = 0;  in tsi721_open_inb_mbox()
1801 	for (i = 0; i < priv->imsg_ring[mbox].size; i++)  in tsi721_open_inb_mbox()
1802 		priv->imsg_ring[mbox].imq_base[i] = NULL;  in tsi721_open_inb_mbox()
1803 	spin_lock_init(&priv->imsg_ring[mbox].lock);  in tsi721_open_inb_mbox()
1806 	priv->imsg_ring[mbox].buf_base =  in tsi721_open_inb_mbox()
1807 		dma_alloc_coherent(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1809 				   &priv->imsg_ring[mbox].buf_phys,  in tsi721_open_inb_mbox()
1812 	if (priv->imsg_ring[mbox].buf_base == NULL) {  in tsi721_open_inb_mbox()
1813 		dev_err(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1820 	priv->imsg_ring[mbox].imfq_base =  in tsi721_open_inb_mbox()
1821 		dma_alloc_coherent(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1823 				   &priv->imsg_ring[mbox].imfq_phys,  in tsi721_open_inb_mbox()
1826 	if (priv->imsg_ring[mbox].imfq_base == NULL) {  in tsi721_open_inb_mbox()
1827 		dev_err(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1834 	priv->imsg_ring[mbox].imd_base =  in tsi721_open_inb_mbox()
1835 		dma_alloc_coherent(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1837 				   &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);  in tsi721_open_inb_mbox()
1839 	if (priv->imsg_ring[mbox].imd_base == NULL) {  in tsi721_open_inb_mbox()
1840 		dev_err(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1848 	free_ptr = priv->imsg_ring[mbox].imfq_base;  in tsi721_open_inb_mbox()
1851 				(u64)(priv->imsg_ring[mbox].buf_phys) +  in tsi721_open_inb_mbox()
1861 	if (!(priv->flags & TSI721_IMSGID_SET)) {  in tsi721_open_inb_mbox()
1862 		iowrite32((u32)priv->mport->host_deviceid,  in tsi721_open_inb_mbox()
1863 			priv->regs + TSI721_IB_DEVID);  in tsi721_open_inb_mbox()
1864 		priv->flags |= TSI721_IMSGID_SET;  in tsi721_open_inb_mbox()
1872 	iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),  in tsi721_open_inb_mbox()
1873 		priv->regs + TSI721_IBDMAC_FQBH(ch));  in tsi721_open_inb_mbox()
1874 	iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &  in tsi721_open_inb_mbox()
1876 		priv->regs+TSI721_IBDMAC_FQBL(ch));  in tsi721_open_inb_mbox()
1878 		priv->regs + TSI721_IBDMAC_FQSZ(ch));  in tsi721_open_inb_mbox()
1881 	iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),  in tsi721_open_inb_mbox()
1882 		priv->regs + TSI721_IBDMAC_DQBH(ch));  in tsi721_open_inb_mbox()
1883 	iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &  in tsi721_open_inb_mbox()
1885 		priv->regs+TSI721_IBDMAC_DQBL(ch));  in tsi721_open_inb_mbox()
1887 		priv->regs + TSI721_IBDMAC_DQSZ(ch));  in tsi721_open_inb_mbox()
1892 	if (priv->flags & TSI721_USING_MSIX) {  in tsi721_open_inb_mbox()
1894 		rc = request_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,  in tsi721_open_inb_mbox()
1896 			priv->msix[TSI721_VECT_IMB0_RCV + mbox].irq_name,  in tsi721_open_inb_mbox()
1900 			dev_dbg(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1906 		rc = request_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,  in tsi721_open_inb_mbox()
1908 			priv->msix[TSI721_VECT_IMB0_INT + mbox].irq_name,  in tsi721_open_inb_mbox()
1912 			dev_dbg(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1916 				priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,  in tsi721_open_inb_mbox()
1923 	tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);  in tsi721_open_inb_mbox()
1926 	iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));  in tsi721_open_inb_mbox()
1927 	ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));  in tsi721_open_inb_mbox()
1929 	priv->imsg_ring[mbox].fq_wrptr = entries - 1;  in tsi721_open_inb_mbox()
1930 	iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));  in tsi721_open_inb_mbox()
1932 	priv->imsg_init[mbox] = 1;  in tsi721_open_inb_mbox()
1937 	dma_free_coherent(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1938 		priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),  in tsi721_open_inb_mbox()
1939 		priv->imsg_ring[mbox].imd_base,  in tsi721_open_inb_mbox()
1940 		priv->imsg_ring[mbox].imd_phys);  in tsi721_open_inb_mbox()
1942 	priv->imsg_ring[mbox].imd_base = NULL;  in tsi721_open_inb_mbox()
1946 	dma_free_coherent(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1947 		priv->imsg_ring[mbox].size * 8,  in tsi721_open_inb_mbox()
1948 		priv->imsg_ring[mbox].imfq_base,  in tsi721_open_inb_mbox()
1949 		priv->imsg_ring[mbox].imfq_phys);  in tsi721_open_inb_mbox()
1951 	priv->imsg_ring[mbox].imfq_base = NULL;  in tsi721_open_inb_mbox()
1954 	dma_free_coherent(&priv->pdev->dev,  in tsi721_open_inb_mbox()
1955 		priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,  in tsi721_open_inb_mbox()
1956 		priv->imsg_ring[mbox].buf_base,  in tsi721_open_inb_mbox()
1957 		priv->imsg_ring[mbox].buf_phys);  in tsi721_open_inb_mbox()
1959 	priv->imsg_ring[mbox].buf_base = NULL;  in tsi721_open_inb_mbox()
1972 	struct tsi721_device *priv = mport->priv;  in tsi721_close_inb_mbox()  local
1976 	if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */  in tsi721_close_inb_mbox()
1978 	priv->imsg_init[mbox] = 0;  in tsi721_close_inb_mbox()
1983 	tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);  in tsi721_close_inb_mbox()
1986 	if (priv->flags & TSI721_USING_MSIX) {  in tsi721_close_inb_mbox()
1987 		free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,  in tsi721_close_inb_mbox()
1989 		free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,  in tsi721_close_inb_mbox()
1995 	for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)  in tsi721_close_inb_mbox()
1996 		priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;  in tsi721_close_inb_mbox()
1999 	dma_free_coherent(&priv->pdev->dev,  in tsi721_close_inb_mbox()
2000 		priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,  in tsi721_close_inb_mbox()
2001 		priv->imsg_ring[mbox].buf_base,  in tsi721_close_inb_mbox()
2002 		priv->imsg_ring[mbox].buf_phys);  in tsi721_close_inb_mbox()
2004 	priv->imsg_ring[mbox].buf_base = NULL;  in tsi721_close_inb_mbox()
2007 	dma_free_coherent(&priv->pdev->dev,  in tsi721_close_inb_mbox()
2008 		priv->imsg_ring[mbox].size * 8,  in tsi721_close_inb_mbox()
2009 		priv->imsg_ring[mbox].imfq_base,  in tsi721_close_inb_mbox()
2010 		priv->imsg_ring[mbox].imfq_phys);  in tsi721_close_inb_mbox()
2012 	priv->imsg_ring[mbox].imfq_base = NULL;  in tsi721_close_inb_mbox()
2015 	dma_free_coherent(&priv->pdev->dev,  in tsi721_close_inb_mbox()
2016 		priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),  in tsi721_close_inb_mbox()
2017 		priv->imsg_ring[mbox].imd_base,  in tsi721_close_inb_mbox()
2018 		priv->imsg_ring[mbox].imd_phys);  in tsi721_close_inb_mbox()
2020 	priv->imsg_ring[mbox].imd_base = NULL;  in tsi721_close_inb_mbox()
2031 	struct tsi721_device *priv = mport->priv;  in tsi721_add_inb_buffer()  local
2035 	rx_slot = priv->imsg_ring[mbox].rx_slot;  in tsi721_add_inb_buffer()
2036 	if (priv->imsg_ring[mbox].imq_base[rx_slot]) {  in tsi721_add_inb_buffer()
2037 		dev_err(&priv->pdev->dev,  in tsi721_add_inb_buffer()
2044 	priv->imsg_ring[mbox].imq_base[rx_slot] = buf;  in tsi721_add_inb_buffer()
2046 	if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)  in tsi721_add_inb_buffer()
2047 		priv->imsg_ring[mbox].rx_slot = 0;  in tsi721_add_inb_buffer()
2062 	struct tsi721_device *priv = mport->priv;  in tsi721_get_inb_message()  local
2072 	if (!priv->imsg_init[mbox])  in tsi721_get_inb_message()
2075 	desc = priv->imsg_ring[mbox].imd_base;  in tsi721_get_inb_message()
2076 	desc += priv->imsg_ring[mbox].desc_rdptr;  in tsi721_get_inb_message()
2081 	rx_slot = priv->imsg_ring[mbox].rx_slot;  in tsi721_get_inb_message()
2082 	while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {  in tsi721_get_inb_message()
2083 		if (++rx_slot == priv->imsg_ring[mbox].size)  in tsi721_get_inb_message()
2090 	rx_virt = priv->imsg_ring[mbox].buf_base +  in tsi721_get_inb_message()
2091 		  (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);  in tsi721_get_inb_message()
2093 	buf = priv->imsg_ring[mbox].imq_base[rx_slot];  in tsi721_get_inb_message()
2099 	priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;  in tsi721_get_inb_message()
2102 	if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)  in tsi721_get_inb_message()
2103 		priv->imsg_ring[mbox].desc_rdptr = 0;  in tsi721_get_inb_message()
2105 	iowrite32(priv->imsg_ring[mbox].desc_rdptr,  in tsi721_get_inb_message()
2106 		priv->regs + TSI721_IBDMAC_DQRP(ch));  in tsi721_get_inb_message()
2109 	free_ptr = priv->imsg_ring[mbox].imfq_base;  in tsi721_get_inb_message()
2110 	free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);  in tsi721_get_inb_message()
2112 	if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)  in tsi721_get_inb_message()
2113 		priv->imsg_ring[mbox].fq_wrptr = 0;  in tsi721_get_inb_message()
2115 	iowrite32(priv->imsg_ring[mbox].fq_wrptr,  in tsi721_get_inb_message()
2116 		priv->regs + TSI721_IBDMAC_FQWP(ch));  in tsi721_get_inb_message()
2127 static int tsi721_messages_init(struct tsi721_device *priv)  in tsi721_messages_init()  argument
2131 	iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);  in tsi721_messages_init()
2132 	iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);  in tsi721_messages_init()
2133 	iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);  in tsi721_messages_init()
2136 	iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);  in tsi721_messages_init()
2142 			priv->regs + TSI721_IBDMAC_INT(ch));  in tsi721_messages_init()
2144 		iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));  in tsi721_messages_init()
2147 				priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));  in tsi721_messages_init()
2149 				priv->regs + TSI721_SMSG_ECC_NCOR(ch));  in tsi721_messages_init()
2159 static void tsi721_disable_ints(struct tsi721_device *priv)  in tsi721_disable_ints()  argument
2164 	iowrite32(0, priv->regs + TSI721_DEV_INTE);  in tsi721_disable_ints()
2167 	iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);  in tsi721_disable_ints()
2171 		iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));  in tsi721_disable_ints()
2175 		iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));  in tsi721_disable_ints()
2178 	iowrite32(0, priv->regs + TSI721_SMSG_INTE);  in tsi721_disable_ints()
2183 			priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);  in tsi721_disable_ints()
2186 	iowrite32(0, priv->regs + TSI721_BDMA_INTE);  in tsi721_disable_ints()
2190 		iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));  in tsi721_disable_ints()
2193 	iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);  in tsi721_disable_ints()
2196 	iowrite32(0, priv->regs + TSI721_PC2SR_INTE);  in tsi721_disable_ints()
2199 	iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);  in tsi721_disable_ints()
2202 	iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);  in tsi721_disable_ints()
2203 	iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);  in tsi721_disable_ints()
2212 static int tsi721_setup_mport(struct tsi721_device *priv)  in tsi721_setup_mport()  argument
2214 	struct pci_dev *pdev = priv->pdev;  in tsi721_setup_mport()
2252 	mport->priv = (void *)priv;  in tsi721_setup_mport()
2255 	priv->mport = mport;  in tsi721_setup_mport()
2268 	if (!tsi721_enable_msix(priv))  in tsi721_setup_mport()
2269 		priv->flags |= TSI721_USING_MSIX;  in tsi721_setup_mport()
2271 		priv->flags |= TSI721_USING_MSI;  in tsi721_setup_mport()
2280 		tsi721_interrupts_init(priv);  in tsi721_setup_mport()
2289 	tsi721_register_dma(priv);  in tsi721_setup_mport()
2292 	iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |  in tsi721_setup_mport()
2294 		  priv->regs + TSI721_DEVCTL);  in tsi721_setup_mport()
2301 			  priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));  in tsi721_setup_mport()
2303 		iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));  in tsi721_setup_mport()
2316 	struct tsi721_device *priv;  in tsi721_probe()  local
2319 	priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);  in tsi721_probe()
2320 	if (priv == NULL) {  in tsi721_probe()
2332 	priv->pdev = pdev;  in tsi721_probe()
2395 	priv->regs = pci_ioremap_bar(pdev, BAR_0);  in tsi721_probe()
2396 	if (!priv->regs) {  in tsi721_probe()
2403 	priv->odb_base = pci_ioremap_bar(pdev, BAR_1);  in tsi721_probe()
2404 	if (!priv->odb_base) {  in tsi721_probe()
2449 	tsi721_disable_ints(priv);  in tsi721_probe()
2451 	tsi721_init_pc2sr_mapping(priv);  in tsi721_probe()
2452 	tsi721_init_sr2pc_mapping(priv);  in tsi721_probe()
2454 	if (tsi721_bdma_maint_init(priv)) {  in tsi721_probe()
2460 	err = tsi721_doorbell_init(priv);  in tsi721_probe()
2464 	tsi721_port_write_init(priv);  in tsi721_probe()
2466 	err = tsi721_messages_init(priv);  in tsi721_probe()
2470 	err = tsi721_setup_mport(priv);  in tsi721_probe()
2477 	tsi721_doorbell_free(priv);  in tsi721_probe()
2479 	tsi721_bdma_maint_free(priv);  in tsi721_probe()
2481 	if (priv->regs)  in tsi721_probe()
2482 		iounmap(priv->regs);  in tsi721_probe()
2483 	if (priv->odb_base)  in tsi721_probe()
2484 		iounmap(priv->odb_base);  in tsi721_probe()
2491 	kfree(priv);  in tsi721_probe()