173 #define HSDEV_FROM_AP(ap)  ((struct sata_dwc_device *)\  argument
174 					(ap)->host->private_data)
175 #define HSDEVP_FROM_AP(ap)   ((struct sata_dwc_device_port *)\  argument
176 					(ap)->private_data)
178 					(qc)->ap->host->private_data)
212 static int sata_dwc_qc_complete(struct ata_port *ap, struct ata_queued_cmd *qc,
214 static void sata_dwc_dma_xfer_complete(struct ata_port *ap, u32 check_status);
215 static void sata_dwc_port_stop(struct ata_port *ap);
254 static void sata_dwc_tf_dump(struct ata_port *ap, struct ata_taskfile *tf)  in sata_dwc_tf_dump()  argument
256 	dev_vdbg(ap->dev,  in sata_dwc_tf_dump()
260 	dev_vdbg(ap->dev,  in sata_dwc_tf_dump()
263 	dev_vdbg(ap->dev,  in sata_dwc_tf_dump()
274 	struct ata_port *ap;  in dma_dwc_xfer_done()  local
280 	ap = host->ports[port];  in dma_dwc_xfer_done()
281 	hsdevp = HSDEVP_FROM_AP(ap);  in dma_dwc_xfer_done()
282 	tag = ap->link.active_tag;  in dma_dwc_xfer_done()
293 		dev_err(ap->dev, "DMA not pending tag=0x%02x pending=%d\n",  in dma_dwc_xfer_done()
298 		sata_dwc_dma_xfer_complete(ap, 1);  in dma_dwc_xfer_done()
305 	struct ata_port *ap = qc->ap;  in dma_dwc_xfer_setup()  local
306 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in dma_dwc_xfer_setup()
307 	struct sata_dwc_device *hsdev = HSDEV_FROM_AP(ap);  in dma_dwc_xfer_setup()
348 		dev_err(link->ap->dev, "%s: Incorrect SCR offset 0x%02x\n",  in sata_dwc_scr_read()
353 	*val = in_le32(link->ap->ioaddr.scr_addr + (scr * 4));  in sata_dwc_scr_read()
354 	dev_dbg(link->ap->dev, "%s: id=%d reg=%d val=val=0x%08x\n",  in sata_dwc_scr_read()
355 		__func__, link->ap->print_id, scr, *val);  in sata_dwc_scr_read()
362 	dev_dbg(link->ap->dev, "%s: id=%d reg=%d val=val=0x%08x\n",  in sata_dwc_scr_write()
363 		__func__, link->ap->print_id, scr, val);  in sata_dwc_scr_write()
365 		dev_err(link->ap->dev, "%s: Incorrect SCR offset 0x%02x\n",  in sata_dwc_scr_write()
369 	out_le32(link->ap->ioaddr.scr_addr + (scr * 4), val);  in sata_dwc_scr_write()
403 static void sata_dwc_error_intr(struct ata_port *ap,  in sata_dwc_error_intr()  argument
406 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_error_intr()
407 	struct ata_eh_info *ehi = &ap->link.eh_info;  in sata_dwc_error_intr()
416 	status = ap->ops->sff_check_status(ap);  in sata_dwc_error_intr()
418 	tag = ap->link.active_tag;  in sata_dwc_error_intr()
420 	dev_err(ap->dev,  in sata_dwc_error_intr()
438 	qc = ata_qc_from_tag(ap, tag);  in sata_dwc_error_intr()
444 	ata_port_abort(ap);  in sata_dwc_error_intr()
458 	struct ata_port *ap;  in sata_dwc_isr()  local
472 	ap = host->ports[port];  in sata_dwc_isr()
473 	hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_isr()
475 	dev_dbg(ap->dev, "%s intpr=0x%08x active_tag=%d\n", __func__, intpr,  in sata_dwc_isr()
476 		ap->link.active_tag);  in sata_dwc_isr()
480 		sata_dwc_error_intr(ap, hsdev, intpr);  in sata_dwc_isr()
490 		dev_dbg(ap->dev, "%s: NEWFP tag=%d\n", __func__, tag);  in sata_dwc_isr()
492 			dev_warn(ap->dev, "CMD tag=%d not pending?\n", tag);  in sata_dwc_isr()
496 		qc = ata_qc_from_tag(ap, tag);  in sata_dwc_isr()
502 		qc->ap->link.active_tag = tag;  in sata_dwc_isr()
513 		if (ap->link.active_tag == ATA_TAG_POISON)  in sata_dwc_isr()
516 			tag = ap->link.active_tag;  in sata_dwc_isr()
517 		qc = ata_qc_from_tag(ap, tag);  in sata_dwc_isr()
521 			dev_err(ap->dev,  in sata_dwc_isr()
524 			ap->ops->sff_check_status(ap);  in sata_dwc_isr()
528 		status = ap->ops->sff_check_status(ap);  in sata_dwc_isr()
530 		qc->ap->link.active_tag = tag;  in sata_dwc_isr()
534 			dev_dbg(ap->dev, "interrupt ATA_ERR (0x%x)\n", status);  in sata_dwc_isr()
535 			sata_dwc_qc_complete(ap, qc, 1);  in sata_dwc_isr()
540 		dev_dbg(ap->dev, "%s non-NCQ cmd interrupt, protocol: %s\n",  in sata_dwc_isr()
553 				dev_err(ap->dev,  in sata_dwc_isr()
560 				sata_dwc_dma_xfer_complete(ap, 1);  in sata_dwc_isr()
562 			ata_sff_hsm_move(ap, qc, status, 0);  in sata_dwc_isr()
566 			if (unlikely(sata_dwc_qc_complete(ap, qc, 1)))  in sata_dwc_isr()
587 		dev_dbg(ap->dev,  in sata_dwc_isr()
595 		dev_warn(ap->dev,  in sata_dwc_isr()
601 	status = ap->ops->sff_check_status(ap);  in sata_dwc_isr()
602 	dev_dbg(ap->dev, "%s ATA status register=0x%x\n", __func__, status);  in sata_dwc_isr()
614 		qc = ata_qc_from_tag(ap, tag);  in sata_dwc_isr()
617 		qc->ap->link.active_tag = tag;  in sata_dwc_isr()
622 			dev_dbg(ap->dev, "%s ATA_ERR (0x%x)\n", __func__,  in sata_dwc_isr()
624 			sata_dwc_qc_complete(ap, qc, 1);  in sata_dwc_isr()
630 		dev_dbg(ap->dev, "%s NCQ command, protocol: %s\n", __func__,  in sata_dwc_isr()
636 				dev_warn(ap->dev, "%s: DMA not pending?\n",  in sata_dwc_isr()
639 				sata_dwc_dma_xfer_complete(ap, 1);  in sata_dwc_isr()
641 			if (unlikely(sata_dwc_qc_complete(ap, qc, 1)))  in sata_dwc_isr()
647 		ap->stats.idle_irq++;  in sata_dwc_isr()
648 		dev_warn(ap->dev, "STILL BUSY IRQ ata%d: irq trap\n",  in sata_dwc_isr()
649 			ap->print_id);  in sata_dwc_isr()
661 		dev_dbg(ap->dev,  in sata_dwc_isr()
698 static void sata_dwc_dma_xfer_complete(struct ata_port *ap, u32 check_status)  in sata_dwc_dma_xfer_complete()  argument
701 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_dma_xfer_complete()
702 	struct sata_dwc_device *hsdev = HSDEV_FROM_AP(ap);  in sata_dwc_dma_xfer_complete()
705 	tag = ap->link.active_tag;  in sata_dwc_dma_xfer_complete()
706 	qc = ata_qc_from_tag(ap, tag);  in sata_dwc_dma_xfer_complete()
708 		dev_err(ap->dev, "failed to get qc");  in sata_dwc_dma_xfer_complete()
714 		dev_info(ap->dev,  in sata_dwc_dma_xfer_complete()
725 			dev_err(ap->dev,  in sata_dwc_dma_xfer_complete()
732 		sata_dwc_qc_complete(ap, qc, check_status);  in sata_dwc_dma_xfer_complete()
733 		ap->link.active_tag = ATA_TAG_POISON;  in sata_dwc_dma_xfer_complete()
735 		sata_dwc_qc_complete(ap, qc, check_status);  in sata_dwc_dma_xfer_complete()
739 static int sata_dwc_qc_complete(struct ata_port *ap, struct ata_queued_cmd *qc,  in sata_dwc_qc_complete()  argument
745 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_qc_complete()
747 	dev_dbg(ap->dev, "%s checkstatus? %x\n", __func__, check_status);  in sata_dwc_qc_complete()
750 		dev_err(ap->dev, "TX DMA PENDING\n");  in sata_dwc_qc_complete()
752 		dev_err(ap->dev, "RX DMA PENDING\n");  in sata_dwc_qc_complete()
753 	dev_dbg(ap->dev,  in sata_dwc_qc_complete()
755 		qc->tf.command, status, ap->print_id, qc->tf.protocol);  in sata_dwc_qc_complete()
826 static int sata_dwc_port_start(struct ata_port *ap)  in sata_dwc_port_start()  argument
835 	hsdev = HSDEV_FROM_AP(ap);  in sata_dwc_port_start()
837 	dev_dbg(ap->dev, "%s: port_no=%d\n", __func__, ap->port_no);  in sata_dwc_port_start()
839 	hsdev->host = ap->host;  in sata_dwc_port_start()
840 	pdev = ap->host->dev;  in sata_dwc_port_start()
842 		dev_err(ap->dev, "%s: no ap->host->dev\n", __func__);  in sata_dwc_port_start()
850 		dev_err(ap->dev, "%s: kmalloc failed for hsdevp\n", __func__);  in sata_dwc_port_start()
874 	ap->bmdma_prd = NULL;	/* set these so libata doesn't use them */  in sata_dwc_port_start()
875 	ap->bmdma_prd_dma = 0;  in sata_dwc_port_start()
877 	if (ap->port_no == 0)  {  in sata_dwc_port_start()
878 		dev_dbg(ap->dev, "%s: clearing TXCHEN, RXCHEN in DMAC\n",  in sata_dwc_port_start()
883 		dev_dbg(ap->dev, "%s: setting burst size in DBTSR\n",  in sata_dwc_port_start()
892 	ap->private_data = hsdevp;  in sata_dwc_port_start()
893 	dev_dbg(ap->dev, "%s: done\n", __func__);  in sata_dwc_port_start()
899 	dev_dbg(ap->dev, "%s: fail. ap->id = %d\n", __func__, ap->print_id);  in sata_dwc_port_start()
903 static void sata_dwc_port_stop(struct ata_port *ap)  in sata_dwc_port_stop()  argument
905 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_port_stop()
907 	dev_dbg(ap->dev, "%s: ap->id = %d\n", __func__, ap->print_id);  in sata_dwc_port_stop()
913 	ap->private_data = NULL;  in sata_dwc_port_stop()
923 static void sata_dwc_exec_command_by_tag(struct ata_port *ap,  in sata_dwc_exec_command_by_tag()  argument
928 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_exec_command_by_tag()
930 	dev_dbg(ap->dev, "%s cmd(0x%02x): %s tag=%d\n", __func__, tf->command,  in sata_dwc_exec_command_by_tag()
933 	spin_lock_irqsave(&ap->host->lock, flags);  in sata_dwc_exec_command_by_tag()
935 	spin_unlock_irqrestore(&ap->host->lock, flags);  in sata_dwc_exec_command_by_tag()
943 	ata_sff_exec_command(ap, tf);  in sata_dwc_exec_command_by_tag()
948 	sata_dwc_exec_command_by_tag(qc->ap, &qc->tf, tag,  in sata_dwc_bmdma_setup_by_tag()
957 		dev_dbg(qc->ap->dev, "%s: ap->link.sactive=0x%08x tag=%d\n",  in sata_dwc_bmdma_setup()
958 			__func__, qc->ap->link.sactive, tag);  in sata_dwc_bmdma_setup()
970 	struct ata_port *ap = qc->ap;  in sata_dwc_bmdma_start_by_tag()  local
971 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_bmdma_start_by_tag()
982 		dev_err(ap->dev,  in sata_dwc_bmdma_start_by_tag()
988 	dev_dbg(ap->dev,  in sata_dwc_bmdma_start_by_tag()
992 	sata_dwc_tf_dump(ap, &qc->tf);  in sata_dwc_bmdma_start_by_tag()
997 			dev_err(ap->dev, "%s: ****** SError=0x%08x ******\n",  in sata_dwc_bmdma_start_by_tag()
1019 		dev_dbg(qc->ap->dev, "%s: ap->link.sactive=0x%08x tag=%d\n",  in sata_dwc_bmdma_start()
1020 			__func__, qc->ap->link.sactive, tag);  in sata_dwc_bmdma_start()
1024 	dev_dbg(qc->ap->dev, "%s\n", __func__);  in sata_dwc_bmdma_start()
1037 	struct ata_port *ap = qc->ap;  in sata_dwc_qc_prep_by_tag()  local
1038 	struct sata_dwc_device_port *hsdevp = HSDEVP_FROM_AP(ap);  in sata_dwc_qc_prep_by_tag()
1040 	dev_dbg(ap->dev, "%s: port=%d dma dir=%s n_elem=%d\n",  in sata_dwc_qc_prep_by_tag()
1041 		__func__, ap->port_no, get_dma_dir_descript(qc->dma_dir),  in sata_dwc_qc_prep_by_tag()
1046 		dev_err(ap->dev, "%s: dma_dwc_xfer_setup returns NULL\n",  in sata_dwc_qc_prep_by_tag()
1057 	struct ata_port *ap = qc->ap;  in sata_dwc_qc_issue()  local
1060 	if (qc->tag > 0 || ap->link.sactive > 1)  in sata_dwc_qc_issue()
1061 		dev_info(ap->dev,  in sata_dwc_qc_issue()
1063 			 __func__, ap->print_id, qc->tf.command,  in sata_dwc_qc_issue()
1066 			 ap->link.active_tag, ap->link.sactive);  in sata_dwc_qc_issue()
1078 		dev_dbg(qc->ap->dev,  in sata_dwc_qc_issue()
1080 			__func__, tag, qc->ap->link.sactive, sactive);  in sata_dwc_qc_issue()
1082 		ap->ops->sff_tf_load(ap, &qc->tf);  in sata_dwc_qc_issue()
1083 		sata_dwc_exec_command_by_tag(ap, &qc->tf, qc->tag,  in sata_dwc_qc_issue()
1105 		dev_info(qc->ap->dev, "%s: qc->tag=%d ap->active_tag=0x%08x\n",  in sata_dwc_qc_prep()
1106 			 __func__, qc->tag, qc->ap->link.active_tag);  in sata_dwc_qc_prep()
1112 static void sata_dwc_error_handler(struct ata_port *ap)  in sata_dwc_error_handler()  argument
1114 	ata_sff_error_handler(ap);  in sata_dwc_error_handler()
1120 	struct sata_dwc_device *hsdev = HSDEV_FROM_AP(link->ap);  in sata_dwc_hardreset()