605 static int mv_port_start(struct ata_port *ap);
606 static void mv_port_stop(struct ata_port *ap);
613 static void mv_eh_freeze(struct ata_port *ap);
614 static void mv_eh_thaw(struct ata_port *ap);
649 static int mv_stop_edma(struct ata_port *ap);
651 static void mv_edma_cfg(struct ata_port *ap, int want_ncq, int want_edma);
653 static void mv_pmp_select(struct ata_port *ap, int pmp);
658 static void mv_pmp_error_handler(struct ata_port *ap);
659 static void mv_process_crpb_entries(struct ata_port *ap,
662 static void mv_sff_irq_clear(struct ata_port *ap);
667 static u8   mv_bmdma_status(struct ata_port *ap);
668 static u8 mv_sff_check_status(struct ata_port *ap);
935 static inline void __iomem *mv_ap_base(struct ata_port *ap)  in mv_ap_base()  argument
937 	return mv_port_base(mv_host_base(ap->host), ap->port_no);  in mv_ap_base()
955 static void mv_save_cached_regs(struct ata_port *ap)  in mv_save_cached_regs()  argument
957 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_save_cached_regs()
958 	struct mv_port_priv *pp = ap->private_data;  in mv_save_cached_regs()
1063 static void mv_enable_port_irqs(struct ata_port *ap,  in mv_enable_port_irqs()  argument
1066 	unsigned int shift, hardport, port = ap->port_no;  in mv_enable_port_irqs()
1073 	mv_set_main_irq_mask(ap->host, disable_bits, enable_bits);  in mv_enable_port_irqs()
1076 static void mv_clear_and_enable_port_irqs(struct ata_port *ap,  in mv_clear_and_enable_port_irqs()  argument
1080 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_clear_and_enable_port_irqs()
1081 	int hardport = mv_hardport_from_port(ap->port_no);  in mv_clear_and_enable_port_irqs()
1083 				mv_host_base(ap->host), ap->port_no);  in mv_clear_and_enable_port_irqs()
1097 	mv_enable_port_irqs(ap, port_irqs);  in mv_clear_and_enable_port_irqs()
1173 static void mv_start_edma(struct ata_port *ap, void __iomem *port_mmio,  in mv_start_edma()  argument
1181 			mv_stop_edma(ap);  in mv_start_edma()
1184 		struct mv_host_priv *hpriv = ap->host->private_data;  in mv_start_edma()
1186 		mv_edma_cfg(ap, want_ncq, 1);  in mv_start_edma()
1189 		mv_clear_and_enable_port_irqs(ap, port_mmio, DONE_IRQ|ERR_IRQ);  in mv_start_edma()
1196 static void mv_wait_for_edma_empty_idle(struct ata_port *ap)  in mv_wait_for_edma_empty_idle()  argument
1198 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_wait_for_edma_empty_idle()
1243 static int mv_stop_edma(struct ata_port *ap)  in mv_stop_edma()  argument
1245 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_stop_edma()
1246 	struct mv_port_priv *pp = ap->private_data;  in mv_stop_edma()
1252 	mv_wait_for_edma_empty_idle(ap);  in mv_stop_edma()
1254 		ata_port_err(ap, "Unable to stop eDMA\n");  in mv_stop_edma()
1257 	mv_edma_cfg(ap, 0, 0);  in mv_stop_edma()
1363 		*val = readl(mv_ap_base(link->ap) + ofs);  in mv_scr_read()
1374 		void __iomem *addr = mv_ap_base(link->ap) + ofs;  in mv_scr_write()
1375 		struct mv_host_priv *hpriv = link->ap->host->private_data;  in mv_scr_write()
1395 					mv_ap_base(link->ap) + LP_PHY_CTL;  in mv_scr_write()
1427 		if (sata_pmp_attached(adev->link->ap)) {  in mv6_dev_config()
1438 	struct ata_port *ap = link->ap;  in mv_qc_defer()  local
1439 	struct mv_port_priv *pp = ap->private_data;  in mv_qc_defer()
1456 	if (unlikely(ap->excl_link)) {  in mv_qc_defer()
1457 		if (link == ap->excl_link) {  in mv_qc_defer()
1458 			if (ap->nr_active_links)  in mv_qc_defer()
1469 	if (ap->nr_active_links == 0)  in mv_qc_defer()
1483 			ap->excl_link = link;  in mv_qc_defer()
1491 static void mv_config_fbs(struct ata_port *ap, int want_ncq, int want_fbs)  in mv_config_fbs()  argument
1493 	struct mv_port_priv *pp = ap->private_data;  in mv_config_fbs()
1514 	port_mmio = mv_ap_base(ap);  in mv_config_fbs()
1520 static void mv_60x1_errata_sata25(struct ata_port *ap, int want_ncq)  in mv_60x1_errata_sata25()  argument
1522 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_60x1_errata_sata25()
1547 static void mv_bmdma_enable_iie(struct ata_port *ap, int enable_bmdma)  in mv_bmdma_enable_iie()  argument
1549 	struct mv_port_priv *pp = ap->private_data;  in mv_bmdma_enable_iie()
1556 	mv_write_cached_reg(mv_ap_base(ap) + EDMA_UNKNOWN_RSVD, old, new);  in mv_bmdma_enable_iie()
1573 static void mv_soc_led_blink_enable(struct ata_port *ap)  in mv_soc_led_blink_enable()  argument
1575 	struct ata_host *host = ap->host;  in mv_soc_led_blink_enable()
1583 	hc_mmio = mv_hc_base_from_port(mv_host_base(host), ap->port_no);  in mv_soc_led_blink_enable()
1588 static void mv_soc_led_blink_disable(struct ata_port *ap)  in mv_soc_led_blink_disable()  argument
1590 	struct ata_host *host = ap->host;  in mv_soc_led_blink_disable()
1609 	hc_mmio = mv_hc_base_from_port(mv_host_base(host), ap->port_no);  in mv_soc_led_blink_disable()
1614 static void mv_edma_cfg(struct ata_port *ap, int want_ncq, int want_edma)  in mv_edma_cfg()  argument
1617 	struct mv_port_priv *pp    = ap->private_data;  in mv_edma_cfg()
1618 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_edma_cfg()
1619 	void __iomem *port_mmio    = mv_ap_base(ap);  in mv_edma_cfg()
1631 		mv_60x1_errata_sata25(ap, want_ncq);  in mv_edma_cfg()
1634 		int want_fbs = sata_pmp_attached(ap);  in mv_edma_cfg()
1645 		mv_config_fbs(ap, want_ncq, want_fbs);  in mv_edma_cfg()
1660 		mv_bmdma_enable_iie(ap, !want_edma);  in mv_edma_cfg()
1664 				mv_soc_led_blink_enable(ap);  in mv_edma_cfg()
1666 				mv_soc_led_blink_disable(ap);  in mv_edma_cfg()
1678 static void mv_port_free_dma_mem(struct ata_port *ap)  in mv_port_free_dma_mem()  argument
1680 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_port_free_dma_mem()
1681 	struct mv_port_priv *pp = ap->private_data;  in mv_port_free_dma_mem()
1717 static int mv_port_start(struct ata_port *ap)  in mv_port_start()  argument
1719 	struct device *dev = ap->host->dev;  in mv_port_start()
1720 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_port_start()
1728 	ap->private_data = pp;  in mv_port_start()
1742 		ap->flags |= ATA_FLAG_AN;  in mv_port_start()
1759 	spin_lock_irqsave(ap->lock, flags);  in mv_port_start()
1760 	mv_save_cached_regs(ap);  in mv_port_start()
1761 	mv_edma_cfg(ap, 0, 0);  in mv_port_start()
1762 	spin_unlock_irqrestore(ap->lock, flags);  in mv_port_start()
1767 	mv_port_free_dma_mem(ap);  in mv_port_start()
1780 static void mv_port_stop(struct ata_port *ap)  in mv_port_stop()  argument
1784 	spin_lock_irqsave(ap->lock, flags);  in mv_port_stop()
1785 	mv_stop_edma(ap);  in mv_port_stop()
1786 	mv_enable_port_irqs(ap, 0);  in mv_port_stop()
1787 	spin_unlock_irqrestore(ap->lock, flags);  in mv_port_stop()
1788 	mv_port_free_dma_mem(ap);  in mv_port_stop()
1802 	struct mv_port_priv *pp = qc->ap->private_data;  in mv_fill_sg()
1852 static void mv_sff_irq_clear(struct ata_port *ap)  in mv_sff_irq_clear()  argument
1854 	mv_clear_and_enable_port_irqs(ap, mv_ap_base(ap), ERR_IRQ);  in mv_sff_irq_clear()
1898 	struct ata_port *ap = qc->ap;  in mv_bmdma_setup()  local
1899 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_bmdma_setup()
1900 	struct mv_port_priv *pp = ap->private_data;  in mv_bmdma_setup()
1914 	ap->ops->sff_exec_command(ap, &qc->tf);  in mv_bmdma_setup()
1926 	struct ata_port *ap = qc->ap;  in mv_bmdma_start()  local
1927 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_bmdma_start()
1944 static void mv_bmdma_stop_ap(struct ata_port *ap)  in mv_bmdma_stop_ap()  argument
1946 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_bmdma_stop_ap()
1956 		ata_sff_dma_pause(ap);  in mv_bmdma_stop_ap()
1962 	mv_bmdma_stop_ap(qc->ap);  in mv_bmdma_stop()
1974 static u8 mv_bmdma_status(struct ata_port *ap)  in mv_bmdma_status()  argument
1976 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_bmdma_status()
1995 		mv_bmdma_stop_ap(ap);  in mv_bmdma_status()
1996 		if (ioread8(ap->ioaddr.altstatus_addr) & ATA_BUSY)  in mv_bmdma_status()
2051 	struct ata_port *ap = qc->ap;  in mv_qc_prep()  local
2052 	struct mv_port_priv *pp = ap->private_data;  in mv_qc_prep()
2152 	struct ata_port *ap = qc->ap;  in mv_qc_prep_iie()  local
2153 	struct mv_port_priv *pp = ap->private_data;  in mv_qc_prep_iie()
2221 static u8 mv_sff_check_status(struct ata_port *ap)  in mv_sff_check_status()  argument
2223 	u8 stat = ioread8(ap->ioaddr.status_addr);  in mv_sff_check_status()
2224 	struct mv_port_priv *pp = ap->private_data;  in mv_sff_check_status()
2240 static unsigned int mv_send_fis(struct ata_port *ap, u32 *fis, int nwords)  in mv_send_fis()  argument
2242 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_send_fis()
2272 		ata_port_warn(ap, "%s transmission error, ifstat=%08x\n",  in mv_send_fis()
2298 	struct ata_port *ap = qc->ap;  in mv_qc_issue_fis()  local
2299 	struct mv_port_priv *pp = ap->private_data;  in mv_qc_issue_fis()
2305 	err = mv_send_fis(ap, fis, ARRAY_SIZE(fis));  in mv_qc_issue_fis()
2314 		ap->hsm_task_state = HSM_ST_FIRST;  in mv_qc_issue_fis()
2319 			ap->hsm_task_state = HSM_ST_FIRST;  in mv_qc_issue_fis()
2321 			ap->hsm_task_state = HSM_ST;  in mv_qc_issue_fis()
2324 		ap->hsm_task_state = HSM_ST_LAST;  in mv_qc_issue_fis()
2348 	struct ata_port *ap = qc->ap;  in mv_qc_issue()  local
2349 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_qc_issue()
2350 	struct mv_port_priv *pp = ap->private_data;  in mv_qc_issue()
2359 			if (!ap->ops->bmdma_setup)  /* no bmdma on GEN_I */  in mv_qc_issue()
2365 		mv_start_edma(ap, port_mmio, pp, qc->tf.protocol);  in mv_qc_issue()
2396 		if (ap->flags & ATA_FLAG_PIO_POLLING)  in mv_qc_issue()
2411 	mv_stop_edma(ap);  in mv_qc_issue()
2412 	mv_clear_and_enable_port_irqs(ap, mv_ap_base(ap), port_irqs);  in mv_qc_issue()
2413 	mv_pmp_select(ap, qc->dev->link->pmp);  in mv_qc_issue()
2416 		struct mv_host_priv *hpriv = ap->host->private_data;  in mv_qc_issue()
2434 static struct ata_queued_cmd *mv_get_active_qc(struct ata_port *ap)  in mv_get_active_qc()  argument
2436 	struct mv_port_priv *pp = ap->private_data;  in mv_get_active_qc()
2441 	qc = ata_qc_from_tag(ap, ap->link.active_tag);  in mv_get_active_qc()
2447 static void mv_pmp_error_handler(struct ata_port *ap)  in mv_pmp_error_handler()  argument
2450 	struct mv_port_priv *pp = ap->private_data;  in mv_pmp_error_handler()
2464 				struct ata_link *link = &ap->pmp_link[pmp];  in mv_pmp_error_handler()
2469 		ata_port_freeze(ap);  in mv_pmp_error_handler()
2471 	sata_pmp_error_handler(ap);  in mv_pmp_error_handler()
2474 static unsigned int mv_get_err_pmp_map(struct ata_port *ap)  in mv_get_err_pmp_map()  argument
2476 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_get_err_pmp_map()
2481 static void mv_pmp_eh_prep(struct ata_port *ap, unsigned int pmp_map)  in mv_pmp_eh_prep()  argument
2489 	ehi = &ap->link.eh_info;  in mv_pmp_eh_prep()
2493 			struct ata_link *link = &ap->pmp_link[pmp];  in mv_pmp_eh_prep()
2506 static int mv_req_q_empty(struct ata_port *ap)  in mv_req_q_empty()  argument
2508 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_req_q_empty()
2518 static int mv_handle_fbs_ncq_dev_err(struct ata_port *ap)  in mv_handle_fbs_ncq_dev_err()  argument
2520 	struct mv_port_priv *pp = ap->private_data;  in mv_handle_fbs_ncq_dev_err()
2537 	new_map = old_map | mv_get_err_pmp_map(ap);  in mv_handle_fbs_ncq_dev_err()
2541 		mv_pmp_eh_prep(ap, new_map & ~old_map);  in mv_handle_fbs_ncq_dev_err()
2545 	ata_port_info(ap,  in mv_handle_fbs_ncq_dev_err()
2548 		      ap->qc_active, failed_links,  in mv_handle_fbs_ncq_dev_err()
2549 		      ap->nr_active_links);  in mv_handle_fbs_ncq_dev_err()
2551 	if (ap->nr_active_links <= failed_links && mv_req_q_empty(ap)) {  in mv_handle_fbs_ncq_dev_err()
2552 		mv_process_crpb_entries(ap, pp);  in mv_handle_fbs_ncq_dev_err()
2553 		mv_stop_edma(ap);  in mv_handle_fbs_ncq_dev_err()
2554 		mv_eh_freeze(ap);  in mv_handle_fbs_ncq_dev_err()
2555 		ata_port_info(ap, "%s: done\n", __func__);  in mv_handle_fbs_ncq_dev_err()
2558 	ata_port_info(ap, "%s: waiting\n", __func__);  in mv_handle_fbs_ncq_dev_err()
2562 static int mv_handle_fbs_non_ncq_dev_err(struct ata_port *ap)  in mv_handle_fbs_non_ncq_dev_err()  argument
2578 static int mv_handle_dev_err(struct ata_port *ap, u32 edma_err_cause)  in mv_handle_dev_err()  argument
2580 	struct mv_port_priv *pp = ap->private_data;  in mv_handle_dev_err()
2600 			ata_port_warn(ap, "%s: err_cause=0x%x pp_flags=0x%x\n",  in mv_handle_dev_err()
2604 		return mv_handle_fbs_ncq_dev_err(ap);  in mv_handle_dev_err()
2612 			ata_port_warn(ap, "%s: err_cause=0x%x pp_flags=0x%x\n",  in mv_handle_dev_err()
2616 		return mv_handle_fbs_non_ncq_dev_err(ap);  in mv_handle_dev_err()
2621 static void mv_unexpected_intr(struct ata_port *ap, int edma_was_enabled)  in mv_unexpected_intr()  argument
2623 	struct ata_eh_info *ehi = &ap->link.eh_info;  in mv_unexpected_intr()
2630 		struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);  in mv_unexpected_intr()
2637 	ata_port_freeze(ap);  in mv_unexpected_intr()
2651 static void mv_err_intr(struct ata_port *ap)  in mv_err_intr()  argument
2653 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_err_intr()
2656 	struct mv_port_priv *pp = ap->private_data;  in mv_err_intr()
2657 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_err_intr()
2659 	struct ata_eh_info *ehi = &ap->link.eh_info;  in mv_err_intr()
2668 	sata_scr_read(&ap->link, SCR_ERROR, &serr);  in mv_err_intr()
2669 	sata_scr_write_flush(&ap->link, SCR_ERROR, serr);  in mv_err_intr()
2683 		if (mv_handle_dev_err(ap, edma_err_cause))  in mv_err_intr()
2687 	qc = mv_get_active_qc(ap);  in mv_err_intr()
2697 			sata_async_notification(ap);  in mv_err_intr()
2767 		mv_eh_freeze(ap);  in mv_err_intr()
2773 		ata_port_freeze(ap);  in mv_err_intr()
2782 			ata_port_abort(ap);  in mv_err_intr()
2786 static bool mv_process_crpb_response(struct ata_port *ap,  in mv_process_crpb_response()  argument
2814 static void mv_process_crpb_entries(struct ata_port *ap, struct mv_port_priv *pp)  in mv_process_crpb_entries()  argument
2816 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_process_crpb_entries()
2817 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_process_crpb_entries()
2836 			tag = ap->link.active_tag;  in mv_process_crpb_entries()
2841 		if (mv_process_crpb_response(ap, response, tag, ncq_enabled))  in mv_process_crpb_entries()
2847 		ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);  in mv_process_crpb_entries()
2856 static void mv_port_intr(struct ata_port *ap, u32 port_cause)  in mv_port_intr()  argument
2866 	pp = ap->private_data;  in mv_port_intr()
2872 		mv_process_crpb_entries(ap, pp);  in mv_port_intr()
2874 			mv_handle_fbs_ncq_dev_err(ap);  in mv_port_intr()
2880 		mv_err_intr(ap);  in mv_port_intr()
2882 		struct ata_queued_cmd *qc = mv_get_active_qc(ap);  in mv_port_intr()
2884 			ata_bmdma_port_intr(ap, qc);  in mv_port_intr()
2886 			mv_unexpected_intr(ap, edma_was_enabled);  in mv_port_intr()
2909 		struct ata_port *ap = host->ports[port];  in mv_host_intr()  local
2958 			mv_port_intr(ap, port_cause);  in mv_host_intr()
2966 	struct ata_port *ap;  in mv_pci_error()  local
2982 		ap = host->ports[i];  in mv_pci_error()
2983 		if (!ata_link_offline(&ap->link)) {  in mv_pci_error()
2984 			ehi = &ap->link.eh_info;  in mv_pci_error()
2991 			qc = ata_qc_from_tag(ap, ap->link.active_tag);  in mv_pci_error()
2997 			ata_port_freeze(ap);  in mv_pci_error()
3072 	struct mv_host_priv *hpriv = link->ap->host->private_data;  in mv5_scr_read()
3074 	void __iomem *addr = mv5_phy_base(mmio, link->ap->port_no);  in mv5_scr_read()
3086 	struct mv_host_priv *hpriv = link->ap->host->private_data;  in mv5_scr_write()
3088 	void __iomem *addr = mv5_phy_base(mmio, link->ap->port_no);  in mv5_scr_write()
3606 static void mv_pmp_select(struct ata_port *ap, int pmp)  in mv_pmp_select()  argument
3608 	if (sata_pmp_supported(ap)) {  in mv_pmp_select()
3609 		void __iomem *port_mmio = mv_ap_base(ap);  in mv_pmp_select()
3623 	mv_pmp_select(link->ap, sata_srst_pmp(link));  in mv_pmp_hardreset()
3630 	mv_pmp_select(link->ap, sata_srst_pmp(link));  in mv_softreset()
3637 	struct ata_port *ap = link->ap;  in mv_hardreset()  local
3638 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_hardreset()
3639 	struct mv_port_priv *pp = ap->private_data;  in mv_hardreset()
3645 	mv_reset_channel(hpriv, mmio, ap->port_no);  in mv_hardreset()
3663 			mv_setup_ifcfg(mv_ap_base(ap), 0);  in mv_hardreset()
3668 	mv_save_cached_regs(ap);  in mv_hardreset()
3669 	mv_edma_cfg(ap, 0, 0);  in mv_hardreset()
3674 static void mv_eh_freeze(struct ata_port *ap)  in mv_eh_freeze()  argument
3676 	mv_stop_edma(ap);  in mv_eh_freeze()
3677 	mv_enable_port_irqs(ap, 0);  in mv_eh_freeze()
3680 static void mv_eh_thaw(struct ata_port *ap)  in mv_eh_thaw()  argument
3682 	struct mv_host_priv *hpriv = ap->host->private_data;  in mv_eh_thaw()
3683 	unsigned int port = ap->port_no;  in mv_eh_thaw()
3686 	void __iomem *port_mmio = mv_ap_base(ap);  in mv_eh_thaw()
3696 	mv_enable_port_irqs(ap, ERR_IRQ);  in mv_eh_thaw()
3977 		struct ata_port *ap = host->ports[port];  in mv_init_host()  local
3980 		mv_port_init(&ap->ioaddr, port_mmio);  in mv_init_host()
4443 		struct ata_port *ap = host->ports[port];  in mv_pci_init_one()  local
4447 		ata_port_pbar_desc(ap, MV_PRIMARY_BAR, -1, "mmio");  in mv_pci_init_one()
4448 		ata_port_pbar_desc(ap, MV_PRIMARY_BAR, offset, "port");  in mv_pci_init_one()