607 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_register_mode()  local
608 	void __iomem *mmio = pp->ctl_block;  in nv_adma_register_mode()
612 	if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)  in nv_adma_register_mode()
640 	pp->flags |= NV_ADMA_PORT_REGISTER_MODE;  in nv_adma_register_mode()
645 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_mode()  local
646 	void __iomem *mmio = pp->ctl_block;  in nv_adma_mode()
650 	if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))  in nv_adma_mode()
653 	WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);  in nv_adma_mode()
670 	pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;  in nv_adma_mode()
676 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_slave_config()  local
728 		pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;  in nv_adma_slave_config()
731 		pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;  in nv_adma_slave_config()
762 		dma_set_mask(&pdev->dev, pp->adma_dma_mask);  in nv_adma_slave_config()
765 					       pp->adma_dma_mask);  in nv_adma_slave_config()
768 					       pp->adma_dma_mask);  in nv_adma_slave_config()
785 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_check_atapi_dma()  local
786 	return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);  in nv_adma_check_atapi_dma()
837 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_check_cpb()  local
838 	u8 flags = pp->cpb[cpb_num].resp_flags;  in nv_adma_check_cpb()
914 		struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_interrupt()  local
915 		void __iomem *mmio = pp->ctl_block;  in nv_adma_interrupt()
923 		if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {  in nv_adma_interrupt()
931 		if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {  in nv_adma_interrupt()
947 		gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);  in nv_adma_interrupt()
1029 		struct nv_adma_port_priv *pp = host->ports[0]->private_data;  in nv_adma_interrupt()  local
1030 		writel(notifier_clears[0], pp->notifier_clear_block);  in nv_adma_interrupt()
1031 		pp = host->ports[1]->private_data;  in nv_adma_interrupt()
1032 		writel(notifier_clears[1], pp->notifier_clear_block);  in nv_adma_interrupt()
1042 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_freeze()  local
1043 	void __iomem *mmio = pp->ctl_block;  in nv_adma_freeze()
1048 	if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)  in nv_adma_freeze()
1064 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_thaw()  local
1065 	void __iomem *mmio = pp->ctl_block;  in nv_adma_thaw()
1070 	if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)  in nv_adma_thaw()
1082 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_irq_clear()  local
1083 	void __iomem *mmio = pp->ctl_block;  in nv_adma_irq_clear()
1086 	if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {  in nv_adma_irq_clear()
1107 	pp = ap->host->ports[0]->private_data;  in nv_adma_irq_clear()
1108 	writel(notifier_clears[0], pp->notifier_clear_block);  in nv_adma_irq_clear()
1109 	pp = ap->host->ports[1]->private_data;  in nv_adma_irq_clear()
1110 	writel(notifier_clears[1], pp->notifier_clear_block);  in nv_adma_irq_clear()
1115 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_post_internal_cmd()  local
1117 	if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)  in nv_adma_post_internal_cmd()
1124 	struct nv_adma_port_priv *pp;  in nv_adma_port_start()  local
1148 	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);  in nv_adma_port_start()
1149 	if (!pp)  in nv_adma_port_start()
1154 	pp->ctl_block = mmio;  in nv_adma_port_start()
1155 	pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;  in nv_adma_port_start()
1156 	pp->notifier_clear_block = pp->gen_block +  in nv_adma_port_start()
1166 	pp->adma_dma_mask = *dev->dma_mask;  in nv_adma_port_start()
1179 	pp->cpb     = mem;  in nv_adma_port_start()
1180 	pp->cpb_dma = mem_dma;  in nv_adma_port_start()
1191 	pp->aprd = mem;  in nv_adma_port_start()
1192 	pp->aprd_dma = mem_dma;  in nv_adma_port_start()
1194 	ap->private_data = pp;  in nv_adma_port_start()
1200 	pp->flags = NV_ADMA_PORT_REGISTER_MODE;  in nv_adma_port_start()
1222 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_port_stop()  local
1223 	void __iomem *mmio = pp->ctl_block;  in nv_adma_port_stop()
1232 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_port_suspend()  local
1233 	void __iomem *mmio = pp->ctl_block;  in nv_adma_port_suspend()
1249 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_port_resume()  local
1250 	void __iomem *mmio = pp->ctl_block;  in nv_adma_port_resume()
1254 	writel(pp->cpb_dma & 0xFFFFFFFF, 	mmio + NV_ADMA_CPB_BASE_LOW);  in nv_adma_port_resume()
1255 	writel((pp->cpb_dma >> 16) >> 16,	mmio + NV_ADMA_CPB_BASE_HIGH);  in nv_adma_port_resume()
1261 	pp->flags |= NV_ADMA_PORT_REGISTER_MODE;  in nv_adma_port_resume()
1350 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_fill_sg()  local
1359 			       &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];  in nv_adma_fill_sg()
1363 		cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));  in nv_adma_fill_sg()
1370 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_use_reg_mode()  local
1374 	if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||  in nv_adma_use_reg_mode()
1387 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_qc_prep()  local
1388 	struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];  in nv_adma_qc_prep()
1393 		BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&  in nv_adma_qc_prep()
1433 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_qc_issue()  local
1434 	void __iomem *mmio = pp->ctl_block;  in nv_adma_qc_issue()
1451 		BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&  in nv_adma_qc_issue()
1462 	if (curr_ncq != pp->last_issue_ncq) {  in nv_adma_qc_issue()
1466 		pp->last_issue_ncq = curr_ncq;  in nv_adma_qc_issue()
1672 	struct nv_adma_port_priv *pp = ap->private_data;  in nv_adma_error_handler()  local
1673 	if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {  in nv_adma_error_handler()
1674 		void __iomem *mmio = pp->ctl_block;  in nv_adma_error_handler()
1681 			u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);  in nv_adma_error_handler()
1694 				struct nv_adma_cpb *cpb = &pp->cpb[i];  in nv_adma_error_handler()
1709 			pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;  in nv_adma_error_handler()
1728 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_qc_to_dq()  local
1729 	struct defer_queue *dq = &pp->defer_queue;  in nv_swncq_qc_to_dq()
1739 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_qc_from_dq()  local
1740 	struct defer_queue *dq = &pp->defer_queue;  in nv_swncq_qc_from_dq()
1756 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_fis_reinit()  local
1758 	pp->dhfis_bits = 0;  in nv_swncq_fis_reinit()
1759 	pp->dmafis_bits = 0;  in nv_swncq_fis_reinit()
1760 	pp->sdbfis_bits = 0;  in nv_swncq_fis_reinit()
1761 	pp->ncq_flags = 0;  in nv_swncq_fis_reinit()
1766 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_pp_reinit()  local
1767 	struct defer_queue *dq = &pp->defer_queue;  in nv_swncq_pp_reinit()
1772 	pp->qc_active = 0;  in nv_swncq_pp_reinit()
1773 	pp->last_issue_tag = ATA_TAG_POISON;  in nv_swncq_pp_reinit()
1779 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_irq_clear()  local
1781 	writew(fis, pp->irq_block);  in nv_swncq_irq_clear()
1794 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_ncq_stop()  local
1804 		pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,  in nv_swncq_ncq_stop()
1805 		pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);  in nv_swncq_ncq_stop()
1811 	sactive = readl(pp->sactive_block);  in nv_swncq_ncq_stop()
1812 	done_mask = pp->qc_active ^ sactive;  in nv_swncq_ncq_stop()
1817 		if (pp->qc_active & (1 << i))  in nv_swncq_ncq_stop()
1826 			     (pp->dhfis_bits >> i) & 0x1,  in nv_swncq_ncq_stop()
1827 			     (pp->dmafis_bits >> i) & 0x1,  in nv_swncq_ncq_stop()
1828 			     (pp->sdbfis_bits >> i) & 0x1,  in nv_swncq_ncq_stop()
1966 	struct nv_swncq_port_priv *pp;  in nv_swncq_port_start()  local
1974 	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);  in nv_swncq_port_start()
1975 	if (!pp)  in nv_swncq_port_start()
1978 	pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,  in nv_swncq_port_start()
1979 				      &pp->prd_dma, GFP_KERNEL);  in nv_swncq_port_start()
1980 	if (!pp->prd)  in nv_swncq_port_start()
1982 	memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);  in nv_swncq_port_start()
1984 	ap->private_data = pp;  in nv_swncq_port_start()
1985 	pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;  in nv_swncq_port_start()
1986 	pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;  in nv_swncq_port_start()
1987 	pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;  in nv_swncq_port_start()
2009 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_fill_sg()  local
2013 	prd = pp->prd + ATA_MAX_PRD * qc->tag;  in nv_swncq_fill_sg()
2044 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_issue_atacmd()  local
2051 	writel((1 << qc->tag), pp->sactive_block);  in nv_swncq_issue_atacmd()
2052 	pp->last_issue_tag = qc->tag;  in nv_swncq_issue_atacmd()
2053 	pp->dhfis_bits &= ~(1 << qc->tag);  in nv_swncq_issue_atacmd()
2054 	pp->dmafis_bits &= ~(1 << qc->tag);  in nv_swncq_issue_atacmd()
2055 	pp->qc_active |= (0x1 << qc->tag);  in nv_swncq_issue_atacmd()
2068 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_qc_issue()  local
2075 	if (!pp->qc_active)  in nv_swncq_qc_issue()
2111 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_sdbfis()  local
2131 	sactive = readl(pp->sactive_block);  in nv_swncq_sdbfis()
2132 	done_mask = pp->qc_active ^ sactive;  in nv_swncq_sdbfis()
2134 	pp->qc_active &= ~done_mask;  in nv_swncq_sdbfis()
2135 	pp->dhfis_bits &= ~done_mask;  in nv_swncq_sdbfis()
2136 	pp->dmafis_bits &= ~done_mask;  in nv_swncq_sdbfis()
2137 	pp->sdbfis_bits |= done_mask;  in nv_swncq_sdbfis()
2146 	if (pp->qc_active & pp->dhfis_bits)  in nv_swncq_sdbfis()
2149 	if ((pp->ncq_flags & ncq_saw_backout) ||  in nv_swncq_sdbfis()
2150 	    (pp->qc_active ^ pp->dhfis_bits))  in nv_swncq_sdbfis()
2159 		ap->print_id, ap->qc_active, pp->qc_active,  in nv_swncq_sdbfis()
2160 		pp->defer_queue.defer_bits, pp->dhfis_bits,  in nv_swncq_sdbfis()
2161 		pp->dmafis_bits, pp->last_issue_tag);  in nv_swncq_sdbfis()
2166 		qc = ata_qc_from_tag(ap, pp->last_issue_tag);  in nv_swncq_sdbfis()
2171 	if (pp->defer_queue.defer_bits) {  in nv_swncq_sdbfis()
2183 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_tag()  local
2186 	tag = readb(pp->tag_block) >> 2;  in nv_swncq_tag()
2196 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_dmafis()  local
2210 	iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,  in nv_swncq_dmafis()
2224 	struct nv_swncq_port_priv *pp = ap->private_data;  in nv_swncq_host_interrupt()  local
2243 	if (!pp->qc_active)  in nv_swncq_host_interrupt()
2264 		pp->ncq_flags |= ncq_saw_backout;  in nv_swncq_host_interrupt()
2268 		pp->ncq_flags |= ncq_saw_sdb;  in nv_swncq_host_interrupt()
2271 			ap->print_id, pp->qc_active, pp->dhfis_bits,  in nv_swncq_host_interrupt()
2272 			pp->dmafis_bits, readl(pp->sactive_block));  in nv_swncq_host_interrupt()
2281 		pp->dhfis_bits |= (0x1 << pp->last_issue_tag);  in nv_swncq_host_interrupt()
2282 		pp->ncq_flags |= ncq_saw_d2h;  in nv_swncq_host_interrupt()
2283 		if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {  in nv_swncq_host_interrupt()
2291 		    !(pp->ncq_flags & ncq_saw_dmas)) {  in nv_swncq_host_interrupt()
2296 			if (pp->defer_queue.defer_bits) {  in nv_swncq_host_interrupt()
2308 		pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));  in nv_swncq_host_interrupt()
2309 		pp->ncq_flags |= ncq_saw_dmas;  in nv_swncq_host_interrupt()
2455 			struct nv_adma_port_priv *pp;  in nv_pci_device_resume()  local
2459 			pp = host->ports[0]->private_data;  in nv_pci_device_resume()
2460 			if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)  in nv_pci_device_resume()
2466 			pp = host->ports[1]->private_data;  in nv_pci_device_resume()
2467 			if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)  in nv_pci_device_resume()