112 static inline u32 spfi_readl(struct img_spfi *spfi, u32 reg)  in spfi_readl()  argument
114 	return readl(spfi->regs + reg);  in spfi_readl()
117 static inline void spfi_writel(struct img_spfi *spfi, u32 val, u32 reg)  in spfi_writel()  argument
119 	writel(val, spfi->regs + reg);  in spfi_writel()
122 static inline void spfi_start(struct img_spfi *spfi)  in spfi_start()  argument
126 	val = spfi_readl(spfi, SPFI_CONTROL);  in spfi_start()
128 	spfi_writel(spfi, val, SPFI_CONTROL);  in spfi_start()
131 static inline void spfi_reset(struct img_spfi *spfi)  in spfi_reset()  argument
133 	spfi_writel(spfi, SPFI_CONTROL_SOFT_RESET, SPFI_CONTROL);  in spfi_reset()
134 	spfi_writel(spfi, 0, SPFI_CONTROL);  in spfi_reset()
137 static int spfi_wait_all_done(struct img_spfi *spfi)  in spfi_wait_all_done()  argument
142 		u32 status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);  in spfi_wait_all_done()
145 			spfi_writel(spfi, SPFI_INTERRUPT_ALLDONETRIG,  in spfi_wait_all_done()
152 	dev_err(spfi->dev, "Timed out waiting for transaction to complete\n");  in spfi_wait_all_done()
153 	spfi_reset(spfi);  in spfi_wait_all_done()
158 static unsigned int spfi_pio_write32(struct img_spfi *spfi, const u32 *buf,  in spfi_pio_write32()  argument
165 		spfi_writel(spfi, SPFI_INTERRUPT_SDFUL, SPFI_INTERRUPT_CLEAR);  in spfi_pio_write32()
166 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);  in spfi_pio_write32()
169 		spfi_writel(spfi, buf[count], SPFI_TX_32BIT_VALID_DATA);  in spfi_pio_write32()
176 static unsigned int spfi_pio_write8(struct img_spfi *spfi, const u8 *buf,  in spfi_pio_write8()  argument
183 		spfi_writel(spfi, SPFI_INTERRUPT_SDFUL, SPFI_INTERRUPT_CLEAR);  in spfi_pio_write8()
184 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);  in spfi_pio_write8()
187 		spfi_writel(spfi, buf[count], SPFI_TX_8BIT_VALID_DATA);  in spfi_pio_write8()
194 static unsigned int spfi_pio_read32(struct img_spfi *spfi, u32 *buf,  in spfi_pio_read32()  argument
201 		spfi_writel(spfi, SPFI_INTERRUPT_GDEX32BIT,  in spfi_pio_read32()
203 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);  in spfi_pio_read32()
206 		buf[count] = spfi_readl(spfi, SPFI_RX_32BIT_VALID_DATA);  in spfi_pio_read32()
213 static unsigned int spfi_pio_read8(struct img_spfi *spfi, u8 *buf,  in spfi_pio_read8()  argument
220 		spfi_writel(spfi, SPFI_INTERRUPT_GDEX8BIT,  in spfi_pio_read8()
222 		status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);  in spfi_pio_read8()
225 		buf[count] = spfi_readl(spfi, SPFI_RX_8BIT_VALID_DATA);  in spfi_pio_read8()
236 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);  in img_spfi_start_pio()  local
248 	spfi_start(spfi);  in img_spfi_start_pio()
257 			tx_count = spfi_pio_write32(spfi, tx_buf, tx_bytes);  in img_spfi_start_pio()
259 			tx_count = spfi_pio_write8(spfi, tx_buf, tx_bytes);  in img_spfi_start_pio()
262 			rx_count = spfi_pio_read32(spfi, rx_buf, rx_bytes);  in img_spfi_start_pio()
264 			rx_count = spfi_pio_read8(spfi, rx_buf, rx_bytes);  in img_spfi_start_pio()
275 		dev_err(spfi->dev, "PIO transfer timed out\n");  in img_spfi_start_pio()
279 	ret = spfi_wait_all_done(spfi);  in img_spfi_start_pio()
288 	struct img_spfi *spfi = data;  in img_spfi_dma_rx_cb()  local
291 	spfi_wait_all_done(spfi);  in img_spfi_dma_rx_cb()
293 	spin_lock_irqsave(&spfi->lock, flags);  in img_spfi_dma_rx_cb()
294 	spfi->rx_dma_busy = false;  in img_spfi_dma_rx_cb()
295 	if (!spfi->tx_dma_busy)  in img_spfi_dma_rx_cb()
296 		spi_finalize_current_transfer(spfi->master);  in img_spfi_dma_rx_cb()
297 	spin_unlock_irqrestore(&spfi->lock, flags);  in img_spfi_dma_rx_cb()
302 	struct img_spfi *spfi = data;  in img_spfi_dma_tx_cb()  local
305 	spfi_wait_all_done(spfi);  in img_spfi_dma_tx_cb()
307 	spin_lock_irqsave(&spfi->lock, flags);  in img_spfi_dma_tx_cb()
308 	spfi->tx_dma_busy = false;  in img_spfi_dma_tx_cb()
309 	if (!spfi->rx_dma_busy)  in img_spfi_dma_tx_cb()
310 		spi_finalize_current_transfer(spfi->master);  in img_spfi_dma_tx_cb()
311 	spin_unlock_irqrestore(&spfi->lock, flags);  in img_spfi_dma_tx_cb()
318 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);  in img_spfi_start_dma()  local
322 	spfi->rx_dma_busy = false;  in img_spfi_start_dma()
323 	spfi->tx_dma_busy = false;  in img_spfi_start_dma()
328 			rxconf.src_addr = spfi->phys + SPFI_RX_32BIT_VALID_DATA;  in img_spfi_start_dma()
332 			rxconf.src_addr = spfi->phys + SPFI_RX_8BIT_VALID_DATA;  in img_spfi_start_dma()
336 		dmaengine_slave_config(spfi->rx_ch, &rxconf);  in img_spfi_start_dma()
338 		rxdesc = dmaengine_prep_slave_sg(spfi->rx_ch, xfer->rx_sg.sgl,  in img_spfi_start_dma()
346 		rxdesc->callback_param = spfi;  in img_spfi_start_dma()
352 			txconf.dst_addr = spfi->phys + SPFI_TX_32BIT_VALID_DATA;  in img_spfi_start_dma()
356 			txconf.dst_addr = spfi->phys + SPFI_TX_8BIT_VALID_DATA;  in img_spfi_start_dma()
360 		dmaengine_slave_config(spfi->tx_ch, &txconf);  in img_spfi_start_dma()
362 		txdesc = dmaengine_prep_slave_sg(spfi->tx_ch, xfer->tx_sg.sgl,  in img_spfi_start_dma()
370 		txdesc->callback_param = spfi;  in img_spfi_start_dma()
374 		spfi->rx_dma_busy = true;  in img_spfi_start_dma()
376 		dma_async_issue_pending(spfi->rx_ch);  in img_spfi_start_dma()
379 	spfi_start(spfi);  in img_spfi_start_dma()
382 		spfi->tx_dma_busy = true;  in img_spfi_start_dma()
384 		dma_async_issue_pending(spfi->tx_ch);  in img_spfi_start_dma()
390 	dmaengine_terminate_all(spfi->rx_ch);  in img_spfi_start_dma()
391 	dmaengine_terminate_all(spfi->tx_ch);  in img_spfi_start_dma()
398 	struct img_spfi *spfi = spi_master_get_devdata(master);  in img_spfi_handle_err()  local
405 	spin_lock_irqsave(&spfi->lock, flags);  in img_spfi_handle_err()
406 	if (spfi->tx_dma_busy || spfi->rx_dma_busy) {  in img_spfi_handle_err()
407 		spfi->tx_dma_busy = false;  in img_spfi_handle_err()
408 		spfi->rx_dma_busy = false;  in img_spfi_handle_err()
410 		dmaengine_terminate_all(spfi->tx_ch);  in img_spfi_handle_err()
411 		dmaengine_terminate_all(spfi->rx_ch);  in img_spfi_handle_err()
413 	spin_unlock_irqrestore(&spfi->lock, flags);  in img_spfi_handle_err()
418 	struct img_spfi *spfi = spi_master_get_devdata(master);  in img_spfi_prepare()  local
421 	val = spfi_readl(spfi, SPFI_PORT_STATE);  in img_spfi_prepare()
430 	spfi_writel(spfi, val, SPFI_PORT_STATE);  in img_spfi_prepare()
438 	struct img_spfi *spfi = spi_master_get_devdata(master);  in img_spfi_unprepare()  local
440 	spfi_reset(spfi);  in img_spfi_unprepare()
496 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);  in img_spfi_config()  local
503 	div = DIV_ROUND_UP(clk_get_rate(spfi->spfi_clk), xfer->speed_hz);  in img_spfi_config()
506 	val = spfi_readl(spfi, SPFI_DEVICE_PARAMETER(spi->chip_select));  in img_spfi_config()
510 	spfi_writel(spfi, val, SPFI_DEVICE_PARAMETER(spi->chip_select));  in img_spfi_config()
512 	spfi_writel(spfi, xfer->len << SPFI_TRANSACTION_TSIZE_SHIFT,  in img_spfi_config()
515 	val = spfi_readl(spfi, SPFI_CONTROL);  in img_spfi_config()
529 	spfi_writel(spfi, val, SPFI_CONTROL);  in img_spfi_config()
536 	struct img_spfi *spfi = spi_master_get_devdata(spi->master);  in img_spfi_transfer_one()  local
540 		dev_err(spfi->dev,  in img_spfi_transfer_one()
565 	struct img_spfi *spfi = (struct img_spfi *)dev_id;  in img_spfi_irq()  local
568 	status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS);  in img_spfi_irq()
570 		spfi_writel(spfi, SPFI_INTERRUPT_IACCESS, SPFI_INTERRUPT_CLEAR);  in img_spfi_irq()
571 		dev_err(spfi->dev, "Illegal access interrupt");  in img_spfi_irq()
581 	struct img_spfi *spfi;  in img_spfi_probe()  local
585 	master = spi_alloc_master(&pdev->dev, sizeof(*spfi));  in img_spfi_probe()
590 	spfi = spi_master_get_devdata(master);  in img_spfi_probe()
591 	spfi->dev = &pdev->dev;  in img_spfi_probe()
592 	spfi->master = master;  in img_spfi_probe()
593 	spin_lock_init(&spfi->lock);  in img_spfi_probe()
596 	spfi->regs = devm_ioremap_resource(spfi->dev, res);  in img_spfi_probe()
597 	if (IS_ERR(spfi->regs)) {  in img_spfi_probe()
598 		ret = PTR_ERR(spfi->regs);  in img_spfi_probe()
601 	spfi->phys = res->start;  in img_spfi_probe()
603 	spfi->irq = platform_get_irq(pdev, 0);  in img_spfi_probe()
604 	if (spfi->irq < 0) {  in img_spfi_probe()
605 		ret = spfi->irq;  in img_spfi_probe()
608 	ret = devm_request_irq(spfi->dev, spfi->irq, img_spfi_irq,  in img_spfi_probe()
609 			       IRQ_TYPE_LEVEL_HIGH, dev_name(spfi->dev), spfi);  in img_spfi_probe()
613 	spfi->sys_clk = devm_clk_get(spfi->dev, "sys");  in img_spfi_probe()
614 	if (IS_ERR(spfi->sys_clk)) {  in img_spfi_probe()
615 		ret = PTR_ERR(spfi->sys_clk);  in img_spfi_probe()
618 	spfi->spfi_clk = devm_clk_get(spfi->dev, "spfi");  in img_spfi_probe()
619 	if (IS_ERR(spfi->spfi_clk)) {  in img_spfi_probe()
620 		ret = PTR_ERR(spfi->spfi_clk);  in img_spfi_probe()
624 	ret = clk_prepare_enable(spfi->sys_clk);  in img_spfi_probe()
627 	ret = clk_prepare_enable(spfi->spfi_clk);  in img_spfi_probe()
631 	spfi_reset(spfi);  in img_spfi_probe()
636 	spfi_writel(spfi, SPFI_INTERRUPT_IACCESS, SPFI_INTERRUPT_ENABLE);  in img_spfi_probe()
641 	if (of_property_read_bool(spfi->dev->of_node, "img,supports-quad-mode"))  in img_spfi_probe()
645 	master->max_speed_hz = clk_get_rate(spfi->spfi_clk) / 4;  in img_spfi_probe()
646 	master->min_speed_hz = clk_get_rate(spfi->spfi_clk) / 512;  in img_spfi_probe()
655 	spfi->tx_ch = dma_request_slave_channel(spfi->dev, "tx");  in img_spfi_probe()
656 	spfi->rx_ch = dma_request_slave_channel(spfi->dev, "rx");  in img_spfi_probe()
657 	if (!spfi->tx_ch || !spfi->rx_ch) {  in img_spfi_probe()
658 		if (spfi->tx_ch)  in img_spfi_probe()
659 			dma_release_channel(spfi->tx_ch);  in img_spfi_probe()
660 		if (spfi->rx_ch)  in img_spfi_probe()
661 			dma_release_channel(spfi->rx_ch);  in img_spfi_probe()
662 		dev_warn(spfi->dev, "Failed to get DMA channels, falling back to PIO mode\n");  in img_spfi_probe()
664 		master->dma_tx = spfi->tx_ch;  in img_spfi_probe()
665 		master->dma_rx = spfi->rx_ch;  in img_spfi_probe()
669 	pm_runtime_set_active(spfi->dev);  in img_spfi_probe()
670 	pm_runtime_enable(spfi->dev);  in img_spfi_probe()
672 	ret = devm_spi_register_master(spfi->dev, master);  in img_spfi_probe()
679 	pm_runtime_disable(spfi->dev);  in img_spfi_probe()
680 	if (spfi->rx_ch)  in img_spfi_probe()
681 		dma_release_channel(spfi->rx_ch);  in img_spfi_probe()
682 	if (spfi->tx_ch)  in img_spfi_probe()
683 		dma_release_channel(spfi->tx_ch);  in img_spfi_probe()
684 	clk_disable_unprepare(spfi->spfi_clk);  in img_spfi_probe()
686 	clk_disable_unprepare(spfi->sys_clk);  in img_spfi_probe()
696 	struct img_spfi *spfi = spi_master_get_devdata(master);  in img_spfi_remove()  local
698 	if (spfi->tx_ch)  in img_spfi_remove()
699 		dma_release_channel(spfi->tx_ch);  in img_spfi_remove()
700 	if (spfi->rx_ch)  in img_spfi_remove()
701 		dma_release_channel(spfi->rx_ch);  in img_spfi_remove()
703 	pm_runtime_disable(spfi->dev);  in img_spfi_remove()
704 	if (!pm_runtime_status_suspended(spfi->dev)) {  in img_spfi_remove()
705 		clk_disable_unprepare(spfi->spfi_clk);  in img_spfi_remove()
706 		clk_disable_unprepare(spfi->sys_clk);  in img_spfi_remove()
718 	struct img_spfi *spfi = spi_master_get_devdata(master);  in img_spfi_runtime_suspend()  local
720 	clk_disable_unprepare(spfi->spfi_clk);  in img_spfi_runtime_suspend()
721 	clk_disable_unprepare(spfi->sys_clk);  in img_spfi_runtime_suspend()
729 	struct img_spfi *spfi = spi_master_get_devdata(master);  in img_spfi_runtime_resume()  local
732 	ret = clk_prepare_enable(spfi->sys_clk);  in img_spfi_runtime_resume()
735 	ret = clk_prepare_enable(spfi->spfi_clk);  in img_spfi_runtime_resume()
737 		clk_disable_unprepare(spfi->sys_clk);  in img_spfi_runtime_resume()
756 	struct img_spfi *spfi = spi_master_get_devdata(master);  in img_spfi_resume()  local
762 	spfi_reset(spfi);  in img_spfi_resume()