204 static void rspi_write8(const struct rspi_data *rspi, u8 data, u16 offset)  in rspi_write8()  argument
206 	iowrite8(data, rspi->addr + offset);  in rspi_write8()
209 static void rspi_write16(const struct rspi_data *rspi, u16 data, u16 offset)  in rspi_write16()  argument
211 	iowrite16(data, rspi->addr + offset);  in rspi_write16()
214 static void rspi_write32(const struct rspi_data *rspi, u32 data, u16 offset)  in rspi_write32()  argument
216 	iowrite32(data, rspi->addr + offset);  in rspi_write32()
219 static u8 rspi_read8(const struct rspi_data *rspi, u16 offset)  in rspi_read8()  argument
221 	return ioread8(rspi->addr + offset);  in rspi_read8()
224 static u16 rspi_read16(const struct rspi_data *rspi, u16 offset)  in rspi_read16()  argument
226 	return ioread16(rspi->addr + offset);  in rspi_read16()
229 static void rspi_write_data(const struct rspi_data *rspi, u16 data)  in rspi_write_data()  argument
231 	if (rspi->byte_access)  in rspi_write_data()
232 		rspi_write8(rspi, data, RSPI_SPDR);  in rspi_write_data()
234 		rspi_write16(rspi, data, RSPI_SPDR);  in rspi_write_data()
237 static u16 rspi_read_data(const struct rspi_data *rspi)  in rspi_read_data()  argument
239 	if (rspi->byte_access)  in rspi_read_data()
240 		return rspi_read8(rspi, RSPI_SPDR);  in rspi_read_data()
242 		return rspi_read16(rspi, RSPI_SPDR);  in rspi_read_data()
247 	int (*set_config_register)(struct rspi_data *rspi, int access_size);
258 static int rspi_set_config_register(struct rspi_data *rspi, int access_size)  in rspi_set_config_register()  argument
263 	rspi_write8(rspi, rspi->sppcr, RSPI_SPPCR);  in rspi_set_config_register()
266 	spbr = DIV_ROUND_UP(clk_get_rate(rspi->clk),  in rspi_set_config_register()
267 			    2 * rspi->max_speed_hz) - 1;  in rspi_set_config_register()
268 	rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);  in rspi_set_config_register()
271 	rspi_write8(rspi, 0, RSPI_SPDCR);  in rspi_set_config_register()
272 	rspi->byte_access = 0;  in rspi_set_config_register()
275 	rspi_write8(rspi, 0x00, RSPI_SPCKD);  in rspi_set_config_register()
276 	rspi_write8(rspi, 0x00, RSPI_SSLND);  in rspi_set_config_register()
277 	rspi_write8(rspi, 0x00, RSPI_SPND);  in rspi_set_config_register()
280 	rspi_write8(rspi, 0x00, RSPI_SPCR2);  in rspi_set_config_register()
283 	rspi->spcmd |= SPCMD_SPB_8_TO_16(access_size);  in rspi_set_config_register()
284 	rspi_write16(rspi, rspi->spcmd, RSPI_SPCMD0);  in rspi_set_config_register()
287 	rspi_write8(rspi, SPCR_MSTR, RSPI_SPCR);  in rspi_set_config_register()
295 static int rspi_rz_set_config_register(struct rspi_data *rspi, int access_size)  in rspi_rz_set_config_register()  argument
300 	rspi_write8(rspi, rspi->sppcr, RSPI_SPPCR);  in rspi_rz_set_config_register()
303 	spbr = DIV_ROUND_UP(clk_get_rate(rspi->clk),  in rspi_rz_set_config_register()
304 			    2 * rspi->max_speed_hz) - 1;  in rspi_rz_set_config_register()
305 	rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);  in rspi_rz_set_config_register()
308 	rspi_write8(rspi, SPDCR_SPLBYTE, RSPI_SPDCR);  in rspi_rz_set_config_register()
309 	rspi->byte_access = 1;  in rspi_rz_set_config_register()
312 	rspi_write8(rspi, 0x00, RSPI_SPCKD);  in rspi_rz_set_config_register()
313 	rspi_write8(rspi, 0x00, RSPI_SSLND);  in rspi_rz_set_config_register()
314 	rspi_write8(rspi, 0x00, RSPI_SPND);  in rspi_rz_set_config_register()
317 	rspi->spcmd |= SPCMD_SPB_8_TO_16(access_size);  in rspi_rz_set_config_register()
318 	rspi_write16(rspi, rspi->spcmd, RSPI_SPCMD0);  in rspi_rz_set_config_register()
321 	rspi_write8(rspi, SPCR_MSTR, RSPI_SPCR);  in rspi_rz_set_config_register()
329 static int qspi_set_config_register(struct rspi_data *rspi, int access_size)  in qspi_set_config_register()  argument
334 	rspi_write8(rspi, rspi->sppcr, RSPI_SPPCR);  in qspi_set_config_register()
337 	spbr = DIV_ROUND_UP(clk_get_rate(rspi->clk), 2 * rspi->max_speed_hz);  in qspi_set_config_register()
338 	rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);  in qspi_set_config_register()
341 	rspi_write8(rspi, 0, RSPI_SPDCR);  in qspi_set_config_register()
342 	rspi->byte_access = 1;  in qspi_set_config_register()
345 	rspi_write8(rspi, 0x00, RSPI_SPCKD);  in qspi_set_config_register()
346 	rspi_write8(rspi, 0x00, RSPI_SSLND);  in qspi_set_config_register()
347 	rspi_write8(rspi, 0x00, RSPI_SPND);  in qspi_set_config_register()
351 		rspi->spcmd |= SPCMD_SPB_8BIT;  in qspi_set_config_register()
353 		rspi->spcmd |= SPCMD_SPB_16BIT;  in qspi_set_config_register()
355 		rspi->spcmd |= SPCMD_SPB_32BIT;  in qspi_set_config_register()
357 	rspi->spcmd |= SPCMD_SCKDEN | SPCMD_SLNDEN | SPCMD_SPNDEN;  in qspi_set_config_register()
360 	rspi_write32(rspi, 0, QSPI_SPBMUL0);  in qspi_set_config_register()
363 	rspi_write8(rspi, SPBFCR_TXRST | SPBFCR_RXRST, QSPI_SPBFCR);  in qspi_set_config_register()
365 	rspi_write8(rspi, 0x00, QSPI_SPBFCR);  in qspi_set_config_register()
368 	rspi_write16(rspi, rspi->spcmd, RSPI_SPCMD0);  in qspi_set_config_register()
371 	rspi_write8(rspi, SPCR_SPE | SPCR_MSTR, RSPI_SPCR);  in qspi_set_config_register()
376 static void qspi_update(const struct rspi_data *rspi, u8 mask, u8 val, u8 reg)  in qspi_update()  argument
380 	data = rspi_read8(rspi, reg);  in qspi_update()
383 	rspi_write8(rspi, data, reg);  in qspi_update()
386 static int qspi_set_send_trigger(struct rspi_data *rspi, unsigned int len)  in qspi_set_send_trigger()  argument
394 		qspi_update(rspi, SPBFCR_TXTRG_MASK,  in qspi_set_send_trigger()
398 		qspi_update(rspi, SPBFCR_TXTRG_MASK,  in qspi_set_send_trigger()
405 static void qspi_set_receive_trigger(struct rspi_data *rspi, unsigned int len)  in qspi_set_receive_trigger()  argument
413 		qspi_update(rspi, SPBFCR_RXTRG_MASK,  in qspi_set_receive_trigger()
417 		qspi_update(rspi, SPBFCR_RXTRG_MASK,  in qspi_set_receive_trigger()
424 static void rspi_enable_irq(const struct rspi_data *rspi, u8 enable)  in rspi_enable_irq()  argument
426 	rspi_write8(rspi, rspi_read8(rspi, RSPI_SPCR) | enable, RSPI_SPCR);  in rspi_enable_irq()
429 static void rspi_disable_irq(const struct rspi_data *rspi, u8 disable)  in rspi_disable_irq()  argument
431 	rspi_write8(rspi, rspi_read8(rspi, RSPI_SPCR) & ~disable, RSPI_SPCR);  in rspi_disable_irq()
434 static int rspi_wait_for_interrupt(struct rspi_data *rspi, u8 wait_mask,  in rspi_wait_for_interrupt()  argument
439 	rspi->spsr = rspi_read8(rspi, RSPI_SPSR);  in rspi_wait_for_interrupt()
440 	if (rspi->spsr & wait_mask)  in rspi_wait_for_interrupt()
443 	rspi_enable_irq(rspi, enable_bit);  in rspi_wait_for_interrupt()
444 	ret = wait_event_timeout(rspi->wait, rspi->spsr & wait_mask, HZ);  in rspi_wait_for_interrupt()
445 	if (ret == 0 && !(rspi->spsr & wait_mask))  in rspi_wait_for_interrupt()
451 static inline int rspi_wait_for_tx_empty(struct rspi_data *rspi)  in rspi_wait_for_tx_empty()  argument
453 	return rspi_wait_for_interrupt(rspi, SPSR_SPTEF, SPCR_SPTIE);  in rspi_wait_for_tx_empty()
456 static inline int rspi_wait_for_rx_full(struct rspi_data *rspi)  in rspi_wait_for_rx_full()  argument
458 	return rspi_wait_for_interrupt(rspi, SPSR_SPRF, SPCR_SPRIE);  in rspi_wait_for_rx_full()
461 static int rspi_data_out(struct rspi_data *rspi, u8 data)  in rspi_data_out()  argument
463 	int error = rspi_wait_for_tx_empty(rspi);  in rspi_data_out()
465 		dev_err(&rspi->master->dev, "transmit timeout\n");  in rspi_data_out()
468 	rspi_write_data(rspi, data);  in rspi_data_out()
472 static int rspi_data_in(struct rspi_data *rspi)  in rspi_data_in()  argument
477 	error = rspi_wait_for_rx_full(rspi);  in rspi_data_in()
479 		dev_err(&rspi->master->dev, "receive timeout\n");  in rspi_data_in()
482 	data = rspi_read_data(rspi);  in rspi_data_in()
486 static int rspi_pio_transfer(struct rspi_data *rspi, const u8 *tx, u8 *rx,  in rspi_pio_transfer()  argument
491 			int ret = rspi_data_out(rspi, *tx++);  in rspi_pio_transfer()
496 			int ret = rspi_data_in(rspi);  in rspi_pio_transfer()
508 	struct rspi_data *rspi = arg;  in rspi_dma_complete()  local
510 	rspi->dma_callbacked = 1;  in rspi_dma_complete()
511 	wake_up_interruptible(&rspi->wait);  in rspi_dma_complete()
514 static int rspi_dma_transfer(struct rspi_data *rspi, struct sg_table *tx,  in rspi_dma_transfer()  argument
525 		desc_rx = dmaengine_prep_slave_sg(rspi->master->dma_rx,  in rspi_dma_transfer()
534 		desc_rx->callback_param = rspi;  in rspi_dma_transfer()
545 		desc_tx = dmaengine_prep_slave_sg(rspi->master->dma_tx,  in rspi_dma_transfer()
558 			desc_tx->callback_param = rspi;  in rspi_dma_transfer()
574 		disable_irq(other_irq = rspi->tx_irq);  in rspi_dma_transfer()
575 	if (rx && rspi->rx_irq != other_irq)  in rspi_dma_transfer()
576 		disable_irq(rspi->rx_irq);  in rspi_dma_transfer()
578 	rspi_enable_irq(rspi, irq_mask);  in rspi_dma_transfer()
579 	rspi->dma_callbacked = 0;  in rspi_dma_transfer()
583 		dma_async_issue_pending(rspi->master->dma_rx);  in rspi_dma_transfer()
585 		dma_async_issue_pending(rspi->master->dma_tx);  in rspi_dma_transfer()
587 	ret = wait_event_interruptible_timeout(rspi->wait,  in rspi_dma_transfer()
588 					       rspi->dma_callbacked, HZ);  in rspi_dma_transfer()
589 	if (ret > 0 && rspi->dma_callbacked)  in rspi_dma_transfer()
592 		dev_err(&rspi->master->dev, "DMA timeout\n");  in rspi_dma_transfer()
595 			dmaengine_terminate_all(rspi->master->dma_tx);  in rspi_dma_transfer()
597 			dmaengine_terminate_all(rspi->master->dma_rx);  in rspi_dma_transfer()
600 	rspi_disable_irq(rspi, irq_mask);  in rspi_dma_transfer()
603 		enable_irq(rspi->tx_irq);  in rspi_dma_transfer()
604 	if (rx && rspi->rx_irq != other_irq)  in rspi_dma_transfer()
605 		enable_irq(rspi->rx_irq);  in rspi_dma_transfer()
611 		dmaengine_terminate_all(rspi->master->dma_rx);  in rspi_dma_transfer()
615 			     dev_driver_string(&rspi->master->dev),  in rspi_dma_transfer()
616 			     dev_name(&rspi->master->dev));  in rspi_dma_transfer()
621 static void rspi_receive_init(const struct rspi_data *rspi)  in rspi_receive_init()  argument
625 	spsr = rspi_read8(rspi, RSPI_SPSR);  in rspi_receive_init()
627 		rspi_read_data(rspi);	/* dummy read */  in rspi_receive_init()
629 		rspi_write8(rspi, rspi_read8(rspi, RSPI_SPSR) & ~SPSR_OVRF,  in rspi_receive_init()
633 static void rspi_rz_receive_init(const struct rspi_data *rspi)  in rspi_rz_receive_init()  argument
635 	rspi_receive_init(rspi);  in rspi_rz_receive_init()
636 	rspi_write8(rspi, SPBFCR_TXRST | SPBFCR_RXRST, RSPI_SPBFCR);  in rspi_rz_receive_init()
637 	rspi_write8(rspi, 0, RSPI_SPBFCR);  in rspi_rz_receive_init()
640 static void qspi_receive_init(const struct rspi_data *rspi)  in qspi_receive_init()  argument
644 	spsr = rspi_read8(rspi, RSPI_SPSR);  in qspi_receive_init()
646 		rspi_read_data(rspi);   /* dummy read */  in qspi_receive_init()
647 	rspi_write8(rspi, SPBFCR_TXRST | SPBFCR_RXRST, QSPI_SPBFCR);  in qspi_receive_init()
648 	rspi_write8(rspi, 0, QSPI_SPBFCR);  in qspi_receive_init()
651 static bool __rspi_can_dma(const struct rspi_data *rspi,  in __rspi_can_dma()  argument
654 	return xfer->len > rspi->ops->fifo_size;  in __rspi_can_dma()
660 	struct rspi_data *rspi = spi_master_get_devdata(master);  in rspi_can_dma()  local
662 	return __rspi_can_dma(rspi, xfer);  in rspi_can_dma()
665 static int rspi_dma_check_then_transfer(struct rspi_data *rspi,  in rspi_dma_check_then_transfer()  argument
668 	if (rspi->master->can_dma && __rspi_can_dma(rspi, xfer)) {  in rspi_dma_check_then_transfer()
670 		int ret = rspi_dma_transfer(rspi, &xfer->tx_sg,  in rspi_dma_check_then_transfer()
679 static int rspi_common_transfer(struct rspi_data *rspi,  in rspi_common_transfer()  argument
684 	ret = rspi_dma_check_then_transfer(rspi, xfer);  in rspi_common_transfer()
688 	ret = rspi_pio_transfer(rspi, xfer->tx_buf, xfer->rx_buf, xfer->len);  in rspi_common_transfer()
693 	rspi_wait_for_tx_empty(rspi);  in rspi_common_transfer()
701 	struct rspi_data *rspi = spi_master_get_devdata(master);  in rspi_transfer_one()  local
704 	spcr = rspi_read8(rspi, RSPI_SPCR);  in rspi_transfer_one()
706 		rspi_receive_init(rspi);  in rspi_transfer_one()
711 	rspi_write8(rspi, spcr, RSPI_SPCR);  in rspi_transfer_one()
713 	return rspi_common_transfer(rspi, xfer);  in rspi_transfer_one()
720 	struct rspi_data *rspi = spi_master_get_devdata(master);  in rspi_rz_transfer_one()  local
722 	rspi_rz_receive_init(rspi);  in rspi_rz_transfer_one()
724 	return rspi_common_transfer(rspi, xfer);  in rspi_rz_transfer_one()
727 static int qspi_trigger_transfer_out_int(struct rspi_data *rspi, const u8 *tx,  in qspi_trigger_transfer_out_int()  argument
734 		n = qspi_set_send_trigger(rspi, len);  in qspi_trigger_transfer_out_int()
735 		qspi_set_receive_trigger(rspi, len);  in qspi_trigger_transfer_out_int()
737 			error = rspi_wait_for_tx_empty(rspi);  in qspi_trigger_transfer_out_int()
739 				dev_err(&rspi->master->dev, "transmit timeout\n");  in qspi_trigger_transfer_out_int()
743 				rspi_write_data(rspi, *tx++);  in qspi_trigger_transfer_out_int()
745 			error = rspi_wait_for_rx_full(rspi);  in qspi_trigger_transfer_out_int()
747 				dev_err(&rspi->master->dev, "receive timeout\n");  in qspi_trigger_transfer_out_int()
751 				*rx++ = rspi_read_data(rspi);  in qspi_trigger_transfer_out_int()
753 			ret = rspi_pio_transfer(rspi, tx, rx, n);  in qspi_trigger_transfer_out_int()
763 static int qspi_transfer_out_in(struct rspi_data *rspi,  in qspi_transfer_out_in()  argument
768 	qspi_receive_init(rspi);  in qspi_transfer_out_in()
770 	ret = rspi_dma_check_then_transfer(rspi, xfer);  in qspi_transfer_out_in()
774 	ret = qspi_trigger_transfer_out_int(rspi, xfer->tx_buf,  in qspi_transfer_out_in()
782 static int qspi_transfer_out(struct rspi_data *rspi, struct spi_transfer *xfer)  in qspi_transfer_out()  argument
786 	if (rspi->master->can_dma && __rspi_can_dma(rspi, xfer)) {  in qspi_transfer_out()
787 		ret = rspi_dma_transfer(rspi, &xfer->tx_sg, NULL);  in qspi_transfer_out()
792 	ret = rspi_pio_transfer(rspi, xfer->tx_buf, NULL, xfer->len);  in qspi_transfer_out()
797 	rspi_wait_for_tx_empty(rspi);  in qspi_transfer_out()
802 static int qspi_transfer_in(struct rspi_data *rspi, struct spi_transfer *xfer)  in qspi_transfer_in()  argument
804 	if (rspi->master->can_dma && __rspi_can_dma(rspi, xfer)) {  in qspi_transfer_in()
805 		int ret = rspi_dma_transfer(rspi, NULL, &xfer->rx_sg);  in qspi_transfer_in()
810 	return rspi_pio_transfer(rspi, NULL, xfer->rx_buf, xfer->len);  in qspi_transfer_in()
816 	struct rspi_data *rspi = spi_master_get_devdata(master);  in qspi_transfer_one()  local
819 		return qspi_transfer_out_in(rspi, xfer);  in qspi_transfer_one()
822 		return qspi_transfer_out(rspi, xfer);  in qspi_transfer_one()
825 		return qspi_transfer_in(rspi, xfer);  in qspi_transfer_one()
828 		return qspi_transfer_out_in(rspi, xfer);  in qspi_transfer_one()
834 	struct rspi_data *rspi = spi_master_get_devdata(spi->master);  in rspi_setup()  local
836 	rspi->max_speed_hz = spi->max_speed_hz;  in rspi_setup()
838 	rspi->spcmd = SPCMD_SSLKP;  in rspi_setup()
840 		rspi->spcmd |= SPCMD_CPOL;  in rspi_setup()
842 		rspi->spcmd |= SPCMD_CPHA;  in rspi_setup()
845 	rspi->sppcr = 0;  in rspi_setup()
847 		rspi->sppcr |= SPPCR_SPLP;  in rspi_setup()
849 	set_config_register(rspi, 8);  in rspi_setup()
878 static int qspi_setup_sequencer(struct rspi_data *rspi,  in qspi_setup_sequencer()  argument
895 			rspi_write32(rspi, len, QSPI_SPBMUL(i - 1));  in qspi_setup_sequencer()
905 		rspi_write16(rspi, rspi->spcmd | mode, RSPI_SPCMD(i));  in qspi_setup_sequencer()
912 		rspi_write32(rspi, len, QSPI_SPBMUL(i - 1));  in qspi_setup_sequencer()
913 		rspi_write8(rspi, i - 1, RSPI_SPSCR);  in qspi_setup_sequencer()
922 	struct rspi_data *rspi = spi_master_get_devdata(master);  in rspi_prepare_message()  local
928 		ret = qspi_setup_sequencer(rspi, msg);  in rspi_prepare_message()
934 	rspi_write8(rspi, rspi_read8(rspi, RSPI_SPCR) | SPCR_SPE, RSPI_SPCR);  in rspi_prepare_message()
941 	struct rspi_data *rspi = spi_master_get_devdata(master);  in rspi_unprepare_message()  local
944 	rspi_write8(rspi, rspi_read8(rspi, RSPI_SPCR) & ~SPCR_SPE, RSPI_SPCR);  in rspi_unprepare_message()
947 	rspi_write16(rspi, rspi->spcmd, RSPI_SPCMD0);  in rspi_unprepare_message()
948 	rspi_write8(rspi, 0, RSPI_SPSCR);  in rspi_unprepare_message()
954 	struct rspi_data *rspi = _sr;  in rspi_irq_mux()  local
959 	rspi->spsr = spsr = rspi_read8(rspi, RSPI_SPSR);  in rspi_irq_mux()
967 		rspi_disable_irq(rspi, disable_irq);  in rspi_irq_mux()
968 		wake_up(&rspi->wait);  in rspi_irq_mux()
976 	struct rspi_data *rspi = _sr;  in rspi_irq_rx()  local
979 	rspi->spsr = spsr = rspi_read8(rspi, RSPI_SPSR);  in rspi_irq_rx()
981 		rspi_disable_irq(rspi, SPCR_SPRIE);  in rspi_irq_rx()
982 		wake_up(&rspi->wait);  in rspi_irq_rx()
991 	struct rspi_data *rspi = _sr;  in rspi_irq_tx()  local
994 	rspi->spsr = spsr = rspi_read8(rspi, RSPI_SPSR);  in rspi_irq_tx()
996 		rspi_disable_irq(rspi, SPCR_SPTIE);  in rspi_irq_tx()
997 		wake_up(&rspi->wait);  in rspi_irq_tx()
1091 	struct rspi_data *rspi = platform_get_drvdata(pdev);  in rspi_remove()  local
1093 	rspi_release_dma(rspi->master);  in rspi_remove()
1177 	struct rspi_data *rspi;  in rspi_probe()  local
1211 	rspi = spi_master_get_devdata(master);  in rspi_probe()
1212 	platform_set_drvdata(pdev, rspi);  in rspi_probe()
1213 	rspi->ops = ops;  in rspi_probe()
1214 	rspi->master = master;  in rspi_probe()
1217 	rspi->addr = devm_ioremap_resource(&pdev->dev, res);  in rspi_probe()
1218 	if (IS_ERR(rspi->addr)) {  in rspi_probe()
1219 		ret = PTR_ERR(rspi->addr);  in rspi_probe()
1223 	rspi->clk = devm_clk_get(&pdev->dev, NULL);  in rspi_probe()
1224 	if (IS_ERR(rspi->clk)) {  in rspi_probe()
1226 		ret = PTR_ERR(rspi->clk);  in rspi_probe()
1232 	init_waitqueue_head(&rspi->wait);  in rspi_probe()
1250 			rspi->rx_irq = rspi->tx_irq = ret;  in rspi_probe()
1252 		rspi->rx_irq = ret;  in rspi_probe()
1255 			rspi->tx_irq = ret;  in rspi_probe()
1262 	if (rspi->rx_irq == rspi->tx_irq) {  in rspi_probe()
1264 		ret = rspi_request_irq(&pdev->dev, rspi->rx_irq, rspi_irq_mux,  in rspi_probe()
1265 				       "mux", rspi);  in rspi_probe()
1268 		ret = rspi_request_irq(&pdev->dev, rspi->rx_irq, rspi_irq_rx,  in rspi_probe()
1269 				       "rx", rspi);  in rspi_probe()
1271 			ret = rspi_request_irq(&pdev->dev, rspi->tx_irq,  in rspi_probe()
1272 					       rspi_irq_tx, "tx", rspi);  in rspi_probe()