126 static void bfin_spi_enable(struct bfin_spi_master_data *drv_data)  in bfin_spi_enable()  argument
128 	bfin_write_or(&drv_data->regs->ctl, BIT_CTL_ENABLE);  in bfin_spi_enable()
131 static void bfin_spi_disable(struct bfin_spi_master_data *drv_data)  in bfin_spi_disable()  argument
133 	bfin_write_and(&drv_data->regs->ctl, ~BIT_CTL_ENABLE);  in bfin_spi_disable()
151 static int bfin_spi_flush(struct bfin_spi_master_data *drv_data)  in bfin_spi_flush()  argument
156 	while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_SPIF) && --limit)  in bfin_spi_flush()
159 	bfin_write(&drv_data->regs->stat, BIT_STAT_CLR);  in bfin_spi_flush()
165 static void bfin_spi_cs_active(struct bfin_spi_master_data *drv_data, struct bfin_spi_slave_data *c…  in bfin_spi_cs_active()  argument
168 		bfin_write_and(&drv_data->regs->flg, ~chip->flag);  in bfin_spi_cs_active()
173 static void bfin_spi_cs_deactive(struct bfin_spi_master_data *drv_data,  in bfin_spi_cs_deactive()  argument
177 		bfin_write_or(&drv_data->regs->flg, chip->flag);  in bfin_spi_cs_deactive()
187 static inline void bfin_spi_cs_enable(struct bfin_spi_master_data *drv_data,  in bfin_spi_cs_enable()  argument
191 		bfin_write_or(&drv_data->regs->flg, chip->flag >> 8);  in bfin_spi_cs_enable()
194 static inline void bfin_spi_cs_disable(struct bfin_spi_master_data *drv_data,  in bfin_spi_cs_disable()  argument
198 		bfin_write_and(&drv_data->regs->flg, ~(chip->flag >> 8));  in bfin_spi_cs_disable()
202 static void bfin_spi_restore_state(struct bfin_spi_master_data *drv_data)  in bfin_spi_restore_state()  argument
204 	struct bfin_spi_slave_data *chip = drv_data->cur_chip;  in bfin_spi_restore_state()
207 	bfin_write(&drv_data->regs->stat, BIT_STAT_CLR);  in bfin_spi_restore_state()
208 	bfin_spi_disable(drv_data);  in bfin_spi_restore_state()
209 	dev_dbg(&drv_data->pdev->dev, "restoring spi ctl state\n");  in bfin_spi_restore_state()
214 	bfin_write(&drv_data->regs->ctl, chip->ctl_reg);  in bfin_spi_restore_state()
215 	bfin_write(&drv_data->regs->baud, chip->baud);  in bfin_spi_restore_state()
217 	bfin_spi_enable(drv_data);  in bfin_spi_restore_state()
218 	bfin_spi_cs_active(drv_data, chip);  in bfin_spi_restore_state()
222 static inline void bfin_spi_dummy_read(struct bfin_spi_master_data *drv_data)  in bfin_spi_dummy_read()  argument
224 	(void) bfin_read(&drv_data->regs->rdbr);  in bfin_spi_dummy_read()
227 static void bfin_spi_u8_writer(struct bfin_spi_master_data *drv_data)  in bfin_spi_u8_writer()  argument
230 	bfin_spi_dummy_read(drv_data);  in bfin_spi_u8_writer()
232 	while (drv_data->tx < drv_data->tx_end) {  in bfin_spi_u8_writer()
233 		bfin_write(&drv_data->regs->tdbr, (*(u8 *) (drv_data->tx++)));  in bfin_spi_u8_writer()
236 		while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_RXS))  in bfin_spi_u8_writer()
239 		bfin_spi_dummy_read(drv_data);  in bfin_spi_u8_writer()
243 static void bfin_spi_u8_reader(struct bfin_spi_master_data *drv_data)  in bfin_spi_u8_reader()  argument
245 	u16 tx_val = drv_data->cur_chip->idle_tx_val;  in bfin_spi_u8_reader()
248 	bfin_spi_dummy_read(drv_data);  in bfin_spi_u8_reader()
250 	while (drv_data->rx < drv_data->rx_end) {  in bfin_spi_u8_reader()
251 		bfin_write(&drv_data->regs->tdbr, tx_val);  in bfin_spi_u8_reader()
252 		while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_RXS))  in bfin_spi_u8_reader()
254 		*(u8 *) (drv_data->rx++) = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_u8_reader()
258 static void bfin_spi_u8_duplex(struct bfin_spi_master_data *drv_data)  in bfin_spi_u8_duplex()  argument
261 	bfin_spi_dummy_read(drv_data);  in bfin_spi_u8_duplex()
263 	while (drv_data->rx < drv_data->rx_end) {  in bfin_spi_u8_duplex()
264 		bfin_write(&drv_data->regs->tdbr, (*(u8 *) (drv_data->tx++)));  in bfin_spi_u8_duplex()
265 		while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_RXS))  in bfin_spi_u8_duplex()
267 		*(u8 *) (drv_data->rx++) = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_u8_duplex()
277 static void bfin_spi_u16_writer(struct bfin_spi_master_data *drv_data)  in bfin_spi_u16_writer()  argument
280 	bfin_spi_dummy_read(drv_data);  in bfin_spi_u16_writer()
282 	while (drv_data->tx < drv_data->tx_end) {  in bfin_spi_u16_writer()
283 		bfin_write(&drv_data->regs->tdbr, (*(u16 *) (drv_data->tx)));  in bfin_spi_u16_writer()
284 		drv_data->tx += 2;  in bfin_spi_u16_writer()
287 		while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_RXS))  in bfin_spi_u16_writer()
290 		bfin_spi_dummy_read(drv_data);  in bfin_spi_u16_writer()
294 static void bfin_spi_u16_reader(struct bfin_spi_master_data *drv_data)  in bfin_spi_u16_reader()  argument
296 	u16 tx_val = drv_data->cur_chip->idle_tx_val;  in bfin_spi_u16_reader()
299 	bfin_spi_dummy_read(drv_data);  in bfin_spi_u16_reader()
301 	while (drv_data->rx < drv_data->rx_end) {  in bfin_spi_u16_reader()
302 		bfin_write(&drv_data->regs->tdbr, tx_val);  in bfin_spi_u16_reader()
303 		while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_RXS))  in bfin_spi_u16_reader()
305 		*(u16 *) (drv_data->rx) = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_u16_reader()
306 		drv_data->rx += 2;  in bfin_spi_u16_reader()
310 static void bfin_spi_u16_duplex(struct bfin_spi_master_data *drv_data)  in bfin_spi_u16_duplex()  argument
313 	bfin_spi_dummy_read(drv_data);  in bfin_spi_u16_duplex()
315 	while (drv_data->rx < drv_data->rx_end) {  in bfin_spi_u16_duplex()
316 		bfin_write(&drv_data->regs->tdbr, (*(u16 *) (drv_data->tx)));  in bfin_spi_u16_duplex()
317 		drv_data->tx += 2;  in bfin_spi_u16_duplex()
318 		while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_RXS))  in bfin_spi_u16_duplex()
320 		*(u16 *) (drv_data->rx) = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_u16_duplex()
321 		drv_data->rx += 2;  in bfin_spi_u16_duplex()
332 static void *bfin_spi_next_transfer(struct bfin_spi_master_data *drv_data)  in bfin_spi_next_transfer()  argument
334 	struct spi_message *msg = drv_data->cur_msg;  in bfin_spi_next_transfer()
335 	struct spi_transfer *trans = drv_data->cur_transfer;  in bfin_spi_next_transfer()
339 		drv_data->cur_transfer =  in bfin_spi_next_transfer()
351 static void bfin_spi_giveback(struct bfin_spi_master_data *drv_data)  in bfin_spi_giveback()  argument
353 	struct bfin_spi_slave_data *chip = drv_data->cur_chip;  in bfin_spi_giveback()
357 	spin_lock_irqsave(&drv_data->lock, flags);  in bfin_spi_giveback()
358 	msg = drv_data->cur_msg;  in bfin_spi_giveback()
359 	drv_data->cur_msg = NULL;  in bfin_spi_giveback()
360 	drv_data->cur_transfer = NULL;  in bfin_spi_giveback()
361 	drv_data->cur_chip = NULL;  in bfin_spi_giveback()
362 	queue_work(drv_data->workqueue, &drv_data->pump_messages);  in bfin_spi_giveback()
363 	spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_giveback()
367 	if (!drv_data->cs_change)  in bfin_spi_giveback()
368 		bfin_spi_cs_deactive(drv_data, chip);  in bfin_spi_giveback()
371 	if (drv_data->tx_dma != 0xFFFF)  in bfin_spi_giveback()
372 		bfin_spi_disable(drv_data);  in bfin_spi_giveback()
381 	struct bfin_spi_master_data *drv_data = dev_id;  in bfin_spi_pio_irq_handler()  local
382 	struct bfin_spi_slave_data *chip = drv_data->cur_chip;  in bfin_spi_pio_irq_handler()
383 	struct spi_message *msg = drv_data->cur_msg;  in bfin_spi_pio_irq_handler()
384 	int n_bytes = drv_data->n_bytes;  in bfin_spi_pio_irq_handler()
388 	while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_RXS))  in bfin_spi_pio_irq_handler()
391 	if ((drv_data->tx && drv_data->tx >= drv_data->tx_end) ||  in bfin_spi_pio_irq_handler()
392 		(drv_data->rx && drv_data->rx >= (drv_data->rx_end - n_bytes))) {  in bfin_spi_pio_irq_handler()
394 		if (drv_data->rx) {  in bfin_spi_pio_irq_handler()
395 			dev_dbg(&drv_data->pdev->dev, "last read\n");  in bfin_spi_pio_irq_handler()
397 				u16 *buf = (u16 *)drv_data->rx;  in bfin_spi_pio_irq_handler()
399 					*buf++ = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
401 				u8 *buf = (u8 *)drv_data->rx;  in bfin_spi_pio_irq_handler()
403 					*buf++ = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
405 			drv_data->rx += n_bytes;  in bfin_spi_pio_irq_handler()
408 		msg->actual_length += drv_data->len_in_bytes;  in bfin_spi_pio_irq_handler()
409 		if (drv_data->cs_change)  in bfin_spi_pio_irq_handler()
410 			bfin_spi_cs_deactive(drv_data, chip);  in bfin_spi_pio_irq_handler()
412 		msg->state = bfin_spi_next_transfer(drv_data);  in bfin_spi_pio_irq_handler()
414 		disable_irq_nosync(drv_data->spi_irq);  in bfin_spi_pio_irq_handler()
417 		tasklet_schedule(&drv_data->pump_transfers);  in bfin_spi_pio_irq_handler()
421 	if (drv_data->rx && drv_data->tx) {  in bfin_spi_pio_irq_handler()
423 		dev_dbg(&drv_data->pdev->dev, "duplex: write_TDBR\n");  in bfin_spi_pio_irq_handler()
425 			u16 *buf = (u16 *)drv_data->rx;  in bfin_spi_pio_irq_handler()
426 			u16 *buf2 = (u16 *)drv_data->tx;  in bfin_spi_pio_irq_handler()
428 				*buf++ = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
429 				bfin_write(&drv_data->regs->tdbr, *buf2++);  in bfin_spi_pio_irq_handler()
432 			u8 *buf = (u8 *)drv_data->rx;  in bfin_spi_pio_irq_handler()
433 			u8 *buf2 = (u8 *)drv_data->tx;  in bfin_spi_pio_irq_handler()
435 				*buf++ = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
436 				bfin_write(&drv_data->regs->tdbr, *buf2++);  in bfin_spi_pio_irq_handler()
439 	} else if (drv_data->rx) {  in bfin_spi_pio_irq_handler()
441 		dev_dbg(&drv_data->pdev->dev, "read: write_TDBR\n");  in bfin_spi_pio_irq_handler()
443 			u16 *buf = (u16 *)drv_data->rx;  in bfin_spi_pio_irq_handler()
445 				*buf++ = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
446 				bfin_write(&drv_data->regs->tdbr, chip->idle_tx_val);  in bfin_spi_pio_irq_handler()
449 			u8 *buf = (u8 *)drv_data->rx;  in bfin_spi_pio_irq_handler()
451 				*buf++ = bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
452 				bfin_write(&drv_data->regs->tdbr, chip->idle_tx_val);  in bfin_spi_pio_irq_handler()
455 	} else if (drv_data->tx) {  in bfin_spi_pio_irq_handler()
457 		dev_dbg(&drv_data->pdev->dev, "write: write_TDBR\n");  in bfin_spi_pio_irq_handler()
459 			u16 *buf = (u16 *)drv_data->tx;  in bfin_spi_pio_irq_handler()
461 				bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
462 				bfin_write(&drv_data->regs->tdbr, *buf++);  in bfin_spi_pio_irq_handler()
465 			u8 *buf = (u8 *)drv_data->tx;  in bfin_spi_pio_irq_handler()
467 				bfin_read(&drv_data->regs->rdbr);  in bfin_spi_pio_irq_handler()
468 				bfin_write(&drv_data->regs->tdbr, *buf++);  in bfin_spi_pio_irq_handler()
473 	if (drv_data->tx)  in bfin_spi_pio_irq_handler()
474 		drv_data->tx += n_bytes;  in bfin_spi_pio_irq_handler()
475 	if (drv_data->rx)  in bfin_spi_pio_irq_handler()
476 		drv_data->rx += n_bytes;  in bfin_spi_pio_irq_handler()
483 	struct bfin_spi_master_data *drv_data = dev_id;  in bfin_spi_dma_irq_handler()  local
484 	struct bfin_spi_slave_data *chip = drv_data->cur_chip;  in bfin_spi_dma_irq_handler()
485 	struct spi_message *msg = drv_data->cur_msg;  in bfin_spi_dma_irq_handler()
487 	unsigned short dmastat = get_dma_curr_irqstat(drv_data->dma_channel);  in bfin_spi_dma_irq_handler()
488 	u16 spistat = bfin_read(&drv_data->regs->stat);  in bfin_spi_dma_irq_handler()
490 	dev_dbg(&drv_data->pdev->dev,  in bfin_spi_dma_irq_handler()
494 	if (drv_data->rx != NULL) {  in bfin_spi_dma_irq_handler()
495 		u16 cr = bfin_read(&drv_data->regs->ctl);  in bfin_spi_dma_irq_handler()
497 		bfin_spi_dummy_read(drv_data);  in bfin_spi_dma_irq_handler()
498 		bfin_write(&drv_data->regs->ctl, cr & ~BIT_CTL_ENABLE); /* Disable SPI */  in bfin_spi_dma_irq_handler()
499 		bfin_write(&drv_data->regs->ctl, cr & ~BIT_CTL_TIMOD); /* Restore State */  in bfin_spi_dma_irq_handler()
500 		bfin_write(&drv_data->regs->stat, BIT_STAT_CLR); /* Clear Status */  in bfin_spi_dma_irq_handler()
503 	clear_dma_irqstat(drv_data->dma_channel);  in bfin_spi_dma_irq_handler()
511 	if (drv_data->tx != NULL) {  in bfin_spi_dma_irq_handler()
512 		while ((bfin_read(&drv_data->regs->stat) & BIT_STAT_TXS) ||  in bfin_spi_dma_irq_handler()
513 		       (bfin_read(&drv_data->regs->stat) & BIT_STAT_TXS))  in bfin_spi_dma_irq_handler()
517 	dev_dbg(&drv_data->pdev->dev,  in bfin_spi_dma_irq_handler()
519 		dmastat, bfin_read(&drv_data->regs->stat));  in bfin_spi_dma_irq_handler()
522 	while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_SPIF))  in bfin_spi_dma_irq_handler()
524 			dev_warn(&drv_data->pdev->dev, "timeout waiting for SPIF\n");  in bfin_spi_dma_irq_handler()
531 		dev_err(&drv_data->pdev->dev, "dma receive: fifo/buffer overflow\n");  in bfin_spi_dma_irq_handler()
533 		msg->actual_length += drv_data->len_in_bytes;  in bfin_spi_dma_irq_handler()
535 		if (drv_data->cs_change)  in bfin_spi_dma_irq_handler()
536 			bfin_spi_cs_deactive(drv_data, chip);  in bfin_spi_dma_irq_handler()
539 		msg->state = bfin_spi_next_transfer(drv_data);  in bfin_spi_dma_irq_handler()
543 	tasklet_schedule(&drv_data->pump_transfers);  in bfin_spi_dma_irq_handler()
546 	dev_dbg(&drv_data->pdev->dev,  in bfin_spi_dma_irq_handler()
548 		drv_data->dma_channel);  in bfin_spi_dma_irq_handler()
549 	dma_disable_irq_nosync(drv_data->dma_channel);  in bfin_spi_dma_irq_handler()
556 	struct bfin_spi_master_data *drv_data = (struct bfin_spi_master_data *)data;  in bfin_spi_pump_transfers()  local
567 	message = drv_data->cur_msg;  in bfin_spi_pump_transfers()
568 	transfer = drv_data->cur_transfer;  in bfin_spi_pump_transfers()
569 	chip = drv_data->cur_chip;  in bfin_spi_pump_transfers()
577 		dev_dbg(&drv_data->pdev->dev, "transfer: we've hit an error\n");  in bfin_spi_pump_transfers()
579 		bfin_spi_giveback(drv_data);  in bfin_spi_pump_transfers()
585 		dev_dbg(&drv_data->pdev->dev, "transfer: all done!\n");  in bfin_spi_pump_transfers()
587 		bfin_spi_flush(drv_data);  in bfin_spi_pump_transfers()
588 		bfin_spi_giveback(drv_data);  in bfin_spi_pump_transfers()
594 		dev_dbg(&drv_data->pdev->dev, "transfer: still running ...\n");  in bfin_spi_pump_transfers()
602 	if (bfin_spi_flush(drv_data) == 0) {  in bfin_spi_pump_transfers()
603 		dev_err(&drv_data->pdev->dev, "pump_transfers: flush failed\n");  in bfin_spi_pump_transfers()
605 		bfin_spi_giveback(drv_data);  in bfin_spi_pump_transfers()
611 		message->state = bfin_spi_next_transfer(drv_data);  in bfin_spi_pump_transfers()
613 		tasklet_schedule(&drv_data->pump_transfers);  in bfin_spi_pump_transfers()
618 		drv_data->tx = (void *)transfer->tx_buf;  in bfin_spi_pump_transfers()
619 		drv_data->tx_end = drv_data->tx + transfer->len;  in bfin_spi_pump_transfers()
620 		dev_dbg(&drv_data->pdev->dev, "tx_buf is %p, tx_end is %p\n",  in bfin_spi_pump_transfers()
621 			transfer->tx_buf, drv_data->tx_end);  in bfin_spi_pump_transfers()
623 		drv_data->tx = NULL;  in bfin_spi_pump_transfers()
628 		drv_data->rx = transfer->rx_buf;  in bfin_spi_pump_transfers()
629 		drv_data->rx_end = drv_data->rx + transfer->len;  in bfin_spi_pump_transfers()
630 		dev_dbg(&drv_data->pdev->dev, "rx_buf is %p, rx_end is %p\n",  in bfin_spi_pump_transfers()
631 			transfer->rx_buf, drv_data->rx_end);  in bfin_spi_pump_transfers()
633 		drv_data->rx = NULL;  in bfin_spi_pump_transfers()
636 	drv_data->rx_dma = transfer->rx_dma;  in bfin_spi_pump_transfers()
637 	drv_data->tx_dma = transfer->tx_dma;  in bfin_spi_pump_transfers()
638 	drv_data->len_in_bytes = transfer->len;  in bfin_spi_pump_transfers()
639 	drv_data->cs_change = transfer->cs_change;  in bfin_spi_pump_transfers()
644 		drv_data->n_bytes = bits_per_word/8;  in bfin_spi_pump_transfers()
645 		drv_data->len = (transfer->len) >> 1;  in bfin_spi_pump_transfers()
647 		drv_data->ops = &bfin_bfin_spi_transfer_ops_u16;  in bfin_spi_pump_transfers()
649 		drv_data->n_bytes = bits_per_word/8;  in bfin_spi_pump_transfers()
650 		drv_data->len = transfer->len;  in bfin_spi_pump_transfers()
651 		drv_data->ops = &bfin_bfin_spi_transfer_ops_u8;  in bfin_spi_pump_transfers()
653 	cr = bfin_read(&drv_data->regs->ctl) & ~(BIT_CTL_TIMOD | BIT_CTL_WORDSIZE);  in bfin_spi_pump_transfers()
655 	bfin_write(&drv_data->regs->ctl, cr);  in bfin_spi_pump_transfers()
657 	dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_transfers()
659 		drv_data->ops, chip->ops, &bfin_bfin_spi_transfer_ops_u8);  in bfin_spi_pump_transfers()
666 		bfin_write(&drv_data->regs->baud, hz_to_spi_baud(transfer->speed_hz));  in bfin_spi_pump_transfers()
668 		bfin_write(&drv_data->regs->baud, chip->baud);  in bfin_spi_pump_transfers()
670 	bfin_write(&drv_data->regs->stat, BIT_STAT_CLR);  in bfin_spi_pump_transfers()
671 	bfin_spi_cs_active(drv_data, chip);  in bfin_spi_pump_transfers()
673 	dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_transfers()
683 	if (!full_duplex && drv_data->cur_chip->enable_dma  in bfin_spi_pump_transfers()
684 				&& drv_data->len > 6) {  in bfin_spi_pump_transfers()
688 		disable_dma(drv_data->dma_channel);  in bfin_spi_pump_transfers()
689 		clear_dma_irqstat(drv_data->dma_channel);  in bfin_spi_pump_transfers()
692 		dev_dbg(&drv_data->pdev->dev, "doing dma transfer\n");  in bfin_spi_pump_transfers()
693 		set_dma_x_count(drv_data->dma_channel, drv_data->len);  in bfin_spi_pump_transfers()
695 			set_dma_x_modify(drv_data->dma_channel, 2);  in bfin_spi_pump_transfers()
698 			set_dma_x_modify(drv_data->dma_channel, 1);  in bfin_spi_pump_transfers()
703 		while (!(bfin_read(&drv_data->regs->stat) & BIT_STAT_SPIF))  in bfin_spi_pump_transfers()
707 		if (drv_data->tx_dma == 0xFFFF) {  in bfin_spi_pump_transfers()
708 			dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_transfers()
714 			set_dma_config(drv_data->dma_channel, dma_config);  in bfin_spi_pump_transfers()
715 			set_dma_start_addr(drv_data->dma_channel,  in bfin_spi_pump_transfers()
716 					(unsigned long)drv_data->tx);  in bfin_spi_pump_transfers()
717 			enable_dma(drv_data->dma_channel);  in bfin_spi_pump_transfers()
720 			bfin_write(&drv_data->regs->ctl, cr | BIT_CTL_TIMOD_DMA_TX);  in bfin_spi_pump_transfers()
726 			bfin_spi_giveback(drv_data);  in bfin_spi_pump_transfers()
732 		if (drv_data->rx != NULL) {  in bfin_spi_pump_transfers()
734 			dev_dbg(&drv_data->pdev->dev, "doing DMA in to %p (size %zx)\n",  in bfin_spi_pump_transfers()
735 				drv_data->rx, drv_data->len_in_bytes);  in bfin_spi_pump_transfers()
738 			if (bfin_addr_dcacheable((unsigned long) drv_data->rx))  in bfin_spi_pump_transfers()
739 				invalidate_dcache_range((unsigned long) drv_data->rx,  in bfin_spi_pump_transfers()
740 							(unsigned long) (drv_data->rx +  in bfin_spi_pump_transfers()
741 							drv_data->len_in_bytes));  in bfin_spi_pump_transfers()
744 			dma_start_addr = (unsigned long)drv_data->rx;  in bfin_spi_pump_transfers()
747 		} else if (drv_data->tx != NULL) {  in bfin_spi_pump_transfers()
748 			dev_dbg(&drv_data->pdev->dev, "doing DMA out.\n");  in bfin_spi_pump_transfers()
751 			if (bfin_addr_dcacheable((unsigned long) drv_data->tx))  in bfin_spi_pump_transfers()
752 				flush_dcache_range((unsigned long) drv_data->tx,  in bfin_spi_pump_transfers()
753 						(unsigned long) (drv_data->tx +  in bfin_spi_pump_transfers()
754 						drv_data->len_in_bytes));  in bfin_spi_pump_transfers()
756 			dma_start_addr = (unsigned long)drv_data->tx;  in bfin_spi_pump_transfers()
771 		set_dma_start_addr(drv_data->dma_channel, dma_start_addr);  in bfin_spi_pump_transfers()
772 		set_dma_config(drv_data->dma_channel, dma_config);  in bfin_spi_pump_transfers()
775 		bfin_write(&drv_data->regs->ctl, cr);  in bfin_spi_pump_transfers()
776 		enable_dma(drv_data->dma_channel);  in bfin_spi_pump_transfers()
777 		dma_enable_irq(drv_data->dma_channel);  in bfin_spi_pump_transfers()
789 	bfin_write(&drv_data->regs->ctl, cr | BIT_CTL_TXMOD);  in bfin_spi_pump_transfers()
795 		bfin_spi_dummy_read(drv_data);  in bfin_spi_pump_transfers()
798 		if (drv_data->tx == NULL)  in bfin_spi_pump_transfers()
799 			bfin_write(&drv_data->regs->tdbr, chip->idle_tx_val);  in bfin_spi_pump_transfers()
803 				u16 *buf = (u16 *)drv_data->tx;  in bfin_spi_pump_transfers()
806 					bfin_write(&drv_data->regs->tdbr, *buf++);  in bfin_spi_pump_transfers()
809 				u8 *buf = (u8 *)drv_data->tx;  in bfin_spi_pump_transfers()
811 					bfin_write(&drv_data->regs->tdbr, *buf++);  in bfin_spi_pump_transfers()
814 			drv_data->tx += drv_data->n_bytes;  in bfin_spi_pump_transfers()
818 		enable_irq(drv_data->spi_irq);  in bfin_spi_pump_transfers()
823 	dev_dbg(&drv_data->pdev->dev, "doing IO transfer\n");  in bfin_spi_pump_transfers()
827 		BUG_ON((drv_data->tx_end - drv_data->tx) !=  in bfin_spi_pump_transfers()
828 		       (drv_data->rx_end - drv_data->rx));  in bfin_spi_pump_transfers()
829 		dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_transfers()
832 		drv_data->ops->duplex(drv_data);  in bfin_spi_pump_transfers()
834 		if (drv_data->tx != drv_data->tx_end)  in bfin_spi_pump_transfers()
836 	} else if (drv_data->tx != NULL) {  in bfin_spi_pump_transfers()
838 		dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_transfers()
841 		drv_data->ops->write(drv_data);  in bfin_spi_pump_transfers()
843 		if (drv_data->tx != drv_data->tx_end)  in bfin_spi_pump_transfers()
845 	} else if (drv_data->rx != NULL) {  in bfin_spi_pump_transfers()
847 		dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_transfers()
850 		drv_data->ops->read(drv_data);  in bfin_spi_pump_transfers()
851 		if (drv_data->rx != drv_data->rx_end)  in bfin_spi_pump_transfers()
856 		dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_transfers()
861 		message->actual_length += drv_data->len_in_bytes;  in bfin_spi_pump_transfers()
863 		message->state = bfin_spi_next_transfer(drv_data);  in bfin_spi_pump_transfers()
864 		if (drv_data->cs_change && message->state != DONE_STATE) {  in bfin_spi_pump_transfers()
865 			bfin_spi_flush(drv_data);  in bfin_spi_pump_transfers()
866 			bfin_spi_cs_deactive(drv_data, chip);  in bfin_spi_pump_transfers()
871 	tasklet_schedule(&drv_data->pump_transfers);  in bfin_spi_pump_transfers()
877 	struct bfin_spi_master_data *drv_data;  in bfin_spi_pump_messages()  local
880 	drv_data = container_of(work, struct bfin_spi_master_data, pump_messages);  in bfin_spi_pump_messages()
883 	spin_lock_irqsave(&drv_data->lock, flags);  in bfin_spi_pump_messages()
884 	if (list_empty(&drv_data->queue) || !drv_data->running) {  in bfin_spi_pump_messages()
886 		drv_data->busy = 0;  in bfin_spi_pump_messages()
887 		spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_pump_messages()
892 	if (drv_data->cur_msg) {  in bfin_spi_pump_messages()
893 		spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_pump_messages()
898 	drv_data->cur_msg = list_entry(drv_data->queue.next,  in bfin_spi_pump_messages()
902 	drv_data->cur_chip = spi_get_ctldata(drv_data->cur_msg->spi);  in bfin_spi_pump_messages()
903 	bfin_spi_restore_state(drv_data);  in bfin_spi_pump_messages()
905 	list_del_init(&drv_data->cur_msg->queue);  in bfin_spi_pump_messages()
908 	drv_data->cur_msg->state = START_STATE;  in bfin_spi_pump_messages()
909 	drv_data->cur_transfer = list_entry(drv_data->cur_msg->transfers.next,  in bfin_spi_pump_messages()
912 	dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_messages()
915 		drv_data->cur_chip->baud, drv_data->cur_chip->flag,  in bfin_spi_pump_messages()
916 		drv_data->cur_chip->ctl_reg);  in bfin_spi_pump_messages()
918 	dev_dbg(&drv_data->pdev->dev,  in bfin_spi_pump_messages()
920 		drv_data->cur_transfer->len);  in bfin_spi_pump_messages()
923 	tasklet_schedule(&drv_data->pump_transfers);  in bfin_spi_pump_messages()
925 	drv_data->busy = 1;  in bfin_spi_pump_messages()
926 	spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_pump_messages()
935 	struct bfin_spi_master_data *drv_data = spi_master_get_devdata(spi->master);  in bfin_spi_transfer()  local
938 	spin_lock_irqsave(&drv_data->lock, flags);  in bfin_spi_transfer()
940 	if (!drv_data->running) {  in bfin_spi_transfer()
941 		spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_transfer()
950 	list_add_tail(&msg->queue, &drv_data->queue);  in bfin_spi_transfer()
952 	if (drv_data->running && !drv_data->busy)  in bfin_spi_transfer()
953 		queue_work(drv_data->workqueue, &drv_data->pump_messages);  in bfin_spi_transfer()
955 	spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_transfer()
981 	struct bfin_spi_master_data *drv_data = spi_master_get_devdata(spi->master);  in bfin_spi_setup()  local
1018 		    && drv_data->master_info->enable_dma;  in bfin_spi_setup()
1063 	if (chip->enable_dma && !drv_data->dma_requested) {  in bfin_spi_setup()
1065 		ret = request_dma(drv_data->dma_channel, "BFIN_SPI_DMA");  in bfin_spi_setup()
1071 		drv_data->dma_requested = 1;  in bfin_spi_setup()
1073 		ret = set_dma_callback(drv_data->dma_channel,  in bfin_spi_setup()
1074 			bfin_spi_dma_irq_handler, drv_data);  in bfin_spi_setup()
1079 		dma_disable_irq(drv_data->dma_channel);  in bfin_spi_setup()
1082 	if (chip->pio_interrupt && !drv_data->irq_requested) {  in bfin_spi_setup()
1083 		ret = request_irq(drv_data->spi_irq, bfin_spi_pio_irq_handler,  in bfin_spi_setup()
1084 			0, "BFIN_SPI", drv_data);  in bfin_spi_setup()
1089 		drv_data->irq_requested = 1;  in bfin_spi_setup()
1091 		disable_irq(drv_data->spi_irq);  in bfin_spi_setup()
1123 	bfin_spi_cs_enable(drv_data, chip);  in bfin_spi_setup()
1124 	bfin_spi_cs_deactive(drv_data, chip);  in bfin_spi_setup()
1136 		if (drv_data->dma_requested)  in bfin_spi_setup()
1137 			free_dma(drv_data->dma_channel);  in bfin_spi_setup()
1138 		drv_data->dma_requested = 0;  in bfin_spi_setup()
1155 	struct bfin_spi_master_data *drv_data = spi_master_get_devdata(spi->master);  in bfin_spi_cleanup()  local
1163 		bfin_spi_cs_disable(drv_data, chip);  in bfin_spi_cleanup()
1172 static int bfin_spi_init_queue(struct bfin_spi_master_data *drv_data)  in bfin_spi_init_queue()  argument
1174 	INIT_LIST_HEAD(&drv_data->queue);  in bfin_spi_init_queue()
1175 	spin_lock_init(&drv_data->lock);  in bfin_spi_init_queue()
1177 	drv_data->running = false;  in bfin_spi_init_queue()
1178 	drv_data->busy = 0;  in bfin_spi_init_queue()
1181 	tasklet_init(&drv_data->pump_transfers,  in bfin_spi_init_queue()
1182 		     bfin_spi_pump_transfers, (unsigned long)drv_data);  in bfin_spi_init_queue()
1185 	INIT_WORK(&drv_data->pump_messages, bfin_spi_pump_messages);  in bfin_spi_init_queue()
1186 	drv_data->workqueue = create_singlethread_workqueue(  in bfin_spi_init_queue()
1187 				dev_name(drv_data->master->dev.parent));  in bfin_spi_init_queue()
1188 	if (drv_data->workqueue == NULL)  in bfin_spi_init_queue()
1194 static int bfin_spi_start_queue(struct bfin_spi_master_data *drv_data)  in bfin_spi_start_queue()  argument
1198 	spin_lock_irqsave(&drv_data->lock, flags);  in bfin_spi_start_queue()
1200 	if (drv_data->running || drv_data->busy) {  in bfin_spi_start_queue()
1201 		spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_start_queue()
1205 	drv_data->running = true;  in bfin_spi_start_queue()
1206 	drv_data->cur_msg = NULL;  in bfin_spi_start_queue()
1207 	drv_data->cur_transfer = NULL;  in bfin_spi_start_queue()
1208 	drv_data->cur_chip = NULL;  in bfin_spi_start_queue()
1209 	spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_start_queue()
1211 	queue_work(drv_data->workqueue, &drv_data->pump_messages);  in bfin_spi_start_queue()
1216 static int bfin_spi_stop_queue(struct bfin_spi_master_data *drv_data)  in bfin_spi_stop_queue()  argument
1222 	spin_lock_irqsave(&drv_data->lock, flags);  in bfin_spi_stop_queue()
1230 	drv_data->running = false;  in bfin_spi_stop_queue()
1231 	while ((!list_empty(&drv_data->queue) || drv_data->busy) && limit--) {  in bfin_spi_stop_queue()
1232 		spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_stop_queue()
1234 		spin_lock_irqsave(&drv_data->lock, flags);  in bfin_spi_stop_queue()
1237 	if (!list_empty(&drv_data->queue) || drv_data->busy)  in bfin_spi_stop_queue()
1240 	spin_unlock_irqrestore(&drv_data->lock, flags);  in bfin_spi_stop_queue()
1245 static int bfin_spi_destroy_queue(struct bfin_spi_master_data *drv_data)  in bfin_spi_destroy_queue()  argument
1249 	status = bfin_spi_stop_queue(drv_data);  in bfin_spi_destroy_queue()
1253 	destroy_workqueue(drv_data->workqueue);  in bfin_spi_destroy_queue()
1263 	struct bfin_spi_master_data *drv_data;  in bfin_spi_probe()  local
1270 	master = spi_alloc_master(dev, sizeof(*drv_data));  in bfin_spi_probe()
1276 	drv_data = spi_master_get_devdata(master);  in bfin_spi_probe()
1277 	drv_data->master = master;  in bfin_spi_probe()
1278 	drv_data->master_info = platform_info;  in bfin_spi_probe()
1279 	drv_data->pdev = pdev;  in bfin_spi_probe()
1280 	drv_data->pin_req = platform_info->pin_req;  in bfin_spi_probe()
1299 	drv_data->regs = ioremap(res->start, resource_size(res));  in bfin_spi_probe()
1300 	if (drv_data->regs == NULL) {  in bfin_spi_probe()
1312 	drv_data->dma_channel = res->start;  in bfin_spi_probe()
1314 	drv_data->spi_irq = platform_get_irq(pdev, 0);  in bfin_spi_probe()
1315 	if (drv_data->spi_irq < 0) {  in bfin_spi_probe()
1322 	status = bfin_spi_init_queue(drv_data);  in bfin_spi_probe()
1328 	status = bfin_spi_start_queue(drv_data);  in bfin_spi_probe()
1334 	status = peripheral_request_list(drv_data->pin_req, DRV_NAME);  in bfin_spi_probe()
1343 	bfin_write(&drv_data->regs->ctl, BIT_CTL_CPHA | BIT_CTL_MASTER);  in bfin_spi_probe()
1344 	bfin_write(&drv_data->regs->flg, 0xFF00);  in bfin_spi_probe()
1347 	platform_set_drvdata(pdev, drv_data);  in bfin_spi_probe()
1355 		DRV_DESC, DRV_VERSION, drv_data->regs,  in bfin_spi_probe()
1356 		drv_data->dma_channel);  in bfin_spi_probe()
1360 	bfin_spi_destroy_queue(drv_data);  in bfin_spi_probe()
1362 	iounmap(drv_data->regs);  in bfin_spi_probe()
1373 	struct bfin_spi_master_data *drv_data = platform_get_drvdata(pdev);  in bfin_spi_remove()  local
1376 	if (!drv_data)  in bfin_spi_remove()
1380 	status = bfin_spi_destroy_queue(drv_data);  in bfin_spi_remove()
1385 	bfin_spi_disable(drv_data);  in bfin_spi_remove()
1388 	if (drv_data->master_info->enable_dma) {  in bfin_spi_remove()
1389 		if (dma_channel_active(drv_data->dma_channel))  in bfin_spi_remove()
1390 			free_dma(drv_data->dma_channel);  in bfin_spi_remove()
1393 	if (drv_data->irq_requested) {  in bfin_spi_remove()
1394 		free_irq(drv_data->spi_irq, drv_data);  in bfin_spi_remove()
1395 		drv_data->irq_requested = 0;  in bfin_spi_remove()
1399 	spi_unregister_master(drv_data->master);  in bfin_spi_remove()
1401 	peripheral_free_list(drv_data->pin_req);  in bfin_spi_remove()
1409 	struct bfin_spi_master_data *drv_data = dev_get_drvdata(dev);  in bfin_spi_suspend()  local
1412 	status = bfin_spi_stop_queue(drv_data);  in bfin_spi_suspend()
1416 	drv_data->ctrl_reg = bfin_read(&drv_data->regs->ctl);  in bfin_spi_suspend()
1417 	drv_data->flag_reg = bfin_read(&drv_data->regs->flg);  in bfin_spi_suspend()
1422 	bfin_write(&drv_data->regs->ctl, BIT_CTL_CPHA | BIT_CTL_MASTER);  in bfin_spi_suspend()
1423 	bfin_write(&drv_data->regs->flg, 0xFF00);  in bfin_spi_suspend()
1430 	struct bfin_spi_master_data *drv_data = dev_get_drvdata(dev);  in bfin_spi_resume()  local
1433 	bfin_write(&drv_data->regs->ctl, drv_data->ctrl_reg);  in bfin_spi_resume()
1434 	bfin_write(&drv_data->regs->flg, drv_data->flag_reg);  in bfin_spi_resume()
1437 	status = bfin_spi_start_queue(drv_data);  in bfin_spi_resume()