55 static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch)  in mic_dma_hw_ring_inc_head()  argument
57 	ch->head = mic_dma_hw_ring_inc(ch->head);  in mic_dma_hw_ring_inc_head()
89 static void mic_dma_cleanup(struct mic_dma_chan *ch)  in mic_dma_cleanup()  argument
95 	spin_lock(&ch->cleanup_lock);  in mic_dma_cleanup()
96 	tail = mic_dma_read_cmp_cnt(ch);  in mic_dma_cleanup()
103 	for (last_tail = ch->last_tail; tail != last_tail;) {  in mic_dma_cleanup()
104 		tx = &ch->tx_array[last_tail];  in mic_dma_cleanup()
116 	ch->last_tail = last_tail;  in mic_dma_cleanup()
117 	spin_unlock(&ch->cleanup_lock);  in mic_dma_cleanup()
132 static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required)  in mic_dma_avail_desc_ring_space()  argument
134 	struct device *dev = mic_dma_ch_to_device(ch);  in mic_dma_avail_desc_ring_space()
137 	count = mic_dma_ring_count(ch->head, ch->last_tail);  in mic_dma_avail_desc_ring_space()
139 		mic_dma_cleanup(ch);  in mic_dma_avail_desc_ring_space()
140 		count = mic_dma_ring_count(ch->head, ch->last_tail);  in mic_dma_avail_desc_ring_space()
154 static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src,  in mic_dma_prog_memcpy_desc()  argument
158 	size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size;  in mic_dma_prog_memcpy_desc()
166 	ret = mic_dma_avail_desc_ring_space(ch, num_desc);  in mic_dma_prog_memcpy_desc()
171 		mic_dma_memcpy_desc(&ch->desc_ring[ch->head],  in mic_dma_prog_memcpy_desc()
173 		mic_dma_hw_ring_inc_head(ch);  in mic_dma_prog_memcpy_desc()
182 static void mic_dma_prog_intr(struct mic_dma_chan *ch)  in mic_dma_prog_intr()  argument
184 	mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,  in mic_dma_prog_intr()
185 				 ch->status_dest_micpa, false);  in mic_dma_prog_intr()
186 	mic_dma_hw_ring_inc_head(ch);  in mic_dma_prog_intr()
187 	mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,  in mic_dma_prog_intr()
188 				 ch->status_dest_micpa, true);  in mic_dma_prog_intr()
189 	mic_dma_hw_ring_inc_head(ch);  in mic_dma_prog_intr()
193 static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src,  in mic_dma_do_dma()  argument
196 	if (len && -ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len)) {  in mic_dma_do_dma()
200 		int ret = mic_dma_avail_desc_ring_space(ch, 3);  in mic_dma_do_dma()
208 		mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,  in mic_dma_do_dma()
209 					 ch->status_dest_micpa, false);  in mic_dma_do_dma()
210 		mic_dma_hw_ring_inc_head(ch);  in mic_dma_do_dma()
214 		mic_dma_prog_intr(ch);  in mic_dma_do_dma()
219 static inline void mic_dma_issue_pending(struct dma_chan *ch)  in mic_dma_issue_pending()  argument
221 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_issue_pending()
244 static inline void mic_dma_update_pending(struct mic_dma_chan *ch)  in mic_dma_update_pending()  argument
246 	if (mic_dma_ring_count(ch->issued, ch->submitted)  in mic_dma_update_pending()
248 		mic_dma_issue_pending(&ch->api_ch);  in mic_dma_update_pending()
271 allocate_tx(struct mic_dma_chan *ch)  in allocate_tx()  argument
273 	u32 idx = mic_dma_hw_ring_dec(ch->head);  in allocate_tx()
274 	struct dma_async_tx_descriptor *tx = &ch->tx_array[idx];  in allocate_tx()
276 	dma_async_tx_descriptor_init(tx, &ch->api_ch);  in allocate_tx()
283 mic_dma_prep_status_lock(struct dma_chan *ch, dma_addr_t dst, u64 src_val,  in mic_dma_prep_status_lock()  argument
286 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_prep_status_lock()
314 mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest,  in mic_dma_prep_memcpy_lock()  argument
317 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_prep_memcpy_lock()
334 mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags)  in mic_dma_prep_interrupt_lock()  argument
336 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_prep_interrupt_lock()
349 mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie,  in mic_dma_tx_status()  argument
352 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_tx_status()
354 	if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate))  in mic_dma_tx_status()
357 	return dma_cookie_status(ch, cookie, txstate);  in mic_dma_tx_status()
368 	struct mic_dma_chan *ch = ((struct mic_dma_chan *)data);  in mic_dma_intr_handler()  local
370 	mic_dma_ack_interrupt(ch);  in mic_dma_intr_handler()
374 static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch)  in mic_dma_alloc_desc_ring()  argument
376 	u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);  in mic_dma_alloc_desc_ring()
377 	struct device *dev = &to_mbus_device(ch)->dev;  in mic_dma_alloc_desc_ring()
380 	ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL);  in mic_dma_alloc_desc_ring()
382 	if (!ch->desc_ring)  in mic_dma_alloc_desc_ring()
385 	ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring,  in mic_dma_alloc_desc_ring()
387 	if (dma_mapping_error(dev, ch->desc_ring_micpa))  in mic_dma_alloc_desc_ring()
390 	ch->tx_array = vzalloc(MIC_DMA_DESC_RX_SIZE * sizeof(*ch->tx_array));  in mic_dma_alloc_desc_ring()
391 	if (!ch->tx_array)  in mic_dma_alloc_desc_ring()
395 	dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size,  in mic_dma_alloc_desc_ring()
398 	kfree(ch->desc_ring);  in mic_dma_alloc_desc_ring()
402 static void mic_dma_free_desc_ring(struct mic_dma_chan *ch)  in mic_dma_free_desc_ring()  argument
404 	u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);  in mic_dma_free_desc_ring()
406 	vfree(ch->tx_array);  in mic_dma_free_desc_ring()
408 	dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa,  in mic_dma_free_desc_ring()
410 	kfree(ch->desc_ring);  in mic_dma_free_desc_ring()
411 	ch->desc_ring = NULL;  in mic_dma_free_desc_ring()
414 static void mic_dma_free_status_dest(struct mic_dma_chan *ch)  in mic_dma_free_status_dest()  argument
416 	dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa,  in mic_dma_free_status_dest()
418 	kfree(ch->status_dest);  in mic_dma_free_status_dest()
421 static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch)  in mic_dma_alloc_status_dest()  argument
423 	struct device *dev = &to_mbus_device(ch)->dev;  in mic_dma_alloc_status_dest()
425 	ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL);  in mic_dma_alloc_status_dest()
426 	if (!ch->status_dest)  in mic_dma_alloc_status_dest()
428 	ch->status_dest_micpa = dma_map_single(dev, ch->status_dest,  in mic_dma_alloc_status_dest()
430 	if (dma_mapping_error(dev, ch->status_dest_micpa)) {  in mic_dma_alloc_status_dest()
431 		kfree(ch->status_dest);  in mic_dma_alloc_status_dest()
432 		ch->status_dest = NULL;  in mic_dma_alloc_status_dest()
438 static int mic_dma_check_chan(struct mic_dma_chan *ch)  in mic_dma_check_chan()  argument
440 	if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) ||  in mic_dma_check_chan()
441 	    mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) {  in mic_dma_check_chan()
442 		mic_dma_disable_chan(ch);  in mic_dma_check_chan()
443 		mic_dma_chan_mask_intr(ch);  in mic_dma_check_chan()
444 		dev_err(mic_dma_ch_to_device(ch),  in mic_dma_check_chan()
446 			__func__, __LINE__, ch->ch_num);  in mic_dma_check_chan()
452 static int mic_dma_chan_setup(struct mic_dma_chan *ch)  in mic_dma_chan_setup()  argument
454 	if (MIC_DMA_CHAN_MIC == ch->owner)  in mic_dma_chan_setup()
455 		mic_dma_chan_set_owner(ch);  in mic_dma_chan_setup()
456 	mic_dma_disable_chan(ch);  in mic_dma_chan_setup()
457 	mic_dma_chan_mask_intr(ch);  in mic_dma_chan_setup()
458 	mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0);  in mic_dma_chan_setup()
459 	mic_dma_chan_set_desc_ring(ch);  in mic_dma_chan_setup()
460 	ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR);  in mic_dma_chan_setup()
461 	ch->head = ch->last_tail;  in mic_dma_chan_setup()
462 	ch->issued = 0;  in mic_dma_chan_setup()
463 	mic_dma_chan_unmask_intr(ch);  in mic_dma_chan_setup()
464 	mic_dma_enable_chan(ch);  in mic_dma_chan_setup()
465 	return mic_dma_check_chan(ch);  in mic_dma_chan_setup()
468 static void mic_dma_chan_destroy(struct mic_dma_chan *ch)  in mic_dma_chan_destroy()  argument
470 	mic_dma_disable_chan(ch);  in mic_dma_chan_destroy()
471 	mic_dma_chan_mask_intr(ch);  in mic_dma_chan_destroy()
479 static int mic_dma_setup_irq(struct mic_dma_chan *ch)  in mic_dma_setup_irq()  argument
481 	ch->cookie =  in mic_dma_setup_irq()
482 		to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),  in mic_dma_setup_irq()
484 			"mic dma_channel", ch, ch->ch_num);  in mic_dma_setup_irq()
485 	if (IS_ERR(ch->cookie))  in mic_dma_setup_irq()
486 		return IS_ERR(ch->cookie);  in mic_dma_setup_irq()
490 static inline void mic_dma_free_irq(struct mic_dma_chan *ch)  in mic_dma_free_irq()  argument
492 	to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch);  in mic_dma_free_irq()
495 static int mic_dma_chan_init(struct mic_dma_chan *ch)  in mic_dma_chan_init()  argument
497 	int ret = mic_dma_alloc_desc_ring(ch);  in mic_dma_chan_init()
501 	ret = mic_dma_alloc_status_dest(ch);  in mic_dma_chan_init()
504 	ret = mic_dma_chan_setup(ch);  in mic_dma_chan_init()
509 	mic_dma_free_status_dest(ch);  in mic_dma_chan_init()
511 	mic_dma_free_desc_ring(ch);  in mic_dma_chan_init()
516 static int mic_dma_drain_chan(struct mic_dma_chan *ch)  in mic_dma_drain_chan()  argument
522 	tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE);  in mic_dma_drain_chan()
532 		err = dma_sync_wait(&ch->api_ch, cookie);  in mic_dma_drain_chan()
534 		dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n",  in mic_dma_drain_chan()
535 			__func__, __LINE__, ch->ch_num);  in mic_dma_drain_chan()
539 	mic_dma_cleanup(ch);  in mic_dma_drain_chan()
543 static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch)  in mic_dma_chan_uninit()  argument
545 	mic_dma_chan_destroy(ch);  in mic_dma_chan_uninit()
546 	mic_dma_cleanup(ch);  in mic_dma_chan_uninit()
547 	mic_dma_free_status_dest(ch);  in mic_dma_chan_uninit()
548 	mic_dma_free_desc_ring(ch);  in mic_dma_chan_uninit()
555 	struct mic_dma_chan *ch;  in mic_dma_init()  local
560 		ch = &mic_dma_dev->mic_ch[i];  in mic_dma_init()
561 		data = (unsigned long)ch;  in mic_dma_init()
562 		ch->ch_num = i;  in mic_dma_init()
563 		ch->owner = owner;  in mic_dma_init()
564 		spin_lock_init(&ch->cleanup_lock);  in mic_dma_init()
565 		spin_lock_init(&ch->prep_lock);  in mic_dma_init()
566 		spin_lock_init(&ch->issue_lock);  in mic_dma_init()
567 		ret = mic_dma_setup_irq(ch);  in mic_dma_init()
574 		mic_dma_free_irq(ch);  in mic_dma_init()
581 	struct mic_dma_chan *ch;  in mic_dma_uninit()  local
584 		ch = &mic_dma_dev->mic_ch[i];  in mic_dma_uninit()
585 		mic_dma_free_irq(ch);  in mic_dma_uninit()
589 static int mic_dma_alloc_chan_resources(struct dma_chan *ch)  in mic_dma_alloc_chan_resources()  argument
591 	int ret = mic_dma_chan_init(to_mic_dma_chan(ch));  in mic_dma_alloc_chan_resources()
597 static void mic_dma_free_chan_resources(struct dma_chan *ch)  in mic_dma_free_chan_resources()  argument
599 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_free_chan_resources()
696 	struct mic_dma_chan *ch;  in mic_dma_reg_seq_show()  local
706 		ch = &mic_dma_dev->mic_ch[i];  in mic_dma_reg_seq_show()
707 		chan_num = ch->ch_num;  in mic_dma_reg_seq_show()
710 			   mic_dma_read_reg(ch, MIC_DMA_REG_DCAR),  in mic_dma_reg_seq_show()
711 			   mic_dma_read_reg(ch, MIC_DMA_REG_DTPR),  in mic_dma_reg_seq_show()
712 			   mic_dma_read_reg(ch, MIC_DMA_REG_DHPR),  in mic_dma_reg_seq_show()
713 			   mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI));  in mic_dma_reg_seq_show()
715 			   mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO),  in mic_dma_reg_seq_show()
716 			   mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR),  in mic_dma_reg_seq_show()
717 			   mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK),  in mic_dma_reg_seq_show()
718 			   mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT));  in mic_dma_reg_seq_show()