54 static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch)  in mic_dma_hw_ring_inc_head()  argument
56 	ch->head = mic_dma_hw_ring_inc(ch->head);  in mic_dma_hw_ring_inc_head()
88 static void mic_dma_cleanup(struct mic_dma_chan *ch)  in mic_dma_cleanup()  argument
94 	spin_lock(&ch->cleanup_lock);  in mic_dma_cleanup()
95 	tail = mic_dma_read_cmp_cnt(ch);  in mic_dma_cleanup()
102 	for (last_tail = ch->last_tail; tail != last_tail;) {  in mic_dma_cleanup()
103 		tx = &ch->tx_array[last_tail];  in mic_dma_cleanup()
115 	ch->last_tail = last_tail;  in mic_dma_cleanup()
116 	spin_unlock(&ch->cleanup_lock);  in mic_dma_cleanup()
131 static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required)  in mic_dma_avail_desc_ring_space()  argument
133 	struct device *dev = mic_dma_ch_to_device(ch);  in mic_dma_avail_desc_ring_space()
136 	count = mic_dma_ring_count(ch->head, ch->last_tail);  in mic_dma_avail_desc_ring_space()
138 		mic_dma_cleanup(ch);  in mic_dma_avail_desc_ring_space()
139 		count = mic_dma_ring_count(ch->head, ch->last_tail);  in mic_dma_avail_desc_ring_space()
153 static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src,  in mic_dma_prog_memcpy_desc()  argument
157 	size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size;  in mic_dma_prog_memcpy_desc()
165 	ret = mic_dma_avail_desc_ring_space(ch, num_desc);  in mic_dma_prog_memcpy_desc()
170 		mic_dma_memcpy_desc(&ch->desc_ring[ch->head],  in mic_dma_prog_memcpy_desc()
172 		mic_dma_hw_ring_inc_head(ch);  in mic_dma_prog_memcpy_desc()
181 static void mic_dma_prog_intr(struct mic_dma_chan *ch)  in mic_dma_prog_intr()  argument
183 	mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,  in mic_dma_prog_intr()
184 				 ch->status_dest_micpa, false);  in mic_dma_prog_intr()
185 	mic_dma_hw_ring_inc_head(ch);  in mic_dma_prog_intr()
186 	mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,  in mic_dma_prog_intr()
187 				 ch->status_dest_micpa, true);  in mic_dma_prog_intr()
188 	mic_dma_hw_ring_inc_head(ch);  in mic_dma_prog_intr()
192 static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src,  in mic_dma_do_dma()  argument
195 	if (-ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len))  in mic_dma_do_dma()
199 		mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,  in mic_dma_do_dma()
200 					 ch->status_dest_micpa, false);  in mic_dma_do_dma()
201 		mic_dma_hw_ring_inc_head(ch);  in mic_dma_do_dma()
205 		mic_dma_prog_intr(ch);  in mic_dma_do_dma()
210 static inline void mic_dma_issue_pending(struct dma_chan *ch)  in mic_dma_issue_pending()  argument
212 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_issue_pending()
235 static inline void mic_dma_update_pending(struct mic_dma_chan *ch)  in mic_dma_update_pending()  argument
237 	if (mic_dma_ring_count(ch->issued, ch->submitted)  in mic_dma_update_pending()
239 		mic_dma_issue_pending(&ch->api_ch);  in mic_dma_update_pending()
262 allocate_tx(struct mic_dma_chan *ch)  in allocate_tx()  argument
264 	u32 idx = mic_dma_hw_ring_dec(ch->head);  in allocate_tx()
265 	struct dma_async_tx_descriptor *tx = &ch->tx_array[idx];  in allocate_tx()
267 	dma_async_tx_descriptor_init(tx, &ch->api_ch);  in allocate_tx()
278 mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest,  in mic_dma_prep_memcpy_lock()  argument
281 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_prep_memcpy_lock()
298 mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags)  in mic_dma_prep_interrupt_lock()  argument
300 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_prep_interrupt_lock()
313 mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie,  in mic_dma_tx_status()  argument
316 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_tx_status()
318 	if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate))  in mic_dma_tx_status()
321 	return dma_cookie_status(ch, cookie, txstate);  in mic_dma_tx_status()
332 	struct mic_dma_chan *ch = ((struct mic_dma_chan *)data);  in mic_dma_intr_handler()  local
334 	mic_dma_ack_interrupt(ch);  in mic_dma_intr_handler()
338 static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch)  in mic_dma_alloc_desc_ring()  argument
340 	u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);  in mic_dma_alloc_desc_ring()
341 	struct device *dev = &to_mbus_device(ch)->dev;  in mic_dma_alloc_desc_ring()
344 	ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL);  in mic_dma_alloc_desc_ring()
346 	if (!ch->desc_ring)  in mic_dma_alloc_desc_ring()
349 	ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring,  in mic_dma_alloc_desc_ring()
351 	if (dma_mapping_error(dev, ch->desc_ring_micpa))  in mic_dma_alloc_desc_ring()
354 	ch->tx_array = vzalloc(MIC_DMA_DESC_RX_SIZE * sizeof(*ch->tx_array));  in mic_dma_alloc_desc_ring()
355 	if (!ch->tx_array)  in mic_dma_alloc_desc_ring()
359 	dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size,  in mic_dma_alloc_desc_ring()
362 	kfree(ch->desc_ring);  in mic_dma_alloc_desc_ring()
366 static void mic_dma_free_desc_ring(struct mic_dma_chan *ch)  in mic_dma_free_desc_ring()  argument
368 	u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);  in mic_dma_free_desc_ring()
370 	vfree(ch->tx_array);  in mic_dma_free_desc_ring()
372 	dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa,  in mic_dma_free_desc_ring()
374 	kfree(ch->desc_ring);  in mic_dma_free_desc_ring()
375 	ch->desc_ring = NULL;  in mic_dma_free_desc_ring()
378 static void mic_dma_free_status_dest(struct mic_dma_chan *ch)  in mic_dma_free_status_dest()  argument
380 	dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa,  in mic_dma_free_status_dest()
382 	kfree(ch->status_dest);  in mic_dma_free_status_dest()
385 static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch)  in mic_dma_alloc_status_dest()  argument
387 	struct device *dev = &to_mbus_device(ch)->dev;  in mic_dma_alloc_status_dest()
389 	ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL);  in mic_dma_alloc_status_dest()
390 	if (!ch->status_dest)  in mic_dma_alloc_status_dest()
392 	ch->status_dest_micpa = dma_map_single(dev, ch->status_dest,  in mic_dma_alloc_status_dest()
394 	if (dma_mapping_error(dev, ch->status_dest_micpa)) {  in mic_dma_alloc_status_dest()
395 		kfree(ch->status_dest);  in mic_dma_alloc_status_dest()
396 		ch->status_dest = NULL;  in mic_dma_alloc_status_dest()
402 static int mic_dma_check_chan(struct mic_dma_chan *ch)  in mic_dma_check_chan()  argument
404 	if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) ||  in mic_dma_check_chan()
405 	    mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) {  in mic_dma_check_chan()
406 		mic_dma_disable_chan(ch);  in mic_dma_check_chan()
407 		mic_dma_chan_mask_intr(ch);  in mic_dma_check_chan()
408 		dev_err(mic_dma_ch_to_device(ch),  in mic_dma_check_chan()
410 			__func__, __LINE__, ch->ch_num);  in mic_dma_check_chan()
416 static int mic_dma_chan_setup(struct mic_dma_chan *ch)  in mic_dma_chan_setup()  argument
418 	if (MIC_DMA_CHAN_MIC == ch->owner)  in mic_dma_chan_setup()
419 		mic_dma_chan_set_owner(ch);  in mic_dma_chan_setup()
420 	mic_dma_disable_chan(ch);  in mic_dma_chan_setup()
421 	mic_dma_chan_mask_intr(ch);  in mic_dma_chan_setup()
422 	mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0);  in mic_dma_chan_setup()
423 	mic_dma_chan_set_desc_ring(ch);  in mic_dma_chan_setup()
424 	ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR);  in mic_dma_chan_setup()
425 	ch->head = ch->last_tail;  in mic_dma_chan_setup()
426 	ch->issued = 0;  in mic_dma_chan_setup()
427 	mic_dma_chan_unmask_intr(ch);  in mic_dma_chan_setup()
428 	mic_dma_enable_chan(ch);  in mic_dma_chan_setup()
429 	return mic_dma_check_chan(ch);  in mic_dma_chan_setup()
432 static void mic_dma_chan_destroy(struct mic_dma_chan *ch)  in mic_dma_chan_destroy()  argument
434 	mic_dma_disable_chan(ch);  in mic_dma_chan_destroy()
435 	mic_dma_chan_mask_intr(ch);  in mic_dma_chan_destroy()
443 static int mic_dma_setup_irq(struct mic_dma_chan *ch)  in mic_dma_setup_irq()  argument
445 	ch->cookie =  in mic_dma_setup_irq()
446 		to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),  in mic_dma_setup_irq()
448 			"mic dma_channel", ch, ch->ch_num);  in mic_dma_setup_irq()
449 	if (IS_ERR(ch->cookie))  in mic_dma_setup_irq()
450 		return IS_ERR(ch->cookie);  in mic_dma_setup_irq()
454 static inline void mic_dma_free_irq(struct mic_dma_chan *ch)  in mic_dma_free_irq()  argument
456 	to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch);  in mic_dma_free_irq()
459 static int mic_dma_chan_init(struct mic_dma_chan *ch)  in mic_dma_chan_init()  argument
461 	int ret = mic_dma_alloc_desc_ring(ch);  in mic_dma_chan_init()
465 	ret = mic_dma_alloc_status_dest(ch);  in mic_dma_chan_init()
468 	ret = mic_dma_chan_setup(ch);  in mic_dma_chan_init()
473 	mic_dma_free_status_dest(ch);  in mic_dma_chan_init()
475 	mic_dma_free_desc_ring(ch);  in mic_dma_chan_init()
480 static int mic_dma_drain_chan(struct mic_dma_chan *ch)  in mic_dma_drain_chan()  argument
486 	tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE);  in mic_dma_drain_chan()
496 		err = dma_sync_wait(&ch->api_ch, cookie);  in mic_dma_drain_chan()
498 		dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n",  in mic_dma_drain_chan()
499 			__func__, __LINE__, ch->ch_num);  in mic_dma_drain_chan()
503 	mic_dma_cleanup(ch);  in mic_dma_drain_chan()
507 static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch)  in mic_dma_chan_uninit()  argument
509 	mic_dma_chan_destroy(ch);  in mic_dma_chan_uninit()
510 	mic_dma_cleanup(ch);  in mic_dma_chan_uninit()
511 	mic_dma_free_status_dest(ch);  in mic_dma_chan_uninit()
512 	mic_dma_free_desc_ring(ch);  in mic_dma_chan_uninit()
519 	struct mic_dma_chan *ch;  in mic_dma_init()  local
524 		ch = &mic_dma_dev->mic_ch[i];  in mic_dma_init()
525 		data = (unsigned long)ch;  in mic_dma_init()
526 		ch->ch_num = i;  in mic_dma_init()
527 		ch->owner = owner;  in mic_dma_init()
528 		spin_lock_init(&ch->cleanup_lock);  in mic_dma_init()
529 		spin_lock_init(&ch->prep_lock);  in mic_dma_init()
530 		spin_lock_init(&ch->issue_lock);  in mic_dma_init()
531 		ret = mic_dma_setup_irq(ch);  in mic_dma_init()
538 		mic_dma_free_irq(ch);  in mic_dma_init()
545 	struct mic_dma_chan *ch;  in mic_dma_uninit()  local
548 		ch = &mic_dma_dev->mic_ch[i];  in mic_dma_uninit()
549 		mic_dma_free_irq(ch);  in mic_dma_uninit()
553 static int mic_dma_alloc_chan_resources(struct dma_chan *ch)  in mic_dma_alloc_chan_resources()  argument
555 	int ret = mic_dma_chan_init(to_mic_dma_chan(ch));  in mic_dma_alloc_chan_resources()
561 static void mic_dma_free_chan_resources(struct dma_chan *ch)  in mic_dma_free_chan_resources()  argument
563 	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);  in mic_dma_free_chan_resources()
658 	struct mic_dma_chan *ch;  in mic_dma_reg_seq_show()  local
668 		ch = &mic_dma_dev->mic_ch[i];  in mic_dma_reg_seq_show()
669 		chan_num = ch->ch_num;  in mic_dma_reg_seq_show()
672 			   mic_dma_read_reg(ch, MIC_DMA_REG_DCAR),  in mic_dma_reg_seq_show()
673 			   mic_dma_read_reg(ch, MIC_DMA_REG_DTPR),  in mic_dma_reg_seq_show()
674 			   mic_dma_read_reg(ch, MIC_DMA_REG_DHPR),  in mic_dma_reg_seq_show()
675 			   mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI));  in mic_dma_reg_seq_show()
677 			   mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO),  in mic_dma_reg_seq_show()
678 			   mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR),  in mic_dma_reg_seq_show()
679 			   mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK),  in mic_dma_reg_seq_show()
680 			   mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT));  in mic_dma_reg_seq_show()