91 static int reset_channel(struct device *dev, int ch)  in reset_channel()  argument
96 	setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET);  in reset_channel()
98 	while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET)  in reset_channel()
103 		dev_err(dev, "failed to reset channel %d\n", ch);  in reset_channel()
108 	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |  in reset_channel()
113 		setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,  in reset_channel()
150 	int ch, err;  in init_device()  local
167 	for (ch = 0; ch < priv->num_channels; ch++) {  in init_device()
168 		err = reset_channel(dev, ch);  in init_device()
197 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,  in talitos_submit()  argument
208 	spin_lock_irqsave(&priv->chan[ch].head_lock, flags);  in talitos_submit()
210 	if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {  in talitos_submit()
212 		spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);  in talitos_submit()
216 	head = priv->chan[ch].head;  in talitos_submit()
217 	request = &priv->chan[ch].fifo[head];  in talitos_submit()
226 	priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);  in talitos_submit()
233 	out_be32(priv->chan[ch].reg + TALITOS_FF,  in talitos_submit()
235 	out_be32(priv->chan[ch].reg + TALITOS_FF_LO,  in talitos_submit()
238 	spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);  in talitos_submit()
247 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)  in flush_channel()  argument
254 	spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);  in flush_channel()
256 	tail = priv->chan[ch].tail;  in flush_channel()
257 	while (priv->chan[ch].fifo[tail].desc) {  in flush_channel()
258 		request = &priv->chan[ch].fifo[tail];  in flush_channel()
284 		priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);  in flush_channel()
286 		spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);  in flush_channel()
288 		atomic_dec(&priv->chan[ch].submit_count);  in flush_channel()
295 		spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);  in flush_channel()
296 		tail = priv->chan[ch].tail;  in flush_channel()
299 	spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);  in flush_channel()
331 DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE)
338 static u32 current_desc_hdr(struct device *dev, int ch)  in DEF_TALITOS_DONE()
344 	cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;  in DEF_TALITOS_DONE()
345 	cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);  in DEF_TALITOS_DONE()
352 	tail = priv->chan[ch].tail;  in DEF_TALITOS_DONE()
355 	while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {  in DEF_TALITOS_DONE()
363 	return priv->chan[ch].fifo[iter].desc->hdr;  in DEF_TALITOS_DONE()
369 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)  in report_eu_error()  argument
375 		desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);  in report_eu_error()
437 			in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),  in report_eu_error()
438 			in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));  in report_eu_error()
448 	int ch, error, reset_dev = 0, reset_ch = 0;  in talitos_error()  local
451 	for (ch = 0; ch < priv->num_channels; ch++) {  in talitos_error()
453 		if (!(isr & (1 << (ch * 2 + 1))))  in talitos_error()
458 		v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR);  in talitos_error()
459 		v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);  in talitos_error()
482 			report_eu_error(dev, ch, current_desc_hdr(dev, ch));  in talitos_error()
492 		flush_channel(dev, ch, error, reset_ch);  in talitos_error()
495 			reset_channel(dev, ch);  in talitos_error()
497 			setbits32(priv->chan[ch].reg + TALITOS_CCCR,  in talitos_error()
499 			setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);  in talitos_error()
500 			while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &  in talitos_error()
505 					ch);  in talitos_error()
515 		for (ch = 0; ch < priv->num_channels; ch++)  in talitos_error()
516 			flush_channel(dev, ch, -EIO, 1);  in talitos_error()
555 DEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0)
642 	int ch;  member
1082 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in ipsec_esp()
1453 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in common_nonsnoop()
1633 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in common_nonsnoop_hash()
2399 	ctx->ch = atomic_inc_return(&priv->last_chan) &  in talitos_cra_init()