131 static int reset_channel(struct device *dev, int ch)  in reset_channel()  argument
138 		setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,  in reset_channel()
141 		while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &  in reset_channel()
145 		setbits32(priv->chan[ch].reg + TALITOS_CCCR,  in reset_channel()
148 		while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &  in reset_channel()
154 		dev_err(dev, "failed to reset channel %d\n", ch);  in reset_channel()
159 	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |  in reset_channel()
164 		setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,  in reset_channel()
202 	int ch, err;  in init_device()  local
220 	for (ch = 0; ch < priv->num_channels; ch++) {  in init_device()
221 		err = reset_channel(dev, ch);  in init_device()
257 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,  in talitos_submit()  argument
269 	spin_lock_irqsave(&priv->chan[ch].head_lock, flags);  in talitos_submit()
271 	if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {  in talitos_submit()
273 		spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);  in talitos_submit()
277 	head = priv->chan[ch].head;  in talitos_submit()
278 	request = &priv->chan[ch].fifo[head];  in talitos_submit()
296 	priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);  in talitos_submit()
303 	out_be32(priv->chan[ch].reg + TALITOS_FF,  in talitos_submit()
305 	out_be32(priv->chan[ch].reg + TALITOS_FF_LO,  in talitos_submit()
308 	spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);  in talitos_submit()
317 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)  in flush_channel()  argument
325 	spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);  in flush_channel()
327 	tail = priv->chan[ch].tail;  in flush_channel()
328 	while (priv->chan[ch].fifo[tail].desc) {  in flush_channel()
331 		request = &priv->chan[ch].fifo[tail];  in flush_channel()
359 		priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);  in flush_channel()
361 		spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);  in flush_channel()
363 		atomic_dec(&priv->chan[ch].submit_count);  in flush_channel()
370 		spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);  in flush_channel()
371 		tail = priv->chan[ch].tail;  in flush_channel()
374 	spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);  in flush_channel()
407 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
436 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
443 static u32 current_desc_hdr(struct device *dev, int ch)  in DEF_TALITOS2_DONE()
449 	cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;  in DEF_TALITOS2_DONE()
450 	cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);  in DEF_TALITOS2_DONE()
457 	tail = priv->chan[ch].tail;  in DEF_TALITOS2_DONE()
460 	while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {  in DEF_TALITOS2_DONE()
468 	return priv->chan[ch].fifo[iter].desc->hdr;  in DEF_TALITOS2_DONE()
474 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)  in report_eu_error()  argument
480 		desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);  in report_eu_error()
542 			in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),  in report_eu_error()
543 			in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));  in report_eu_error()
553 	int ch, error, reset_dev = 0;  in talitos_error()  local
558 	for (ch = 0; ch < priv->num_channels; ch++) {  in talitos_error()
562 			if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))  in talitos_error()
565 			if (!(isr & (1 << (ch * 2 + 1))))  in talitos_error()
571 		v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);  in talitos_error()
597 			report_eu_error(dev, ch, current_desc_hdr(dev, ch));  in talitos_error()
609 		flush_channel(dev, ch, error, reset_ch);  in talitos_error()
612 			reset_channel(dev, ch);  in talitos_error()
614 			setbits32(priv->chan[ch].reg + TALITOS_CCCR,  in talitos_error()
616 			setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);  in talitos_error()
617 			while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &  in talitos_error()
622 					ch);  in talitos_error()
637 		for (ch = 0; ch < priv->num_channels; ch++)  in talitos_error()
638 			flush_channel(dev, ch, -EIO, 1);  in talitos_error()
678 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
713 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
812 	int ch;  member
1212 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in ipsec_esp()
1598 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in common_nonsnoop()
1783 	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);  in common_nonsnoop_hash()
2608 	ctx->ch = atomic_inc_return(&priv->last_chan) &  in talitos_init_common()