43 #define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
50 #define AES_REG_KEY(dd, x)		((dd)->pdata->key_ofs - \  argument
52 #define AES_REG_IV(dd, x)		((dd)->pdata->iv_ofs + ((x) * 0x04))  argument
54 #define AES_REG_CTRL(dd)		((dd)->pdata->ctrl_ofs)  argument
68 #define AES_REG_DATA_N(dd, x)		((dd)->pdata->data_ofs + ((x) * 0x04))  argument
70 #define AES_REG_REV(dd)			((dd)->pdata->rev_ofs)  argument
72 #define AES_REG_MASK(dd)		((dd)->pdata->mask_ofs)  argument
82 #define AES_REG_IRQ_STATUS(dd)         ((dd)->pdata->irq_status_ofs)  argument
83 #define AES_REG_IRQ_ENABLE(dd)         ((dd)->pdata->irq_enable_ofs)  argument
101 	struct omap_aes_dev *dd;  member
125 	void		(*trigger)(struct omap_aes_dev *dd, int length);
196 #define omap_aes_read(dd, offset)				\  argument
199 	_read_ret = __raw_readl(dd->io_base + offset);		\
205 static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)  in omap_aes_read()  argument
207 	return __raw_readl(dd->io_base + offset);  in omap_aes_read()
212 #define omap_aes_write(dd, offset, value)				\  argument
216 		__raw_writel(value, dd->io_base + offset);		\
219 static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,  in omap_aes_write()  argument
222 	__raw_writel(value, dd->io_base + offset);  in omap_aes_write()
226 static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,  in omap_aes_write_mask()  argument
231 	val = omap_aes_read(dd, offset);  in omap_aes_write_mask()
234 	omap_aes_write(dd, offset, val);  in omap_aes_write_mask()
237 static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,  in omap_aes_write_n()  argument
241 		omap_aes_write(dd, offset, *value);  in omap_aes_write_n()
244 static int omap_aes_hw_init(struct omap_aes_dev *dd)  in omap_aes_hw_init()  argument
246 	if (!(dd->flags & FLAGS_INIT)) {  in omap_aes_hw_init()
247 		dd->flags |= FLAGS_INIT;  in omap_aes_hw_init()
248 		dd->err = 0;  in omap_aes_hw_init()
254 static int omap_aes_write_ctrl(struct omap_aes_dev *dd)  in omap_aes_write_ctrl()  argument
260 	err = omap_aes_hw_init(dd);  in omap_aes_write_ctrl()
264 	key32 = dd->ctx->keylen / sizeof(u32);  in omap_aes_write_ctrl()
268 		omap_aes_write(dd, AES_REG_KEY(dd, i),  in omap_aes_write_ctrl()
269 			__le32_to_cpu(dd->ctx->key[i]));  in omap_aes_write_ctrl()
272 	if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->info)  in omap_aes_write_ctrl()
273 		omap_aes_write_n(dd, AES_REG_IV(dd, 0), dd->req->info, 4);  in omap_aes_write_ctrl()
275 	val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);  in omap_aes_write_ctrl()
276 	if (dd->flags & FLAGS_CBC)  in omap_aes_write_ctrl()
278 	if (dd->flags & FLAGS_CTR)  in omap_aes_write_ctrl()
281 	if (dd->flags & FLAGS_ENCRYPT)  in omap_aes_write_ctrl()
284 	omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK);  in omap_aes_write_ctrl()
289 static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length)  in omap_aes_dma_trigger_omap2()  argument
293 	val = dd->pdata->dma_start;  in omap_aes_dma_trigger_omap2()
295 	if (dd->dma_lch_out != NULL)  in omap_aes_dma_trigger_omap2()
296 		val |= dd->pdata->dma_enable_out;  in omap_aes_dma_trigger_omap2()
297 	if (dd->dma_lch_in != NULL)  in omap_aes_dma_trigger_omap2()
298 		val |= dd->pdata->dma_enable_in;  in omap_aes_dma_trigger_omap2()
300 	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |  in omap_aes_dma_trigger_omap2()
301 	       dd->pdata->dma_start;  in omap_aes_dma_trigger_omap2()
303 	omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask);  in omap_aes_dma_trigger_omap2()
307 static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length)  in omap_aes_dma_trigger_omap4()  argument
309 	omap_aes_write(dd, AES_REG_LENGTH_N(0), length);  in omap_aes_dma_trigger_omap4()
310 	omap_aes_write(dd, AES_REG_LENGTH_N(1), 0);  in omap_aes_dma_trigger_omap4()
312 	omap_aes_dma_trigger_omap2(dd, length);  in omap_aes_dma_trigger_omap4()
315 static void omap_aes_dma_stop(struct omap_aes_dev *dd)  in omap_aes_dma_stop()  argument
319 	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |  in omap_aes_dma_stop()
320 	       dd->pdata->dma_start;  in omap_aes_dma_stop()
322 	omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask);  in omap_aes_dma_stop()
327 	struct omap_aes_dev *dd = NULL, *tmp;  in omap_aes_find_dev()  local
330 	if (!ctx->dd) {  in omap_aes_find_dev()
333 			dd = tmp;  in omap_aes_find_dev()
336 		ctx->dd = dd;  in omap_aes_find_dev()
339 		dd = ctx->dd;  in omap_aes_find_dev()
343 	return dd;  in omap_aes_find_dev()
348 	struct omap_aes_dev *dd = data;  in omap_aes_dma_out_callback()  local
351 	tasklet_schedule(&dd->done_task);  in omap_aes_dma_out_callback()
354 static int omap_aes_dma_init(struct omap_aes_dev *dd)  in omap_aes_dma_init()  argument
359 	dd->dma_lch_out = NULL;  in omap_aes_dma_init()
360 	dd->dma_lch_in = NULL;  in omap_aes_dma_init()
365 	dd->dma_lch_in = dma_request_slave_channel_compat(mask,  in omap_aes_dma_init()
367 							  &dd->dma_in,  in omap_aes_dma_init()
368 							  dd->dev, "rx");  in omap_aes_dma_init()
369 	if (!dd->dma_lch_in) {  in omap_aes_dma_init()
370 		dev_err(dd->dev, "Unable to request in DMA channel\n");  in omap_aes_dma_init()
374 	dd->dma_lch_out = dma_request_slave_channel_compat(mask,  in omap_aes_dma_init()
376 							   &dd->dma_out,  in omap_aes_dma_init()
377 							   dd->dev, "tx");  in omap_aes_dma_init()
378 	if (!dd->dma_lch_out) {  in omap_aes_dma_init()
379 		dev_err(dd->dev, "Unable to request out DMA channel\n");  in omap_aes_dma_init()
386 	dma_release_channel(dd->dma_lch_in);  in omap_aes_dma_init()
393 static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)  in omap_aes_dma_cleanup()  argument
395 	dma_release_channel(dd->dma_lch_out);  in omap_aes_dma_cleanup()
396 	dma_release_channel(dd->dma_lch_in);  in omap_aes_dma_cleanup()
418 	struct omap_aes_dev *dd = ctx->dd;  in omap_aes_crypt_dma()  local
423 	if (dd->pio_only) {  in omap_aes_crypt_dma()
424 		scatterwalk_start(&dd->in_walk, dd->in_sg);  in omap_aes_crypt_dma()
425 		scatterwalk_start(&dd->out_walk, dd->out_sg);  in omap_aes_crypt_dma()
429 		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);  in omap_aes_crypt_dma()
433 	dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);  in omap_aes_crypt_dma()
437 	cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);  in omap_aes_crypt_dma()
438 	cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);  in omap_aes_crypt_dma()
445 	ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);  in omap_aes_crypt_dma()
447 		dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",  in omap_aes_crypt_dma()
452 	tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,  in omap_aes_crypt_dma()
456 		dev_err(dd->dev, "IN prep_slave_sg() failed\n");  in omap_aes_crypt_dma()
461 	tx_in->callback_param = dd;  in omap_aes_crypt_dma()
464 	ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);  in omap_aes_crypt_dma()
466 		dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",  in omap_aes_crypt_dma()
471 	tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,  in omap_aes_crypt_dma()
475 		dev_err(dd->dev, "OUT prep_slave_sg() failed\n");  in omap_aes_crypt_dma()
480 	tx_out->callback_param = dd;  in omap_aes_crypt_dma()
485 	dma_async_issue_pending(dd->dma_lch_in);  in omap_aes_crypt_dma()
486 	dma_async_issue_pending(dd->dma_lch_out);  in omap_aes_crypt_dma()
489 	dd->pdata->trigger(dd, dd->total);  in omap_aes_crypt_dma()
494 static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)  in omap_aes_crypt_dma_start()  argument
497 					crypto_ablkcipher_reqtfm(dd->req));  in omap_aes_crypt_dma_start()
500 	pr_debug("total: %d\n", dd->total);  in omap_aes_crypt_dma_start()
502 	if (!dd->pio_only) {  in omap_aes_crypt_dma_start()
503 		err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,  in omap_aes_crypt_dma_start()
506 			dev_err(dd->dev, "dma_map_sg() error\n");  in omap_aes_crypt_dma_start()
510 		err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_aes_crypt_dma_start()
513 			dev_err(dd->dev, "dma_map_sg() error\n");  in omap_aes_crypt_dma_start()
518 	err = omap_aes_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,  in omap_aes_crypt_dma_start()
519 				 dd->out_sg_len);  in omap_aes_crypt_dma_start()
520 	if (err && !dd->pio_only) {  in omap_aes_crypt_dma_start()
521 		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_aes_crypt_dma_start()
522 		dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_aes_crypt_dma_start()
529 static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)  in omap_aes_finish_req()  argument
531 	struct ablkcipher_request *req = dd->req;  in omap_aes_finish_req()
535 	dd->flags &= ~FLAGS_BUSY;  in omap_aes_finish_req()
540 static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)  in omap_aes_crypt_dma_stop()  argument
544 	pr_debug("total: %d\n", dd->total);  in omap_aes_crypt_dma_stop()
546 	omap_aes_dma_stop(dd);  in omap_aes_crypt_dma_stop()
548 	dmaengine_terminate_all(dd->dma_lch_in);  in omap_aes_crypt_dma_stop()
549 	dmaengine_terminate_all(dd->dma_lch_out);  in omap_aes_crypt_dma_stop()
577 static int omap_aes_copy_sgs(struct omap_aes_dev *dd)  in omap_aes_copy_sgs()  argument
582 	total = ALIGN(dd->total, AES_BLOCK_SIZE);  in omap_aes_copy_sgs()
593 	dd->orig_out = dd->out_sg;  in omap_aes_copy_sgs()
595 	sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);  in omap_aes_copy_sgs()
597 	sg_init_table(&dd->in_sgl, 1);  in omap_aes_copy_sgs()
598 	sg_set_buf(&dd->in_sgl, buf_in, total);  in omap_aes_copy_sgs()
599 	dd->in_sg = &dd->in_sgl;  in omap_aes_copy_sgs()
601 	sg_init_table(&dd->out_sgl, 1);  in omap_aes_copy_sgs()
602 	sg_set_buf(&dd->out_sgl, buf_out, total);  in omap_aes_copy_sgs()
603 	dd->out_sg = &dd->out_sgl;  in omap_aes_copy_sgs()
608 static int omap_aes_handle_queue(struct omap_aes_dev *dd,  in omap_aes_handle_queue()  argument
617 	spin_lock_irqsave(&dd->lock, flags);  in omap_aes_handle_queue()
619 		ret = ablkcipher_enqueue_request(&dd->queue, req);  in omap_aes_handle_queue()
620 	if (dd->flags & FLAGS_BUSY) {  in omap_aes_handle_queue()
621 		spin_unlock_irqrestore(&dd->lock, flags);  in omap_aes_handle_queue()
624 	backlog = crypto_get_backlog(&dd->queue);  in omap_aes_handle_queue()
625 	async_req = crypto_dequeue_request(&dd->queue);  in omap_aes_handle_queue()
627 		dd->flags |= FLAGS_BUSY;  in omap_aes_handle_queue()
628 	spin_unlock_irqrestore(&dd->lock, flags);  in omap_aes_handle_queue()
639 	dd->req = req;  in omap_aes_handle_queue()
640 	dd->total = req->nbytes;  in omap_aes_handle_queue()
641 	dd->total_save = req->nbytes;  in omap_aes_handle_queue()
642 	dd->in_sg = req->src;  in omap_aes_handle_queue()
643 	dd->out_sg = req->dst;  in omap_aes_handle_queue()
645 	if (omap_aes_check_aligned(dd->in_sg, dd->total) ||  in omap_aes_handle_queue()
646 	    omap_aes_check_aligned(dd->out_sg, dd->total)) {  in omap_aes_handle_queue()
647 		if (omap_aes_copy_sgs(dd))  in omap_aes_handle_queue()
649 		dd->sgs_copied = 1;  in omap_aes_handle_queue()
651 		dd->sgs_copied = 0;  in omap_aes_handle_queue()
654 	len = ALIGN(dd->total, AES_BLOCK_SIZE);  in omap_aes_handle_queue()
655 	dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, len);  in omap_aes_handle_queue()
656 	dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, len);  in omap_aes_handle_queue()
657 	BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0);  in omap_aes_handle_queue()
662 	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;  in omap_aes_handle_queue()
664 	dd->ctx = ctx;  in omap_aes_handle_queue()
665 	ctx->dd = dd;  in omap_aes_handle_queue()
667 	err = omap_aes_write_ctrl(dd);  in omap_aes_handle_queue()
669 		err = omap_aes_crypt_dma_start(dd);  in omap_aes_handle_queue()
672 		omap_aes_finish_req(dd, err);  in omap_aes_handle_queue()
673 		tasklet_schedule(&dd->queue_task);  in omap_aes_handle_queue()
681 	struct omap_aes_dev *dd = (struct omap_aes_dev *)data;  in omap_aes_done_task()  local
687 	if (!dd->pio_only) {  in omap_aes_done_task()
688 		dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_aes_done_task()
690 		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_aes_done_task()
691 		dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_aes_done_task()
693 		omap_aes_crypt_dma_stop(dd);  in omap_aes_done_task()
696 	if (dd->sgs_copied) {  in omap_aes_done_task()
697 		buf_in = sg_virt(&dd->in_sgl);  in omap_aes_done_task()
698 		buf_out = sg_virt(&dd->out_sgl);  in omap_aes_done_task()
700 		sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);  in omap_aes_done_task()
702 		len = ALIGN(dd->total_save, AES_BLOCK_SIZE);  in omap_aes_done_task()
708 	omap_aes_finish_req(dd, 0);  in omap_aes_done_task()
709 	omap_aes_handle_queue(dd, NULL);  in omap_aes_done_task()
716 	struct omap_aes_dev *dd = (struct omap_aes_dev *)data;  in omap_aes_queue_task()  local
718 	omap_aes_handle_queue(dd, NULL);  in omap_aes_queue_task()
726 	struct omap_aes_dev *dd;  in omap_aes_crypt()  local
732 	dd = omap_aes_find_dev(ctx);  in omap_aes_crypt()
733 	if (!dd)  in omap_aes_crypt()
738 	return omap_aes_handle_queue(dd, req);  in omap_aes_crypt()
792 	struct omap_aes_dev *dd = NULL;  in omap_aes_cra_init()  local
797 	list_for_each_entry(dd, &dev_list, list) {  in omap_aes_cra_init()
802 	err = pm_runtime_get_sync(dd->dev);  in omap_aes_cra_init()
804 		dev_err(dd->dev, "%s: failed to get_sync(%d)\n",  in omap_aes_cra_init()
816 	struct omap_aes_dev *dd = NULL;  in omap_aes_cra_exit()  local
820 	list_for_each_entry(dd, &dev_list, list) {  in omap_aes_cra_exit()
825 	pm_runtime_put_sync(dd->dev);  in omap_aes_cra_exit()
984 	struct omap_aes_dev *dd = dev_id;  in omap_aes_irq()  local
988 	status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd));  in omap_aes_irq()
990 		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);  in omap_aes_irq()
992 		BUG_ON(!dd->in_sg);  in omap_aes_irq()
994 		BUG_ON(_calc_walked(in) > dd->in_sg->length);  in omap_aes_irq()
996 		src = sg_virt(dd->in_sg) + _calc_walked(in);  in omap_aes_irq()
999 			omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src);  in omap_aes_irq()
1001 			scatterwalk_advance(&dd->in_walk, 4);  in omap_aes_irq()
1002 			if (dd->in_sg->length == _calc_walked(in)) {  in omap_aes_irq()
1003 				dd->in_sg = sg_next(dd->in_sg);  in omap_aes_irq()
1004 				if (dd->in_sg) {  in omap_aes_irq()
1005 					scatterwalk_start(&dd->in_walk,  in omap_aes_irq()
1006 							  dd->in_sg);  in omap_aes_irq()
1007 					src = sg_virt(dd->in_sg) +  in omap_aes_irq()
1017 		omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);  in omap_aes_irq()
1020 		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4);  in omap_aes_irq()
1023 		omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);  in omap_aes_irq()
1025 		BUG_ON(!dd->out_sg);  in omap_aes_irq()
1027 		BUG_ON(_calc_walked(out) > dd->out_sg->length);  in omap_aes_irq()
1029 		dst = sg_virt(dd->out_sg) + _calc_walked(out);  in omap_aes_irq()
1032 			*dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));  in omap_aes_irq()
1033 			scatterwalk_advance(&dd->out_walk, 4);  in omap_aes_irq()
1034 			if (dd->out_sg->length == _calc_walked(out)) {  in omap_aes_irq()
1035 				dd->out_sg = sg_next(dd->out_sg);  in omap_aes_irq()
1036 				if (dd->out_sg) {  in omap_aes_irq()
1037 					scatterwalk_start(&dd->out_walk,  in omap_aes_irq()
1038 							  dd->out_sg);  in omap_aes_irq()
1039 					dst = sg_virt(dd->out_sg) +  in omap_aes_irq()
1047 		dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total);  in omap_aes_irq()
1051 		omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);  in omap_aes_irq()
1053 		if (!dd->total)  in omap_aes_irq()
1055 			tasklet_schedule(&dd->done_task);  in omap_aes_irq()
1058 			omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);  in omap_aes_irq()
1081 static int omap_aes_get_res_of(struct omap_aes_dev *dd,  in omap_aes_get_res_of()  argument
1102 	dd->dma_out = -1; /* Dummy value that's unused */  in omap_aes_get_res_of()
1103 	dd->dma_in = -1; /* Dummy value that's unused */  in omap_aes_get_res_of()
1105 	dd->pdata = match->data;  in omap_aes_get_res_of()
1115 static int omap_aes_get_res_of(struct omap_aes_dev *dd,  in omap_aes_get_res_of()  argument
1122 static int omap_aes_get_res_pdev(struct omap_aes_dev *dd,  in omap_aes_get_res_pdev()  argument
1145 	dd->dma_out = r->start;  in omap_aes_get_res_pdev()
1154 	dd->dma_in = r->start;  in omap_aes_get_res_pdev()
1157 	dd->pdata = &omap_aes_pdata_omap2;  in omap_aes_get_res_pdev()
1166 	struct omap_aes_dev *dd;  in omap_aes_probe()  local
1172 	dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL);  in omap_aes_probe()
1173 	if (dd == NULL) {  in omap_aes_probe()
1177 	dd->dev = dev;  in omap_aes_probe()
1178 	platform_set_drvdata(pdev, dd);  in omap_aes_probe()
1180 	spin_lock_init(&dd->lock);  in omap_aes_probe()
1181 	crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH);  in omap_aes_probe()
1183 	err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) :  in omap_aes_probe()
1184 			       omap_aes_get_res_pdev(dd, pdev, &res);  in omap_aes_probe()
1188 	dd->io_base = devm_ioremap_resource(dev, &res);  in omap_aes_probe()
1189 	if (IS_ERR(dd->io_base)) {  in omap_aes_probe()
1190 		err = PTR_ERR(dd->io_base);  in omap_aes_probe()
1193 	dd->phys_base = res.start;  in omap_aes_probe()
1203 	omap_aes_dma_stop(dd);  in omap_aes_probe()
1205 	reg = omap_aes_read(dd, AES_REG_REV(dd));  in omap_aes_probe()
1210 		 (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,  in omap_aes_probe()
1211 		 (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);  in omap_aes_probe()
1213 	tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);  in omap_aes_probe()
1214 	tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);  in omap_aes_probe()
1216 	err = omap_aes_dma_init(dd);  in omap_aes_probe()
1217 	if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) {  in omap_aes_probe()
1218 		dd->pio_only = 1;  in omap_aes_probe()
1227 				dev_name(dev), dd);  in omap_aes_probe()
1235 	INIT_LIST_HEAD(&dd->list);  in omap_aes_probe()
1237 	list_add_tail(&dd->list, &dev_list);  in omap_aes_probe()
1240 	for (i = 0; i < dd->pdata->algs_info_size; i++) {  in omap_aes_probe()
1241 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {  in omap_aes_probe()
1242 			algp = &dd->pdata->algs_info[i].algs_list[j];  in omap_aes_probe()
1251 			dd->pdata->algs_info[i].registered++;  in omap_aes_probe()
1257 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)  in omap_aes_probe()
1258 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)  in omap_aes_probe()
1260 					&dd->pdata->algs_info[i].algs_list[j]);  in omap_aes_probe()
1261 	if (!dd->pio_only)  in omap_aes_probe()
1262 		omap_aes_dma_cleanup(dd);  in omap_aes_probe()
1264 	tasklet_kill(&dd->done_task);  in omap_aes_probe()
1265 	tasklet_kill(&dd->queue_task);  in omap_aes_probe()
1268 	dd = NULL;  in omap_aes_probe()
1276 	struct omap_aes_dev *dd = platform_get_drvdata(pdev);  in omap_aes_remove()  local
1279 	if (!dd)  in omap_aes_remove()
1283 	list_del(&dd->list);  in omap_aes_remove()
1286 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)  in omap_aes_remove()
1287 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)  in omap_aes_remove()
1289 					&dd->pdata->algs_info[i].algs_list[j]);  in omap_aes_remove()
1291 	tasklet_kill(&dd->done_task);  in omap_aes_remove()
1292 	tasklet_kill(&dd->queue_task);  in omap_aes_remove()
1293 	omap_aes_dma_cleanup(dd);  in omap_aes_remove()
1294 	pm_runtime_disable(dd->dev);  in omap_aes_remove()
1295 	dd = NULL;  in omap_aes_remove()