281 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_copy_hash_omap2()  local
282 	struct omap_sham_dev *dd = ctx->dd;  in omap_sham_copy_hash_omap2()
283 	u32 *hash = (u32 *)ctx->digest;  in omap_sham_copy_hash_omap2()
296 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_copy_hash_omap4()  local
297 	struct omap_sham_dev *dd = ctx->dd;  in omap_sham_copy_hash_omap4()
300 	if (ctx->flags & BIT(FLAGS_HMAC)) {  in omap_sham_copy_hash_omap4()
321 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_copy_ready_hash()  local
322 	u32 *in = (u32 *)ctx->digest;  in omap_sham_copy_ready_hash()
329 	switch (ctx->flags & FLAGS_MODE_MASK) {  in omap_sham_copy_ready_hash()
335 		if (test_bit(FLAGS_BE32_SHA1, &ctx->dd->flags))  in omap_sham_copy_ready_hash()
384 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_write_ctrl_omap2()  local
387 	if (likely(ctx->digcnt))  in omap_sham_write_ctrl_omap2()
388 		omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);  in omap_sham_write_ctrl_omap2()
397 	if ((ctx->flags & FLAGS_MODE_MASK) == FLAGS_MODE_SHA1)  in omap_sham_write_ctrl_omap2()
399 	if (!ctx->digcnt)  in omap_sham_write_ctrl_omap2()
419 static int get_block_size(struct omap_sham_reqctx *ctx)  in get_block_size()  argument
423 	switch (ctx->flags & FLAGS_MODE_MASK) {  in get_block_size()
453 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_write_ctrl_omap4()  local
461 	val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT);  in omap_sham_write_ctrl_omap4()
462 	if (!ctx->digcnt) {  in omap_sham_write_ctrl_omap4()
470 		if (ctx->flags & BIT(FLAGS_HMAC)) {  in omap_sham_write_ctrl_omap4()
471 			bs = get_block_size(ctx);  in omap_sham_write_ctrl_omap4()
478 			ctx->digcnt += bs;  in omap_sham_write_ctrl_omap4()
485 		if (ctx->flags & BIT(FLAGS_HMAC))  in omap_sham_write_ctrl_omap4()
493 	dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);  in omap_sham_write_ctrl_omap4()
516 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_xmit_cpu()  local
521 						ctx->digcnt, length, final);  in omap_sham_xmit_cpu()
527 	ctx->digcnt += length;  in omap_sham_xmit_cpu()
535 	bs32 = get_block_size(ctx) / sizeof(u32);  in omap_sham_xmit_cpu()
561 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_xmit_dma()  local
564 	int len32, ret, dma_min = get_block_size(ctx);  in omap_sham_xmit_dma()
567 						ctx->digcnt, length, final);  in omap_sham_xmit_dma()
590 		sg_init_table(&ctx->sgl, 1);  in omap_sham_xmit_dma()
591 		sg_assign_page(&ctx->sgl, sg_page(ctx->sg));  in omap_sham_xmit_dma()
592 		ctx->sgl.offset = ctx->sg->offset;  in omap_sham_xmit_dma()
593 		sg_dma_len(&ctx->sgl) = len32;  in omap_sham_xmit_dma()
594 		sg_dma_address(&ctx->sgl) = sg_dma_address(ctx->sg);  in omap_sham_xmit_dma()
596 		tx = dmaengine_prep_slave_sg(dd->dma_lch, &ctx->sgl, 1,  in omap_sham_xmit_dma()
613 	ctx->digcnt += length;  in omap_sham_xmit_dma()
628 static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx,  in omap_sham_append_buffer()  argument
631 	size_t count = min(length, ctx->buflen - ctx->bufcnt);  in omap_sham_append_buffer()
633 	count = min(count, ctx->total);  in omap_sham_append_buffer()
636 	memcpy(ctx->buffer + ctx->bufcnt, data, count);  in omap_sham_append_buffer()
637 	ctx->bufcnt += count;  in omap_sham_append_buffer()
642 static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)  in omap_sham_append_sg()  argument
647 	while (ctx->sg) {  in omap_sham_append_sg()
648 		vaddr = kmap_atomic(sg_page(ctx->sg));  in omap_sham_append_sg()
649 		vaddr += ctx->sg->offset;  in omap_sham_append_sg()
651 		count = omap_sham_append_buffer(ctx,  in omap_sham_append_sg()
652 				vaddr + ctx->offset,  in omap_sham_append_sg()
653 				ctx->sg->length - ctx->offset);  in omap_sham_append_sg()
659 		ctx->offset += count;  in omap_sham_append_sg()
660 		ctx->total -= count;  in omap_sham_append_sg()
661 		if (ctx->offset == ctx->sg->length) {  in omap_sham_append_sg()
662 			ctx->sg = sg_next(ctx->sg);  in omap_sham_append_sg()
663 			if (ctx->sg)  in omap_sham_append_sg()
664 				ctx->offset = 0;  in omap_sham_append_sg()
666 				ctx->total = 0;  in omap_sham_append_sg()
674 					struct omap_sham_reqctx *ctx,  in omap_sham_xmit_dma_map()  argument
679 	ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,  in omap_sham_xmit_dma_map()
681 	if (dma_mapping_error(dd->dev, ctx->dma_addr)) {  in omap_sham_xmit_dma_map()
682 		dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);  in omap_sham_xmit_dma_map()
686 	ctx->flags &= ~BIT(FLAGS_SG);  in omap_sham_xmit_dma_map()
688 	ret = omap_sham_xmit_dma(dd, ctx->dma_addr, length, final, 0);  in omap_sham_xmit_dma_map()
690 		dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,  in omap_sham_xmit_dma_map()
698 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_update_dma_slow()  local
702 	omap_sham_append_sg(ctx);  in omap_sham_update_dma_slow()
704 	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;  in omap_sham_update_dma_slow()
707 					 ctx->bufcnt, ctx->digcnt, final);  in omap_sham_update_dma_slow()
709 	if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {  in omap_sham_update_dma_slow()
710 		count = ctx->bufcnt;  in omap_sham_update_dma_slow()
711 		ctx->bufcnt = 0;  in omap_sham_update_dma_slow()
712 		return omap_sham_xmit_dma_map(dd, ctx, count, final);  in omap_sham_update_dma_slow()
725 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_update_dma_start()  local
730 	if (!ctx->total)  in omap_sham_update_dma_start()
733 	if (ctx->bufcnt || ctx->offset)  in omap_sham_update_dma_start()
742 	if (ctx->total < get_block_size(ctx))  in omap_sham_update_dma_start()
746 			ctx->digcnt, ctx->bufcnt, ctx->total);  in omap_sham_update_dma_start()
748 	sg = ctx->sg;  in omap_sham_update_dma_start()
749 	bs = get_block_size(ctx);  in omap_sham_update_dma_start()
758 	length = min(ctx->total, sg->length);  in omap_sham_update_dma_start()
761 		if (!(ctx->flags & BIT(FLAGS_FINUP))) {  in omap_sham_update_dma_start()
771 	if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {  in omap_sham_update_dma_start()
776 	ctx->flags |= BIT(FLAGS_SG);  in omap_sham_update_dma_start()
778 	ctx->total -= length;  in omap_sham_update_dma_start()
779 	ctx->offset = length; /* offset where to start slow */  in omap_sham_update_dma_start()
781 	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;  in omap_sham_update_dma_start()
783 	ret = omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final, 1);  in omap_sham_update_dma_start()
785 		dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);  in omap_sham_update_dma_start()
792 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_update_cpu()  local
795 	if (!ctx->total)  in omap_sham_update_cpu()
798 	omap_sham_append_sg(ctx);  in omap_sham_update_cpu()
800 	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;  in omap_sham_update_cpu()
803 		ctx->bufcnt, ctx->digcnt, final);  in omap_sham_update_cpu()
805 	if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {  in omap_sham_update_cpu()
806 		bufcnt = ctx->bufcnt;  in omap_sham_update_cpu()
807 		ctx->bufcnt = 0;  in omap_sham_update_cpu()
808 		return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final);  in omap_sham_update_cpu()
816 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);  in omap_sham_update_dma_stop()  local
820 	if (ctx->flags & BIT(FLAGS_SG)) {  in omap_sham_update_dma_stop()
821 		dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);  in omap_sham_update_dma_stop()
822 		if (ctx->sg->length == ctx->offset) {  in omap_sham_update_dma_stop()
823 			ctx->sg = sg_next(ctx->sg);  in omap_sham_update_dma_stop()
824 			if (ctx->sg)  in omap_sham_update_dma_stop()
825 				ctx->offset = 0;  in omap_sham_update_dma_stop()
828 		dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,  in omap_sham_update_dma_stop()
839 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_init()  local
855 	ctx->dd = dd;  in omap_sham_init()
857 	ctx->flags = 0;  in omap_sham_init()
864 		ctx->flags |= FLAGS_MODE_MD5;  in omap_sham_init()
868 		ctx->flags |= FLAGS_MODE_SHA1;  in omap_sham_init()
872 		ctx->flags |= FLAGS_MODE_SHA224;  in omap_sham_init()
876 		ctx->flags |= FLAGS_MODE_SHA256;  in omap_sham_init()
880 		ctx->flags |= FLAGS_MODE_SHA384;  in omap_sham_init()
884 		ctx->flags |= FLAGS_MODE_SHA512;  in omap_sham_init()
889 	ctx->bufcnt = 0;  in omap_sham_init()
890 	ctx->digcnt = 0;  in omap_sham_init()
891 	ctx->buflen = BUFLEN;  in omap_sham_init()
897 			memcpy(ctx->buffer, bctx->ipad, bs);  in omap_sham_init()
898 			ctx->bufcnt = bs;  in omap_sham_init()
901 		ctx->flags |= BIT(FLAGS_HMAC);  in omap_sham_init()
911 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_update_req()  local
915 		 ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0);  in omap_sham_update_req()
917 	if (ctx->flags & BIT(FLAGS_CPU))  in omap_sham_update_req()
923 	dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);  in omap_sham_update_req()
931 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_final_req()  local
934 	if ((ctx->bufcnt <= get_block_size(ctx)) || dd->polling_mode)  in omap_sham_final_req()
942 		err = omap_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1);  in omap_sham_final_req()
944 		err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);  in omap_sham_final_req()
946 	ctx->bufcnt = 0;  in omap_sham_final_req()
971 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_finish()  local
972 	struct omap_sham_dev *dd = ctx->dd;  in omap_sham_finish()
975 	if (ctx->digcnt) {  in omap_sham_finish()
977 		if ((ctx->flags & BIT(FLAGS_HMAC)) &&  in omap_sham_finish()
982 	dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);  in omap_sham_finish()
989 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_finish_req()  local
990 	struct omap_sham_dev *dd = ctx->dd;  in omap_sham_finish_req()
997 		ctx->flags |= BIT(FLAGS_ERROR);  in omap_sham_finish_req()
1017 	struct omap_sham_reqctx *ctx;  in omap_sham_handle_queue()  local
1042 	ctx = ahash_request_ctx(req);  in omap_sham_handle_queue()
1045 						ctx->op, req->nbytes);  in omap_sham_handle_queue()
1051 	if (ctx->digcnt)  in omap_sham_handle_queue()
1055 	if (ctx->op == OP_UPDATE) {  in omap_sham_handle_queue()
1057 		if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP)))  in omap_sham_handle_queue()
1060 	} else if (ctx->op == OP_FINAL) {  in omap_sham_handle_queue()
1075 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_enqueue()  local
1079 	ctx->op = op;  in omap_sham_enqueue()
1086 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_update()  local
1087 	struct omap_sham_dev *dd = ctx->dd;  in omap_sham_update()
1088 	int bs = get_block_size(ctx);  in omap_sham_update()
1093 	ctx->total = req->nbytes;  in omap_sham_update()
1094 	ctx->sg = req->src;  in omap_sham_update()
1095 	ctx->offset = 0;  in omap_sham_update()
1097 	if (ctx->flags & BIT(FLAGS_FINUP)) {  in omap_sham_update()
1098 		if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) {  in omap_sham_update()
1104 			omap_sham_append_sg(ctx);  in omap_sham_update()
1106 		} else if ((ctx->bufcnt + ctx->total <= bs) ||  in omap_sham_update()
1112 			ctx->flags |= BIT(FLAGS_CPU);  in omap_sham_update()
1114 	} else if (ctx->bufcnt + ctx->total < ctx->buflen) {  in omap_sham_update()
1115 		omap_sham_append_sg(ctx);  in omap_sham_update()
1120 		ctx->flags |= BIT(FLAGS_CPU);  in omap_sham_update()
1139 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_final_shash()  local
1142 				      ctx->buffer, ctx->bufcnt, req->result);  in omap_sham_final_shash()
1147 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_final()  local
1149 	ctx->flags |= BIT(FLAGS_FINUP);  in omap_sham_final()
1151 	if (ctx->flags & BIT(FLAGS_ERROR))  in omap_sham_final()
1156 	if ((ctx->digcnt + ctx->bufcnt) < 9)  in omap_sham_final()
1158 	else if (ctx->bufcnt)  in omap_sham_final()
1167 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);  in omap_sham_finup()  local
1170 	ctx->flags |= BIT(FLAGS_FINUP);  in omap_sham_finup()