605 			struct ahash_edesc *edesc,  in ahash_unmap()  argument
608 	if (edesc->src_nents)  in ahash_unmap()
609 		dma_unmap_sg_chained(dev, req->src, edesc->src_nents,  in ahash_unmap()
610 				     DMA_TO_DEVICE, edesc->chained);  in ahash_unmap()
611 	if (edesc->dst_dma)  in ahash_unmap()
612 		dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);  in ahash_unmap()
614 	if (edesc->sec4_sg_bytes)  in ahash_unmap()
615 		dma_unmap_single(dev, edesc->sec4_sg_dma,  in ahash_unmap()
616 				 edesc->sec4_sg_bytes, DMA_TO_DEVICE);  in ahash_unmap()
620 			struct ahash_edesc *edesc,  in ahash_unmap_ctx()  argument
629 	ahash_unmap(dev, edesc, req, dst_len);  in ahash_unmap_ctx()
636 	struct ahash_edesc *edesc;  in ahash_done()  local
646 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done()
651 	ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_done()
652 	kfree(edesc);  in ahash_done()
671 	struct ahash_edesc *edesc;  in ahash_done_bi()  local
681 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done_bi()
686 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);  in ahash_done_bi()
687 	kfree(edesc);  in ahash_done_bi()
706 	struct ahash_edesc *edesc;  in ahash_done_ctx_src()  local
716 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done_ctx_src()
721 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);  in ahash_done_ctx_src()
722 	kfree(edesc);  in ahash_done_ctx_src()
741 	struct ahash_edesc *edesc;  in ahash_done_ctx_dst()  local
751 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done_ctx_dst()
756 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);  in ahash_done_ctx_dst()
757 	kfree(edesc);  in ahash_done_ctx_dst()
790 	struct ahash_edesc *edesc;  in ahash_update_ctx()  local
810 		edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN +  in ahash_update_ctx()
812 		if (!edesc) {  in ahash_update_ctx()
818 		edesc->src_nents = src_nents;  in ahash_update_ctx()
819 		edesc->chained = chained;  in ahash_update_ctx()
820 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_ctx()
821 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_update_ctx()
825 					 edesc->sec4_sg, DMA_BIDIRECTIONAL);  in ahash_update_ctx()
830 							edesc->sec4_sg + 1,  in ahash_update_ctx()
836 					   edesc->sec4_sg + sec4_sg_src_index,  in ahash_update_ctx()
845 			(edesc->sec4_sg + sec4_sg_src_index - 1)->len |=  in ahash_update_ctx()
850 		desc = edesc->hw_desc;  in ahash_update_ctx()
854 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_ctx()
857 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_ctx()
862 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +  in ahash_update_ctx()
877 			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,  in ahash_update_ctx()
879 			kfree(edesc);  in ahash_update_ctx()
914 	struct ahash_edesc *edesc;  in ahash_final_ctx()  local
922 	edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN +  in ahash_final_ctx()
924 	if (!edesc) {  in ahash_final_ctx()
930 	desc = edesc->hw_desc;  in ahash_final_ctx()
933 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_final_ctx()
934 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_final_ctx()
936 	edesc->src_nents = 0;  in ahash_final_ctx()
939 				 edesc->sec4_sg, DMA_TO_DEVICE);  in ahash_final_ctx()
943 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,  in ahash_final_ctx()
946 	(edesc->sec4_sg + sec4_sg_src_index - 1)->len |= SEC4_SG_LEN_FIN;  in ahash_final_ctx()
948 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_final_ctx()
950 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_final_ctx()
955 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,  in ahash_final_ctx()
958 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_final_ctx()
960 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_final_ctx()
974 		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_final_ctx()
975 		kfree(edesc);  in ahash_final_ctx()
998 	struct ahash_edesc *edesc;  in ahash_finup_ctx()  local
1009 	edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN +  in ahash_finup_ctx()
1011 	if (!edesc) {  in ahash_finup_ctx()
1017 	desc = edesc->hw_desc;  in ahash_finup_ctx()
1020 	edesc->src_nents = src_nents;  in ahash_finup_ctx()
1021 	edesc->chained = chained;  in ahash_finup_ctx()
1022 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_finup_ctx()
1023 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_finup_ctx()
1027 				 edesc->sec4_sg, DMA_TO_DEVICE);  in ahash_finup_ctx()
1031 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,  in ahash_finup_ctx()
1035 	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg +  in ahash_finup_ctx()
1038 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_finup_ctx()
1040 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_finup_ctx()
1045 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +  in ahash_finup_ctx()
1048 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_finup_ctx()
1050 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_finup_ctx()
1064 		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_finup_ctx()
1065 		kfree(edesc);  in ahash_finup_ctx()
1083 	struct ahash_edesc *edesc;  in ahash_digest()  local
1095 	edesc = kmalloc(sizeof(struct ahash_edesc) + sec4_sg_bytes +  in ahash_digest()
1097 	if (!edesc) {  in ahash_digest()
1101 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_digest()
1103 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_digest()
1104 	edesc->src_nents = src_nents;  in ahash_digest()
1105 	edesc->chained = chained;  in ahash_digest()
1108 	desc = edesc->hw_desc;  in ahash_digest()
1112 		sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);  in ahash_digest()
1113 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_digest()
1115 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_digest()
1119 		src_dma = edesc->sec4_sg_dma;  in ahash_digest()
1127 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_digest()
1129 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_digest()
1143 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_digest()
1144 		kfree(edesc);  in ahash_digest()
1164 	struct ahash_edesc *edesc;  in ahash_final_no_ctx()  local
1169 	edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN,  in ahash_final_no_ctx()
1171 	if (!edesc) {  in ahash_final_no_ctx()
1176 	edesc->sec4_sg_bytes = 0;  in ahash_final_no_ctx()
1178 	desc = edesc->hw_desc;  in ahash_final_no_ctx()
1189 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_final_no_ctx()
1191 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_final_no_ctx()
1195 	edesc->src_nents = 0;  in ahash_final_no_ctx()
1206 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_final_no_ctx()
1207 		kfree(edesc);  in ahash_final_no_ctx()
1229 	struct ahash_edesc *edesc;  in ahash_update_no_ctx()  local
1249 		edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN +  in ahash_update_no_ctx()
1251 		if (!edesc) {  in ahash_update_no_ctx()
1257 		edesc->src_nents = src_nents;  in ahash_update_no_ctx()
1258 		edesc->chained = chained;  in ahash_update_no_ctx()
1259 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_no_ctx()
1260 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_update_no_ctx()
1262 		edesc->dst_dma = 0;  in ahash_update_no_ctx()
1264 		state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg,  in ahash_update_no_ctx()
1267 				   edesc->sec4_sg + 1, chained);  in ahash_update_no_ctx()
1276 		desc = edesc->hw_desc;  in ahash_update_no_ctx()
1280 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_no_ctx()
1283 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_no_ctx()
1288 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);  in ahash_update_no_ctx()
1307 			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,  in ahash_update_no_ctx()
1309 			kfree(edesc);  in ahash_update_no_ctx()
1345 	struct ahash_edesc *edesc;  in ahash_finup_no_ctx()  local
1356 	edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN +  in ahash_finup_no_ctx()
1358 	if (!edesc) {  in ahash_finup_no_ctx()
1364 	desc = edesc->hw_desc;  in ahash_finup_no_ctx()
1367 	edesc->src_nents = src_nents;  in ahash_finup_no_ctx()
1368 	edesc->chained = chained;  in ahash_finup_no_ctx()
1369 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_finup_no_ctx()
1370 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_finup_no_ctx()
1373 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf,  in ahash_finup_no_ctx()
1377 	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1,  in ahash_finup_no_ctx()
1380 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_finup_no_ctx()
1382 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_finup_no_ctx()
1387 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen +  in ahash_finup_no_ctx()
1390 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_finup_no_ctx()
1392 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_finup_no_ctx()
1406 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_finup_no_ctx()
1407 		kfree(edesc);  in ahash_finup_no_ctx()
1431 	struct ahash_edesc *edesc;  in ahash_update_first()  local
1451 		edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN +  in ahash_update_first()
1453 		if (!edesc) {  in ahash_update_first()
1459 		edesc->src_nents = src_nents;  in ahash_update_first()
1460 		edesc->chained = chained;  in ahash_update_first()
1461 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_first()
1462 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_update_first()
1464 		edesc->dst_dma = 0;  in ahash_update_first()
1468 					   edesc->sec4_sg, 0);  in ahash_update_first()
1469 			edesc->sec4_sg_dma = dma_map_single(jrdev,  in ahash_update_first()
1470 							    edesc->sec4_sg,  in ahash_update_first()
1473 			if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_first()
1477 			src_dma = edesc->sec4_sg_dma;  in ahash_update_first()
1489 		desc = edesc->hw_desc;  in ahash_update_first()
1513 			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,  in ahash_update_first()
1515 			kfree(edesc);  in ahash_update_first()