611 			struct ahash_edesc *edesc,  in ahash_unmap()  argument
614 	if (edesc->src_nents)  in ahash_unmap()
615 		dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);  in ahash_unmap()
616 	if (edesc->dst_dma)  in ahash_unmap()
617 		dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);  in ahash_unmap()
619 	if (edesc->sec4_sg_bytes)  in ahash_unmap()
620 		dma_unmap_single(dev, edesc->sec4_sg_dma,  in ahash_unmap()
621 				 edesc->sec4_sg_bytes, DMA_TO_DEVICE);  in ahash_unmap()
625 			struct ahash_edesc *edesc,  in ahash_unmap_ctx()  argument
634 	ahash_unmap(dev, edesc, req, dst_len);  in ahash_unmap_ctx()
641 	struct ahash_edesc *edesc;  in ahash_done()  local
651 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done()
656 	ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_done()
657 	kfree(edesc);  in ahash_done()
676 	struct ahash_edesc *edesc;  in ahash_done_bi()  local
686 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done_bi()
691 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);  in ahash_done_bi()
692 	kfree(edesc);  in ahash_done_bi()
711 	struct ahash_edesc *edesc;  in ahash_done_ctx_src()  local
721 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done_ctx_src()
726 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);  in ahash_done_ctx_src()
727 	kfree(edesc);  in ahash_done_ctx_src()
746 	struct ahash_edesc *edesc;  in ahash_done_ctx_dst()  local
756 	edesc = (struct ahash_edesc *)((char *)desc -  in ahash_done_ctx_dst()
761 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);  in ahash_done_ctx_dst()
762 	kfree(edesc);  in ahash_done_ctx_dst()
795 	struct ahash_edesc *edesc;  in ahash_update_ctx()  local
814 		edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN +  in ahash_update_ctx()
816 		if (!edesc) {  in ahash_update_ctx()
822 		edesc->src_nents = src_nents;  in ahash_update_ctx()
823 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_ctx()
824 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_update_ctx()
828 					 edesc->sec4_sg, DMA_BIDIRECTIONAL);  in ahash_update_ctx()
833 							edesc->sec4_sg + 1,  in ahash_update_ctx()
839 					   edesc->sec4_sg + sec4_sg_src_index);  in ahash_update_ctx()
845 			(edesc->sec4_sg + sec4_sg_src_index - 1)->len |=  in ahash_update_ctx()
852 		desc = edesc->hw_desc;  in ahash_update_ctx()
856 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_ctx()
859 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_ctx()
864 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +  in ahash_update_ctx()
879 			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,  in ahash_update_ctx()
881 			kfree(edesc);  in ahash_update_ctx()
916 	struct ahash_edesc *edesc;  in ahash_final_ctx()  local
924 	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes,  in ahash_final_ctx()
926 	if (!edesc) {  in ahash_final_ctx()
932 	desc = edesc->hw_desc;  in ahash_final_ctx()
935 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_final_ctx()
936 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_final_ctx()
938 	edesc->src_nents = 0;  in ahash_final_ctx()
941 				 edesc->sec4_sg, DMA_TO_DEVICE);  in ahash_final_ctx()
945 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,  in ahash_final_ctx()
948 	(edesc->sec4_sg + sec4_sg_src_index - 1)->len |= SEC4_SG_LEN_FIN;  in ahash_final_ctx()
950 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_final_ctx()
952 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_final_ctx()
957 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,  in ahash_final_ctx()
960 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_final_ctx()
962 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_final_ctx()
976 		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_final_ctx()
977 		kfree(edesc);  in ahash_final_ctx()
1000 	struct ahash_edesc *edesc;  in ahash_finup_ctx()  local
1010 	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes,  in ahash_finup_ctx()
1012 	if (!edesc) {  in ahash_finup_ctx()
1018 	desc = edesc->hw_desc;  in ahash_finup_ctx()
1021 	edesc->src_nents = src_nents;  in ahash_finup_ctx()
1022 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_finup_ctx()
1023 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_finup_ctx()
1027 				 edesc->sec4_sg, DMA_TO_DEVICE);  in ahash_finup_ctx()
1031 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,  in ahash_finup_ctx()
1035 	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg +  in ahash_finup_ctx()
1038 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_finup_ctx()
1040 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_finup_ctx()
1045 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +  in ahash_finup_ctx()
1048 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_finup_ctx()
1050 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_finup_ctx()
1064 		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_finup_ctx()
1065 		kfree(edesc);  in ahash_finup_ctx()
1083 	struct ahash_edesc *edesc;  in ahash_digest()  local
1093 	edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes + DESC_JOB_IO_LEN,  in ahash_digest()
1095 	if (!edesc) {  in ahash_digest()
1099 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_digest()
1101 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_digest()
1102 	edesc->src_nents = src_nents;  in ahash_digest()
1105 	desc = edesc->hw_desc;  in ahash_digest()
1109 		sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);  in ahash_digest()
1110 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_digest()
1112 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_digest()
1116 		src_dma = edesc->sec4_sg_dma;  in ahash_digest()
1124 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_digest()
1126 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_digest()
1140 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_digest()
1141 		kfree(edesc);  in ahash_digest()
1161 	struct ahash_edesc *edesc;  in ahash_final_no_ctx()  local
1166 	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | flags);  in ahash_final_no_ctx()
1167 	if (!edesc) {  in ahash_final_no_ctx()
1172 	edesc->sec4_sg_bytes = 0;  in ahash_final_no_ctx()
1174 	desc = edesc->hw_desc;  in ahash_final_no_ctx()
1185 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_final_no_ctx()
1187 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_final_no_ctx()
1191 	edesc->src_nents = 0;  in ahash_final_no_ctx()
1202 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_final_no_ctx()
1203 		kfree(edesc);  in ahash_final_no_ctx()
1225 	struct ahash_edesc *edesc;  in ahash_update_no_ctx()  local
1244 		edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN +  in ahash_update_no_ctx()
1246 		if (!edesc) {  in ahash_update_no_ctx()
1252 		edesc->src_nents = src_nents;  in ahash_update_no_ctx()
1253 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_no_ctx()
1254 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_update_no_ctx()
1256 		edesc->dst_dma = 0;  in ahash_update_no_ctx()
1258 		state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg,  in ahash_update_no_ctx()
1261 				   edesc->sec4_sg + 1);  in ahash_update_no_ctx()
1271 		desc = edesc->hw_desc;  in ahash_update_no_ctx()
1275 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_no_ctx()
1278 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_no_ctx()
1283 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);  in ahash_update_no_ctx()
1302 			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,  in ahash_update_no_ctx()
1304 			kfree(edesc);  in ahash_update_no_ctx()
1340 	struct ahash_edesc *edesc;  in ahash_finup_no_ctx()  local
1350 	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes,  in ahash_finup_no_ctx()
1352 	if (!edesc) {  in ahash_finup_no_ctx()
1358 	desc = edesc->hw_desc;  in ahash_finup_no_ctx()
1361 	edesc->src_nents = src_nents;  in ahash_finup_no_ctx()
1362 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_finup_no_ctx()
1363 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_finup_no_ctx()
1366 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf,  in ahash_finup_no_ctx()
1370 	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1);  in ahash_finup_no_ctx()
1372 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_finup_no_ctx()
1374 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_finup_no_ctx()
1379 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen +  in ahash_finup_no_ctx()
1382 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,  in ahash_finup_no_ctx()
1384 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {  in ahash_finup_no_ctx()
1398 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_finup_no_ctx()
1399 		kfree(edesc);  in ahash_finup_no_ctx()
1423 	struct ahash_edesc *edesc;  in ahash_update_first()  local
1440 		edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN +  in ahash_update_first()
1442 		if (!edesc) {  in ahash_update_first()
1448 		edesc->src_nents = src_nents;  in ahash_update_first()
1449 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_first()
1450 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +  in ahash_update_first()
1452 		edesc->dst_dma = 0;  in ahash_update_first()
1456 					   edesc->sec4_sg, 0);  in ahash_update_first()
1457 			edesc->sec4_sg_dma = dma_map_single(jrdev,  in ahash_update_first()
1458 							    edesc->sec4_sg,  in ahash_update_first()
1461 			if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_first()
1465 			src_dma = edesc->sec4_sg_dma;  in ahash_update_first()
1477 		desc = edesc->hw_desc;  in ahash_update_first()
1501 			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,  in ahash_update_first()
1503 			kfree(edesc);  in ahash_update_first()