761 			     struct talitos_edesc *edesc,  in talitos_sg_unmap()  argument
765 	unsigned int src_nents = edesc->src_nents ? : 1;  in talitos_sg_unmap()
766 	unsigned int dst_nents = edesc->dst_nents ? : 1;  in talitos_sg_unmap()
769 		if (edesc->src_chained)  in talitos_sg_unmap()
775 			if (edesc->dst_chained)  in talitos_sg_unmap()
783 		if (edesc->src_chained)  in talitos_sg_unmap()
790 			    struct talitos_edesc *edesc,  in ipsec_esp_unmap()  argument
793 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);  in ipsec_esp_unmap()
794 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);  in ipsec_esp_unmap()
795 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);  in ipsec_esp_unmap()
796 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);  in ipsec_esp_unmap()
798 	if (edesc->assoc_chained)  in ipsec_esp_unmap()
803 			     edesc->assoc_nents ? edesc->assoc_nents - 1 : 1,  in ipsec_esp_unmap()
806 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst);  in ipsec_esp_unmap()
808 	if (edesc->dma_len)  in ipsec_esp_unmap()
809 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in ipsec_esp_unmap()
823 	struct talitos_edesc *edesc;  in ipsec_esp_encrypt_done()  local
827 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_encrypt_done()
829 	ipsec_esp_unmap(dev, edesc, areq);  in ipsec_esp_encrypt_done()
832 	if (edesc->dst_nents) {  in ipsec_esp_encrypt_done()
833 		icvdata = &edesc->link_tbl[edesc->src_nents +  in ipsec_esp_encrypt_done()
834 					   edesc->dst_nents + 2 +  in ipsec_esp_encrypt_done()
835 					   edesc->assoc_nents];  in ipsec_esp_encrypt_done()
836 		sg = sg_last(areq->dst, edesc->dst_nents);  in ipsec_esp_encrypt_done()
841 	kfree(edesc);  in ipsec_esp_encrypt_done()
853 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_swauth_done()  local
857 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_swauth_done()
859 	ipsec_esp_unmap(dev, edesc, req);  in ipsec_esp_decrypt_swauth_done()
863 		if (edesc->dma_len)  in ipsec_esp_decrypt_swauth_done()
864 			icvdata = &edesc->link_tbl[edesc->src_nents +  in ipsec_esp_decrypt_swauth_done()
865 						   edesc->dst_nents + 2 +  in ipsec_esp_decrypt_swauth_done()
866 						   edesc->assoc_nents];  in ipsec_esp_decrypt_swauth_done()
868 			icvdata = &edesc->link_tbl[0];  in ipsec_esp_decrypt_swauth_done()
870 		sg = sg_last(req->dst, edesc->dst_nents ? : 1);  in ipsec_esp_decrypt_swauth_done()
875 	kfree(edesc);  in ipsec_esp_decrypt_swauth_done()
885 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_hwauth_done()  local
887 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_hwauth_done()
889 	ipsec_esp_unmap(dev, edesc, req);  in ipsec_esp_decrypt_hwauth_done()
896 	kfree(edesc);  in ipsec_esp_decrypt_hwauth_done()
940 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,  in ipsec_esp()  argument
948 	struct talitos_desc *desc = &edesc->desc;  in ipsec_esp()
961 	if (edesc->assoc_nents) {  in ipsec_esp()
962 		int tbl_off = edesc->src_nents + edesc->dst_nents + 2;  in ipsec_esp()
963 		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];  in ipsec_esp()
965 		to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *  in ipsec_esp()
970 		sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1,  in ipsec_esp()
977 		to_talitos_ptr(tbl_ptr, edesc->iv_dma);  in ipsec_esp()
981 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in ipsec_esp()
982 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in ipsec_esp()
988 			to_talitos_ptr(&desc->ptr[1], edesc->iv_dma);  in ipsec_esp()
993 	to_talitos_ptr(&desc->ptr[2], edesc->iv_dma);  in ipsec_esp()
997 	dma_sync_single_for_device(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);  in ipsec_esp()
1013 	sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,  in ipsec_esp()
1016 				  edesc->src_chained);  in ipsec_esp()
1023 		if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)  in ipsec_esp()
1027 					  &edesc->link_tbl[0]);  in ipsec_esp()
1030 			to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl);  in ipsec_esp()
1031 			dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in ipsec_esp()
1032 						   edesc->dma_len,  in ipsec_esp()
1047 					  edesc->dst_nents ? : 1,  in ipsec_esp()
1048 					  DMA_FROM_DEVICE, edesc->dst_chained);  in ipsec_esp()
1053 		int tbl_off = edesc->src_nents + 1;  in ipsec_esp()
1054 		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];  in ipsec_esp()
1056 		to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +  in ipsec_esp()
1069 		to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +  in ipsec_esp()
1070 			       (tbl_off + edesc->dst_nents + 1 +  in ipsec_esp()
1071 				edesc->assoc_nents) *  in ipsec_esp()
1074 		dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,  in ipsec_esp()
1075 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in ipsec_esp()
1084 		ipsec_esp_unmap(dev, edesc, areq);  in ipsec_esp()
1085 		kfree(edesc);  in ipsec_esp()
1126 	struct talitos_edesc *edesc;  in talitos_edesc_alloc()  local
1185 	edesc = kmalloc(alloc_len, GFP_DMA | flags);  in talitos_edesc_alloc()
1186 	if (!edesc) {  in talitos_edesc_alloc()
1201 	edesc->assoc_nents = assoc_nents;  in talitos_edesc_alloc()
1202 	edesc->src_nents = src_nents;  in talitos_edesc_alloc()
1203 	edesc->dst_nents = dst_nents;  in talitos_edesc_alloc()
1204 	edesc->assoc_chained = assoc_chained;  in talitos_edesc_alloc()
1205 	edesc->src_chained = src_chained;  in talitos_edesc_alloc()
1206 	edesc->dst_chained = dst_chained;  in talitos_edesc_alloc()
1207 	edesc->iv_dma = iv_dma;  in talitos_edesc_alloc()
1208 	edesc->dma_len = dma_len;  in talitos_edesc_alloc()
1210 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],  in talitos_edesc_alloc()
1211 						     edesc->dma_len,  in talitos_edesc_alloc()
1214 	return edesc;  in talitos_edesc_alloc()
1234 	struct talitos_edesc *edesc;  in aead_encrypt()  local
1237 	edesc = aead_edesc_alloc(req, req->iv, 0, true);  in aead_encrypt()
1238 	if (IS_ERR(edesc))  in aead_encrypt()
1239 		return PTR_ERR(edesc);  in aead_encrypt()
1242 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in aead_encrypt()
1244 	return ipsec_esp(edesc, req, 0, ipsec_esp_encrypt_done);  in aead_encrypt()
1253 	struct talitos_edesc *edesc;  in aead_decrypt()  local
1260 	edesc = aead_edesc_alloc(req, req->iv, 1, false);  in aead_decrypt()
1261 	if (IS_ERR(edesc))  in aead_decrypt()
1262 		return PTR_ERR(edesc);  in aead_decrypt()
1265 	    ((!edesc->src_nents && !edesc->dst_nents) ||  in aead_decrypt()
1269 		edesc->desc.hdr = ctx->desc_hdr_template |  in aead_decrypt()
1274 		edesc->desc.hdr_lo = 0;  in aead_decrypt()
1276 		return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_hwauth_done);  in aead_decrypt()
1280 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in aead_decrypt()
1283 	if (edesc->dma_len)  in aead_decrypt()
1284 		icvdata = &edesc->link_tbl[edesc->src_nents +  in aead_decrypt()
1285 					   edesc->dst_nents + 2 +  in aead_decrypt()
1286 					   edesc->assoc_nents];  in aead_decrypt()
1288 		icvdata = &edesc->link_tbl[0];  in aead_decrypt()
1290 	sg = sg_last(req->src, edesc->src_nents ? : 1);  in aead_decrypt()
1295 	return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_swauth_done);  in aead_decrypt()
1303 	struct talitos_edesc *edesc;  in aead_givencrypt()  local
1306 	edesc = aead_edesc_alloc(areq, req->giv, 0, true);  in aead_givencrypt()
1307 	if (IS_ERR(edesc))  in aead_givencrypt()
1308 		return PTR_ERR(edesc);  in aead_givencrypt()
1311 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in aead_givencrypt()
1317 	return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done);  in aead_givencrypt()
1332 				  struct talitos_edesc *edesc,  in common_nonsnoop_unmap()  argument
1335 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_unmap()
1336 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);  in common_nonsnoop_unmap()
1337 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);  in common_nonsnoop_unmap()
1339 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst);  in common_nonsnoop_unmap()
1341 	if (edesc->dma_len)  in common_nonsnoop_unmap()
1342 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_unmap()
1351 	struct talitos_edesc *edesc;  in ablkcipher_done()  local
1353 	edesc = container_of(desc, struct talitos_edesc, desc);  in ablkcipher_done()
1355 	common_nonsnoop_unmap(dev, edesc, areq);  in ablkcipher_done()
1357 	kfree(edesc);  in ablkcipher_done()
1362 static int common_nonsnoop(struct talitos_edesc *edesc,  in common_nonsnoop()  argument
1371 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop()
1382 	to_talitos_ptr(&desc->ptr[1], edesc->iv_dma);  in common_nonsnoop()
1396 	sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,  in common_nonsnoop()
1399 				  edesc->src_chained);  in common_nonsnoop()
1405 					  &edesc->link_tbl[0]);  in common_nonsnoop()
1407 			to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);  in common_nonsnoop()
1409 			dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop()
1410 						   edesc->dma_len,  in common_nonsnoop()
1425 					  edesc->dst_nents ? : 1,  in common_nonsnoop()
1426 					  DMA_FROM_DEVICE, edesc->dst_chained);  in common_nonsnoop()
1432 			&edesc->link_tbl[edesc->src_nents + 1];  in common_nonsnoop()
1434 		to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +  in common_nonsnoop()
1435 					      (edesc->src_nents + 1) *  in common_nonsnoop()
1440 		dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,  in common_nonsnoop()
1441 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop()
1455 		common_nonsnoop_unmap(dev, edesc, areq);  in common_nonsnoop()
1456 		kfree(edesc);  in common_nonsnoop()
1477 	struct talitos_edesc *edesc;  in ablkcipher_encrypt()  local
1480 	edesc = ablkcipher_edesc_alloc(areq, true);  in ablkcipher_encrypt()
1481 	if (IS_ERR(edesc))  in ablkcipher_encrypt()
1482 		return PTR_ERR(edesc);  in ablkcipher_encrypt()
1485 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in ablkcipher_encrypt()
1487 	return common_nonsnoop(edesc, areq, ablkcipher_done);  in ablkcipher_encrypt()
1494 	struct talitos_edesc *edesc;  in ablkcipher_decrypt()  local
1497 	edesc = ablkcipher_edesc_alloc(areq, false);  in ablkcipher_decrypt()
1498 	if (IS_ERR(edesc))  in ablkcipher_decrypt()
1499 		return PTR_ERR(edesc);  in ablkcipher_decrypt()
1501 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in ablkcipher_decrypt()
1503 	return common_nonsnoop(edesc, areq, ablkcipher_done);  in ablkcipher_decrypt()
1507 				       struct talitos_edesc *edesc,  in common_nonsnoop_hash_unmap()  argument
1512 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_hash_unmap()
1515 	if (edesc->desc.ptr[1].len)  in common_nonsnoop_hash_unmap()
1516 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],  in common_nonsnoop_hash_unmap()
1519 	if (edesc->desc.ptr[2].len)  in common_nonsnoop_hash_unmap()
1520 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],  in common_nonsnoop_hash_unmap()
1523 	talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL);  in common_nonsnoop_hash_unmap()
1525 	if (edesc->dma_len)  in common_nonsnoop_hash_unmap()
1526 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_hash_unmap()
1536 	struct talitos_edesc *edesc =  in ahash_done()  local
1545 	common_nonsnoop_hash_unmap(dev, edesc, areq);  in ahash_done()
1547 	kfree(edesc);  in ahash_done()
1552 static int common_nonsnoop_hash(struct talitos_edesc *edesc,  in common_nonsnoop_hash()  argument
1562 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash()
1595 				  edesc->src_nents ? : 1,  in common_nonsnoop_hash()
1596 				  DMA_TO_DEVICE, edesc->src_chained);  in common_nonsnoop_hash()
1602 					  &edesc->link_tbl[0]);  in common_nonsnoop_hash()
1605 			to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);  in common_nonsnoop_hash()
1607 						   edesc->dma_link_tbl,  in common_nonsnoop_hash()
1608 						   edesc->dma_len,  in common_nonsnoop_hash()
1635 		common_nonsnoop_hash_unmap(dev, edesc, areq);  in common_nonsnoop_hash()
1636 		kfree(edesc);  in common_nonsnoop_hash()
1701 	struct talitos_edesc *edesc;  in ahash_process_req()  local
1754 	edesc = ahash_edesc_alloc(areq, nbytes_to_hash);  in ahash_process_req()
1755 	if (IS_ERR(edesc))  in ahash_process_req()
1756 		return PTR_ERR(edesc);  in ahash_process_req()
1758 	edesc->desc.hdr = ctx->desc_hdr_template;  in ahash_process_req()
1762 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;  in ahash_process_req()
1764 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;  in ahash_process_req()
1768 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;  in ahash_process_req()
1774 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;  in ahash_process_req()
1776 	return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,  in ahash_process_req()