121 	struct ccp_mem src;  member
272 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_aes()
273 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_aes()
276 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_aes()
302 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_xts_aes()
303 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_xts_aes()
306 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_xts_aes()
328 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_sha()
329 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_sha()
332 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_sha()
356 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_rsa()
359 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_rsa()
376 	if (op->src.type == CCP_MEMTYPE_SYSTEM)  in ccp_perform_passthru()
377 		cr[1] = op->src.u.dma.length - 1;  in ccp_perform_passthru()
381 	if (op->src.type == CCP_MEMTYPE_SYSTEM) {  in ccp_perform_passthru()
382 		cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_passthru()
384 			| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_passthru()
389 		cr[2] = op->src.u.ksb * CCP_KSB_BYTES;  in ccp_perform_passthru()
417 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_ecc()
418 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_ecc()
420 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_ecc()
750 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,  in ccp_prepare_data()  argument
761 	sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;  in ccp_prepare_data()
762 	sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);  in ccp_prepare_data()
766 		sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);  in ccp_prepare_data()
785 		int cp_len = ccp_fill_queue_buf(src);  in ccp_prepare_data()
788 		op->src.u.dma.address = src->dm_wa.dma.address;  in ccp_prepare_data()
789 		op->src.u.dma.offset = 0;  in ccp_prepare_data()
790 		op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;  in ccp_prepare_data()
795 		op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);  in ccp_prepare_data()
796 		op->src.u.dma.offset = src->sg_wa.sg_used;  in ccp_prepare_data()
797 		op->src.u.dma.length = op_len & ~(block_size - 1);  in ccp_prepare_data()
799 		ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);  in ccp_prepare_data()
811 			op->dst.u.dma.length = op->src.u.dma.length;  in ccp_prepare_data()
818 			op->dst.u.dma.length = op->src.u.dma.length;  in ccp_prepare_data()
823 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,  in ccp_process_data()  argument
851 		op.src.type = CCP_MEMTYPE_KSB;  in ccp_copy_to_from_ksb()
852 		op.src.u.ksb = ksb;  in ccp_copy_to_from_ksb()
857 		op.src.type = CCP_MEMTYPE_SYSTEM;  in ccp_copy_to_from_ksb()
858 		op.src.u.dma.address = wa->dma.address;  in ccp_copy_to_from_ksb()
859 		op.src.u.dma.length = wa->length;  in ccp_copy_to_from_ksb()
888 	struct ccp_data src;  in ccp_run_aes_cmac_cmd()  local
904 	if (!aes->key || !aes->iv || !aes->src)  in ccp_run_aes_cmac_cmd()
969 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmac_cmd()
974 	while (src.sg_wa.bytes_left) {  in ccp_run_aes_cmac_cmd()
975 		ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);  in ccp_run_aes_cmac_cmd()
976 		if (aes->cmac_final && !src.sg_wa.bytes_left) {  in ccp_run_aes_cmac_cmd()
1004 		ccp_process_data(&src, NULL, &op);  in ccp_run_aes_cmac_cmd()
1022 	ccp_free_data(&src, cmd_q);  in ccp_run_aes_cmac_cmd()
1037 	struct ccp_data src, dst;  in ccp_run_aes_cmd()  local
1057 	if (!aes->key || !aes->src || !aes->dst)  in ccp_run_aes_cmd()
1128 	if (sg_virt(aes->src) == sg_virt(aes->dst))  in ccp_run_aes_cmd()
1131 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmd()
1138 		dst = src;  in ccp_run_aes_cmd()
1147 	while (src.sg_wa.bytes_left) {  in ccp_run_aes_cmd()
1148 		ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);  in ccp_run_aes_cmd()
1149 		if (!src.sg_wa.bytes_left) {  in ccp_run_aes_cmd()
1166 		ccp_process_data(&src, &dst, &op);  in ccp_run_aes_cmd()
1190 	ccp_free_data(&src, cmd_q);  in ccp_run_aes_cmd()
1206 	struct ccp_data src, dst;  in ccp_run_xts_aes_cmd()  local
1242 	if (!xts->key || !xts->iv || !xts->src || !xts->dst)  in ccp_run_xts_aes_cmd()
1301 	if (sg_virt(xts->src) == sg_virt(xts->dst))  in ccp_run_xts_aes_cmd()
1304 	ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,  in ccp_run_xts_aes_cmd()
1311 		dst = src;  in ccp_run_xts_aes_cmd()
1320 	while (src.sg_wa.bytes_left) {  in ccp_run_xts_aes_cmd()
1321 		ccp_prepare_data(&src, &dst, &op, unit_size, true);  in ccp_run_xts_aes_cmd()
1322 		if (!src.sg_wa.bytes_left)  in ccp_run_xts_aes_cmd()
1331 		ccp_process_data(&src, &dst, &op);  in ccp_run_xts_aes_cmd()
1353 	ccp_free_data(&src, cmd_q);  in ccp_run_xts_aes_cmd()
1368 	struct ccp_data src;  in ccp_run_sha_cmd()  local
1417 	if (!sha->src)  in ccp_run_sha_cmd()
1469 	ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,  in ccp_run_sha_cmd()
1474 	while (src.sg_wa.bytes_left) {  in ccp_run_sha_cmd()
1475 		ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);  in ccp_run_sha_cmd()
1476 		if (sha->final && !src.sg_wa.bytes_left)  in ccp_run_sha_cmd()
1485 		ccp_process_data(&src, NULL, &op);  in ccp_run_sha_cmd()
1545 		hmac_cmd.u.sha.src = &sg;  in ccp_run_sha_cmd()
1561 	ccp_free_data(&src, cmd_q);  in ccp_run_sha_cmd()
1572 	struct ccp_dm_workarea exp, src;  in ccp_run_rsa_cmd()  local
1581 	if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)  in ccp_run_rsa_cmd()
1627 	ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);  in ccp_run_rsa_cmd()
1631 	ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,  in ccp_run_rsa_cmd()
1635 	src.address += o_len;	/* Adjust the address for the copy operation */  in ccp_run_rsa_cmd()
1636 	ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,  in ccp_run_rsa_cmd()
1640 	src.address -= o_len;	/* Reset the address to original value */  in ccp_run_rsa_cmd()
1649 	op.src.u.dma.address = src.dma.address;  in ccp_run_rsa_cmd()
1650 	op.src.u.dma.offset = 0;  in ccp_run_rsa_cmd()
1651 	op.src.u.dma.length = i_len;  in ccp_run_rsa_cmd()
1671 	ccp_dm_free(&src);  in ccp_run_rsa_cmd()
1687 	struct ccp_data src, dst;  in ccp_run_passthru_cmd()  local
1696 	if (!pt->src || !pt->dst)  in ccp_run_passthru_cmd()
1736 	if (sg_virt(pt->src) == sg_virt(pt->dst))  in ccp_run_passthru_cmd()
1739 	ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,  in ccp_run_passthru_cmd()
1746 		dst = src;  in ccp_run_passthru_cmd()
1762 	for (i = 1; i <= src.sg_wa.dma_count; i++) {  in ccp_run_passthru_cmd()
1764 		    (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {  in ccp_run_passthru_cmd()
1769 		if (i == src.sg_wa.dma_count) {  in ccp_run_passthru_cmd()
1774 		op.src.type = CCP_MEMTYPE_SYSTEM;  in ccp_run_passthru_cmd()
1775 		op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);  in ccp_run_passthru_cmd()
1776 		op.src.u.dma.offset = 0;  in ccp_run_passthru_cmd()
1777 		op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);  in ccp_run_passthru_cmd()
1782 		op.dst.u.dma.length = op.src.u.dma.length;  in ccp_run_passthru_cmd()
1790 		dst.sg_wa.sg_used += src.sg_wa.sg->length;  in ccp_run_passthru_cmd()
1795 		src.sg_wa.sg = sg_next(src.sg_wa.sg);  in ccp_run_passthru_cmd()
1803 	ccp_free_data(&src, cmd_q);  in ccp_run_passthru_cmd()
1815 	struct ccp_dm_workarea src, dst;  in ccp_run_ecc_mm_cmd()  local
1842 	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,  in ccp_run_ecc_mm_cmd()
1850 	save = src.address;  in ccp_run_ecc_mm_cmd()
1853 	ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,  in ccp_run_ecc_mm_cmd()
1857 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_mm_cmd()
1860 	ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,  in ccp_run_ecc_mm_cmd()
1865 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_mm_cmd()
1869 		ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,  in ccp_run_ecc_mm_cmd()
1874 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_mm_cmd()
1878 	src.address = save;  in ccp_run_ecc_mm_cmd()
1887 	op.src.u.dma.address = src.dma.address;  in ccp_run_ecc_mm_cmd()
1888 	op.src.u.dma.offset = 0;  in ccp_run_ecc_mm_cmd()
1889 	op.src.u.dma.length = src.length;  in ccp_run_ecc_mm_cmd()
1916 	ccp_dm_free(&src);  in ccp_run_ecc_mm_cmd()
1924 	struct ccp_dm_workarea src, dst;  in ccp_run_ecc_pm_cmd()  local
1967 	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,  in ccp_run_ecc_pm_cmd()
1975 	save = src.address;  in ccp_run_ecc_pm_cmd()
1978 	ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,  in ccp_run_ecc_pm_cmd()
1982 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1985 	ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,  in ccp_run_ecc_pm_cmd()
1990 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1991 	ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,  in ccp_run_ecc_pm_cmd()
1996 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1999 	*src.address = 0x01;  in ccp_run_ecc_pm_cmd()
2000 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2004 		ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,  in ccp_run_ecc_pm_cmd()
2009 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2010 		ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,  in ccp_run_ecc_pm_cmd()
2015 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2018 		*src.address = 0x01;  in ccp_run_ecc_pm_cmd()
2019 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2022 		ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,  in ccp_run_ecc_pm_cmd()
2027 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2031 			ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,  in ccp_run_ecc_pm_cmd()
2037 			src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2042 	src.address = save;  in ccp_run_ecc_pm_cmd()
2051 	op.src.u.dma.address = src.dma.address;  in ccp_run_ecc_pm_cmd()
2052 	op.src.u.dma.offset = 0;  in ccp_run_ecc_pm_cmd()
2053 	op.src.u.dma.length = src.length;  in ccp_run_ecc_pm_cmd()
2093 	ccp_dm_free(&src);  in ccp_run_ecc_pm_cmd()