122 	struct ccp_mem src;  member
273 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_aes()
274 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_aes()
277 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_aes()
303 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_xts_aes()
304 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_xts_aes()
307 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_xts_aes()
329 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_sha()
330 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_sha()
333 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_sha()
357 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_rsa()
360 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_rsa()
377 	if (op->src.type == CCP_MEMTYPE_SYSTEM)  in ccp_perform_passthru()
378 		cr[1] = op->src.u.dma.length - 1;  in ccp_perform_passthru()
382 	if (op->src.type == CCP_MEMTYPE_SYSTEM) {  in ccp_perform_passthru()
383 		cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_passthru()
385 			| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_passthru()
390 		cr[2] = op->src.u.ksb * CCP_KSB_BYTES;  in ccp_perform_passthru()
418 	cr[1] = op->src.u.dma.length - 1;  in ccp_perform_ecc()
419 	cr[2] = ccp_addr_lo(&op->src.u.dma);  in ccp_perform_ecc()
421 		| ccp_addr_hi(&op->src.u.dma);  in ccp_perform_ecc()
746 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,  in ccp_prepare_data()  argument
757 	sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;  in ccp_prepare_data()
758 	sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);  in ccp_prepare_data()
762 		sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);  in ccp_prepare_data()
781 		int cp_len = ccp_fill_queue_buf(src);  in ccp_prepare_data()
784 		op->src.u.dma.address = src->dm_wa.dma.address;  in ccp_prepare_data()
785 		op->src.u.dma.offset = 0;  in ccp_prepare_data()
786 		op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;  in ccp_prepare_data()
791 		op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);  in ccp_prepare_data()
792 		op->src.u.dma.offset = src->sg_wa.sg_used;  in ccp_prepare_data()
793 		op->src.u.dma.length = op_len & ~(block_size - 1);  in ccp_prepare_data()
795 		ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);  in ccp_prepare_data()
807 			op->dst.u.dma.length = op->src.u.dma.length;  in ccp_prepare_data()
814 			op->dst.u.dma.length = op->src.u.dma.length;  in ccp_prepare_data()
819 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,  in ccp_process_data()  argument
847 		op.src.type = CCP_MEMTYPE_KSB;  in ccp_copy_to_from_ksb()
848 		op.src.u.ksb = ksb;  in ccp_copy_to_from_ksb()
853 		op.src.type = CCP_MEMTYPE_SYSTEM;  in ccp_copy_to_from_ksb()
854 		op.src.u.dma.address = wa->dma.address;  in ccp_copy_to_from_ksb()
855 		op.src.u.dma.length = wa->length;  in ccp_copy_to_from_ksb()
884 	struct ccp_data src;  in ccp_run_aes_cmac_cmd()  local
900 	if (!aes->key || !aes->iv || !aes->src)  in ccp_run_aes_cmac_cmd()
965 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmac_cmd()
970 	while (src.sg_wa.bytes_left) {  in ccp_run_aes_cmac_cmd()
971 		ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);  in ccp_run_aes_cmac_cmd()
972 		if (aes->cmac_final && !src.sg_wa.bytes_left) {  in ccp_run_aes_cmac_cmd()
1000 		ccp_process_data(&src, NULL, &op);  in ccp_run_aes_cmac_cmd()
1018 	ccp_free_data(&src, cmd_q);  in ccp_run_aes_cmac_cmd()
1033 	struct ccp_data src, dst;  in ccp_run_aes_cmd()  local
1053 	if (!aes->key || !aes->src || !aes->dst)  in ccp_run_aes_cmd()
1124 	if (sg_virt(aes->src) == sg_virt(aes->dst))  in ccp_run_aes_cmd()
1127 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmd()
1134 		dst = src;  in ccp_run_aes_cmd()
1143 	while (src.sg_wa.bytes_left) {  in ccp_run_aes_cmd()
1144 		ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);  in ccp_run_aes_cmd()
1145 		if (!src.sg_wa.bytes_left) {  in ccp_run_aes_cmd()
1162 		ccp_process_data(&src, &dst, &op);  in ccp_run_aes_cmd()
1186 	ccp_free_data(&src, cmd_q);  in ccp_run_aes_cmd()
1202 	struct ccp_data src, dst;  in ccp_run_xts_aes_cmd()  local
1238 	if (!xts->key || !xts->iv || !xts->src || !xts->dst)  in ccp_run_xts_aes_cmd()
1297 	if (sg_virt(xts->src) == sg_virt(xts->dst))  in ccp_run_xts_aes_cmd()
1300 	ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,  in ccp_run_xts_aes_cmd()
1307 		dst = src;  in ccp_run_xts_aes_cmd()
1316 	while (src.sg_wa.bytes_left) {  in ccp_run_xts_aes_cmd()
1317 		ccp_prepare_data(&src, &dst, &op, unit_size, true);  in ccp_run_xts_aes_cmd()
1318 		if (!src.sg_wa.bytes_left)  in ccp_run_xts_aes_cmd()
1327 		ccp_process_data(&src, &dst, &op);  in ccp_run_xts_aes_cmd()
1349 	ccp_free_data(&src, cmd_q);  in ccp_run_xts_aes_cmd()
1364 	struct ccp_data src;  in ccp_run_sha_cmd()  local
1413 	if (!sha->src)  in ccp_run_sha_cmd()
1465 	ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,  in ccp_run_sha_cmd()
1470 	while (src.sg_wa.bytes_left) {  in ccp_run_sha_cmd()
1471 		ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);  in ccp_run_sha_cmd()
1472 		if (sha->final && !src.sg_wa.bytes_left)  in ccp_run_sha_cmd()
1481 		ccp_process_data(&src, NULL, &op);  in ccp_run_sha_cmd()
1541 		hmac_cmd.u.sha.src = &sg;  in ccp_run_sha_cmd()
1557 	ccp_free_data(&src, cmd_q);  in ccp_run_sha_cmd()
1568 	struct ccp_dm_workarea exp, src;  in ccp_run_rsa_cmd()  local
1577 	if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)  in ccp_run_rsa_cmd()
1621 	ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);  in ccp_run_rsa_cmd()
1625 	ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES,  in ccp_run_rsa_cmd()
1627 	src.address += o_len;	/* Adjust the address for the copy operation */  in ccp_run_rsa_cmd()
1628 	ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES,  in ccp_run_rsa_cmd()
1630 	src.address -= o_len;	/* Reset the address to original value */  in ccp_run_rsa_cmd()
1639 	op.src.u.dma.address = src.dma.address;  in ccp_run_rsa_cmd()
1640 	op.src.u.dma.offset = 0;  in ccp_run_rsa_cmd()
1641 	op.src.u.dma.length = i_len;  in ccp_run_rsa_cmd()
1661 	ccp_dm_free(&src);  in ccp_run_rsa_cmd()
1677 	struct ccp_data src, dst;  in ccp_run_passthru_cmd()  local
1686 	if (!pt->src || !pt->dst)  in ccp_run_passthru_cmd()
1726 	if (sg_virt(pt->src) == sg_virt(pt->dst))  in ccp_run_passthru_cmd()
1729 	ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,  in ccp_run_passthru_cmd()
1736 		dst = src;  in ccp_run_passthru_cmd()
1752 	for (i = 1; i <= src.sg_wa.dma_count; i++) {  in ccp_run_passthru_cmd()
1754 		    (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {  in ccp_run_passthru_cmd()
1759 		if (i == src.sg_wa.dma_count) {  in ccp_run_passthru_cmd()
1764 		op.src.type = CCP_MEMTYPE_SYSTEM;  in ccp_run_passthru_cmd()
1765 		op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);  in ccp_run_passthru_cmd()
1766 		op.src.u.dma.offset = 0;  in ccp_run_passthru_cmd()
1767 		op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);  in ccp_run_passthru_cmd()
1772 		op.dst.u.dma.length = op.src.u.dma.length;  in ccp_run_passthru_cmd()
1780 		dst.sg_wa.sg_used += src.sg_wa.sg->length;  in ccp_run_passthru_cmd()
1785 		src.sg_wa.sg = sg_next(src.sg_wa.sg);  in ccp_run_passthru_cmd()
1793 	ccp_free_data(&src, cmd_q);  in ccp_run_passthru_cmd()
1805 	struct ccp_dm_workarea src, dst;  in ccp_run_ecc_mm_cmd()  local
1832 	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,  in ccp_run_ecc_mm_cmd()
1840 	save = src.address;  in ccp_run_ecc_mm_cmd()
1843 	ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,  in ccp_run_ecc_mm_cmd()
1845 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_mm_cmd()
1848 	ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,  in ccp_run_ecc_mm_cmd()
1851 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_mm_cmd()
1855 		ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,  in ccp_run_ecc_mm_cmd()
1858 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_mm_cmd()
1862 	src.address = save;  in ccp_run_ecc_mm_cmd()
1871 	op.src.u.dma.address = src.dma.address;  in ccp_run_ecc_mm_cmd()
1872 	op.src.u.dma.offset = 0;  in ccp_run_ecc_mm_cmd()
1873 	op.src.u.dma.length = src.length;  in ccp_run_ecc_mm_cmd()
1900 	ccp_dm_free(&src);  in ccp_run_ecc_mm_cmd()
1908 	struct ccp_dm_workarea src, dst;  in ccp_run_ecc_pm_cmd()  local
1951 	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,  in ccp_run_ecc_pm_cmd()
1959 	save = src.address;  in ccp_run_ecc_pm_cmd()
1962 	ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,  in ccp_run_ecc_pm_cmd()
1964 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1967 	ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,  in ccp_run_ecc_pm_cmd()
1970 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1971 	ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,  in ccp_run_ecc_pm_cmd()
1974 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1977 	*src.address = 0x01;  in ccp_run_ecc_pm_cmd()
1978 	src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1982 		ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,  in ccp_run_ecc_pm_cmd()
1985 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1986 		ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,  in ccp_run_ecc_pm_cmd()
1989 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1992 		*src.address = 0x01;  in ccp_run_ecc_pm_cmd()
1993 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
1996 		ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,  in ccp_run_ecc_pm_cmd()
1999 		src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2003 			ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,  in ccp_run_ecc_pm_cmd()
2006 			src.address += CCP_ECC_OPERAND_SIZE;  in ccp_run_ecc_pm_cmd()
2011 	src.address = save;  in ccp_run_ecc_pm_cmd()
2020 	op.src.u.dma.address = src.dma.address;  in ccp_run_ecc_pm_cmd()
2021 	op.src.u.dma.offset = 0;  in ccp_run_ecc_pm_cmd()
2022 	op.src.u.dma.length = src.length;  in ccp_run_ecc_pm_cmd()
2062 	ccp_dm_free(&src);  in ccp_run_ecc_pm_cmd()