111 	struct ccp_cmd_queue *cmd_q;  member
193 	struct ccp_cmd_queue *cmd_q = op->cmd_q;  in ccp_do_cmd()  local
194 	struct ccp_device *ccp = cmd_q->ccp;  in ccp_do_cmd()
204 	cmd_q->free_slots--;  in ccp_do_cmd()
206 	cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)  in ccp_do_cmd()
214 	if (op->ioc || !cmd_q->free_slots)  in ccp_do_cmd()
234 		ret = wait_event_interruptible(cmd_q->int_queue,  in ccp_do_cmd()
235 					       cmd_q->int_rcvd);  in ccp_do_cmd()
236 		if (ret || cmd_q->cmd_error) {  in ccp_do_cmd()
238 			cmd = (cmd_q->id << DEL_Q_ID_SHIFT)  in ccp_do_cmd()
248 			      | (cmd_q->id << DEL_Q_ID_SHIFT)  in ccp_do_cmd()
254 		cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);  in ccp_do_cmd()
256 		cmd_q->int_rcvd = 0;  in ccp_do_cmd()
554 				struct ccp_cmd_queue *cmd_q,  in ccp_init_dm_workarea()  argument
563 	wa->dev = cmd_q->ccp->dev;  in ccp_init_dm_workarea()
567 		wa->dma_pool = cmd_q->dma_pool;  in ccp_init_dm_workarea()
673 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)  in ccp_free_data()  argument
679 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,  in ccp_init_data()  argument
688 	ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,  in ccp_init_data()
693 	ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);  in ccp_init_data()
700 	ccp_free_data(data, cmd_q);  in ccp_init_data()
837 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,  in ccp_copy_to_from_ksb()  argument
845 	op.cmd_q = cmd_q;  in ccp_copy_to_from_ksb()
869 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,  in ccp_copy_to_ksb()  argument
873 	return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);  in ccp_copy_to_ksb()
876 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,  in ccp_copy_from_ksb()  argument
880 	return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);  in ccp_copy_from_ksb()
883 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,  in ccp_run_aes_cmac_cmd()  argument
920 	op.cmd_q = cmd_q;  in ccp_run_aes_cmac_cmd()
921 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_aes_cmac_cmd()
922 	op.ksb_key = cmd_q->ksb_key;  in ccp_run_aes_cmac_cmd()
923 	op.ksb_ctx = cmd_q->ksb_ctx;  in ccp_run_aes_cmac_cmd()
934 	ret = ccp_init_dm_workarea(&key, cmd_q,  in ccp_run_aes_cmac_cmd()
942 	ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,  in ccp_run_aes_cmac_cmd()
945 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmac_cmd()
953 	ret = ccp_init_dm_workarea(&ctx, cmd_q,  in ccp_run_aes_cmac_cmd()
961 	ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_aes_cmac_cmd()
964 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmac_cmd()
969 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmac_cmd()
980 			ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,  in ccp_run_aes_cmac_cmd()
984 				cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmac_cmd()
990 			ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_aes_cmac_cmd()
993 				cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmac_cmd()
1000 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmac_cmd()
1010 	ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_aes_cmac_cmd()
1013 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmac_cmd()
1022 	ccp_free_data(&src, cmd_q);  in ccp_run_aes_cmac_cmd()
1033 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)  in ccp_run_aes_cmd()  argument
1044 		return ccp_run_aes_cmac_cmd(cmd_q, cmd);  in ccp_run_aes_cmd()
1073 	op.cmd_q = cmd_q;  in ccp_run_aes_cmd()
1074 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_aes_cmd()
1075 	op.ksb_key = cmd_q->ksb_key;  in ccp_run_aes_cmd()
1076 	op.ksb_ctx = cmd_q->ksb_ctx;  in ccp_run_aes_cmd()
1087 	ret = ccp_init_dm_workarea(&key, cmd_q,  in ccp_run_aes_cmd()
1095 	ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,  in ccp_run_aes_cmd()
1098 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmd()
1106 	ret = ccp_init_dm_workarea(&ctx, cmd_q,  in ccp_run_aes_cmd()
1116 		ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_aes_cmd()
1119 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmd()
1131 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmd()
1140 		ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,  in ccp_run_aes_cmd()
1162 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmd()
1173 		ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_aes_cmd()
1176 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_aes_cmd()
1187 		ccp_free_data(&dst, cmd_q);  in ccp_run_aes_cmd()
1190 	ccp_free_data(&src, cmd_q);  in ccp_run_aes_cmd()
1201 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,  in ccp_run_xts_aes_cmd()  argument
1250 	op.cmd_q = cmd_q;  in ccp_run_xts_aes_cmd()
1251 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_xts_aes_cmd()
1252 	op.ksb_key = cmd_q->ksb_key;  in ccp_run_xts_aes_cmd()
1253 	op.ksb_ctx = cmd_q->ksb_ctx;  in ccp_run_xts_aes_cmd()
1263 	ret = ccp_init_dm_workarea(&key, cmd_q,  in ccp_run_xts_aes_cmd()
1272 	ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,  in ccp_run_xts_aes_cmd()
1275 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_xts_aes_cmd()
1283 	ret = ccp_init_dm_workarea(&ctx, cmd_q,  in ccp_run_xts_aes_cmd()
1290 	ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_xts_aes_cmd()
1293 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_xts_aes_cmd()
1304 	ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,  in ccp_run_xts_aes_cmd()
1313 		ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,  in ccp_run_xts_aes_cmd()
1327 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_xts_aes_cmd()
1337 	ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_xts_aes_cmd()
1340 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_xts_aes_cmd()
1350 		ccp_free_data(&dst, cmd_q);  in ccp_run_xts_aes_cmd()
1353 	ccp_free_data(&src, cmd_q);  in ccp_run_xts_aes_cmd()
1364 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)  in ccp_run_sha_cmd()  argument
1423 	op.cmd_q = cmd_q;  in ccp_run_sha_cmd()
1424 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_sha_cmd()
1425 	op.ksb_ctx = cmd_q->ksb_ctx;  in ccp_run_sha_cmd()
1433 	ret = ccp_init_dm_workarea(&ctx, cmd_q,  in ccp_run_sha_cmd()
1461 	ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_sha_cmd()
1464 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_sha_cmd()
1469 	ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,  in ccp_run_sha_cmd()
1481 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_sha_cmd()
1491 	ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,  in ccp_run_sha_cmd()
1494 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_sha_cmd()
1553 		ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);  in ccp_run_sha_cmd()
1561 	ccp_free_data(&src, cmd_q);  in ccp_run_sha_cmd()
1569 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)  in ccp_run_rsa_cmd()  argument
1596 	op.cmd_q = cmd_q;  in ccp_run_rsa_cmd()
1597 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_rsa_cmd()
1598 	op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);  in ccp_run_rsa_cmd()
1608 	ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);  in ccp_run_rsa_cmd()
1616 	ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,  in ccp_run_rsa_cmd()
1619 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_rsa_cmd()
1627 	ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);  in ccp_run_rsa_cmd()
1643 	ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,  in ccp_run_rsa_cmd()
1661 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_rsa_cmd()
1668 	ccp_free_data(&dst, cmd_q);  in ccp_run_rsa_cmd()
1677 	ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);  in ccp_run_rsa_cmd()
1682 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,  in ccp_run_passthru_cmd()  argument
1709 	op.cmd_q = cmd_q;  in ccp_run_passthru_cmd()
1710 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_passthru_cmd()
1714 		op.ksb_key = cmd_q->ksb_key;  in ccp_run_passthru_cmd()
1716 		ret = ccp_init_dm_workarea(&mask, cmd_q,  in ccp_run_passthru_cmd()
1724 		ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,  in ccp_run_passthru_cmd()
1727 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_passthru_cmd()
1739 	ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,  in ccp_run_passthru_cmd()
1748 		ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,  in ccp_run_passthru_cmd()
1786 			cmd->engine_error = cmd_q->cmd_error;  in ccp_run_passthru_cmd()
1800 		ccp_free_data(&dst, cmd_q);  in ccp_run_passthru_cmd()
1803 	ccp_free_data(&src, cmd_q);  in ccp_run_passthru_cmd()
1812 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)  in ccp_run_ecc_mm_cmd()  argument
1834 	op.cmd_q = cmd_q;  in ccp_run_ecc_mm_cmd()
1835 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_ecc_mm_cmd()
1842 	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,  in ccp_run_ecc_mm_cmd()
1881 	ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,  in ccp_run_ecc_mm_cmd()
1898 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_ecc_mm_cmd()
1921 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)  in ccp_run_ecc_pm_cmd()  argument
1959 	op.cmd_q = cmd_q;  in ccp_run_ecc_pm_cmd()
1960 	op.jobid = ccp_gen_jobid(cmd_q->ccp);  in ccp_run_ecc_pm_cmd()
1967 	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,  in ccp_run_ecc_pm_cmd()
2045 	ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,  in ccp_run_ecc_pm_cmd()
2062 		cmd->engine_error = cmd_q->cmd_error;  in ccp_run_ecc_pm_cmd()
2098 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)  in ccp_run_ecc_cmd()  argument
2112 		return ccp_run_ecc_mm_cmd(cmd_q, cmd);  in ccp_run_ecc_cmd()
2117 		return ccp_run_ecc_pm_cmd(cmd_q, cmd);  in ccp_run_ecc_cmd()
2124 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)  in ccp_run_cmd()  argument
2129 	cmd_q->cmd_error = 0;  in ccp_run_cmd()
2130 	cmd_q->int_rcvd = 0;  in ccp_run_cmd()
2131 	cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));  in ccp_run_cmd()
2135 		ret = ccp_run_aes_cmd(cmd_q, cmd);  in ccp_run_cmd()
2138 		ret = ccp_run_xts_aes_cmd(cmd_q, cmd);  in ccp_run_cmd()
2141 		ret = ccp_run_sha_cmd(cmd_q, cmd);  in ccp_run_cmd()
2144 		ret = ccp_run_rsa_cmd(cmd_q, cmd);  in ccp_run_cmd()
2147 		ret = ccp_run_passthru_cmd(cmd_q, cmd);  in ccp_run_cmd()
2150 		ret = ccp_run_ecc_cmd(cmd_q, cmd);  in ccp_run_cmd()