64 	int (*complete)(struct aead_request *req, u32 flags);
91 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
94 	struct aead_request *req)  in crypto_gcm_reqctx()  argument
96 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_gcm_reqctx()
98 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_gcm_reqctx()
101 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)  in crypto_gcm_setkey_done()  argument
103 	struct crypto_gcm_setkey_result *result = req->data;  in crypto_gcm_setkey_done()
125 		struct ablkcipher_request req;  in crypto_gcm_setkey()  member
145 	ablkcipher_request_set_tfm(&data->req, ctr);  in crypto_gcm_setkey()
146 	ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |  in crypto_gcm_setkey()
150 	ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,  in crypto_gcm_setkey()
153 	err = crypto_ablkcipher_encrypt(&data->req);  in crypto_gcm_setkey()
195 static void crypto_gcm_init_common(struct aead_request *req)  in crypto_gcm_init_common()  argument
197 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_common()
202 	memcpy(pctx->iv, req->iv, 12);  in crypto_gcm_init_common()
207 	sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);  in crypto_gcm_init_common()
211 	if (req->src != req->dst) {  in crypto_gcm_init_common()
214 		sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);  in crypto_gcm_init_common()
220 static void crypto_gcm_init_crypt(struct aead_request *req,  in crypto_gcm_init_crypt()  argument
223 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_init_crypt()
225 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_crypt()
229 	dst = req->src == req->dst ? pctx->src : pctx->dst;  in crypto_gcm_init_crypt()
245 static int gcm_hash_update(struct aead_request *req,  in gcm_hash_update()  argument
250 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_update()
253 	ahash_request_set_callback(ahreq, flags, compl, req);  in gcm_hash_update()
259 static int gcm_hash_remain(struct aead_request *req,  in gcm_hash_remain()  argument
263 	return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);  in gcm_hash_remain()
266 static int gcm_hash_len(struct aead_request *req, u32 flags)  in gcm_hash_len()  argument
268 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len()
273 	lengths.a = cpu_to_be64(req->assoclen * 8);  in gcm_hash_len()
277 	ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);  in gcm_hash_len()
284 static int gcm_hash_len_continue(struct aead_request *req, u32 flags)  in gcm_hash_len_continue()  argument
286 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len_continue()
289 	return gctx->complete(req, flags);  in gcm_hash_len_continue()
294 	struct aead_request *req = areq->data;  in gcm_hash_len_done()  local
299 	err = gcm_hash_len_continue(req, 0);  in gcm_hash_len_done()
304 	aead_request_complete(req, err);  in gcm_hash_len_done()
307 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_remain_continue()  argument
309 	return gcm_hash_len(req, flags) ?:  in gcm_hash_crypt_remain_continue()
310 	       gcm_hash_len_continue(req, flags);  in gcm_hash_crypt_remain_continue()
316 	struct aead_request *req = areq->data;  in gcm_hash_crypt_remain_done()  local
321 	err = gcm_hash_crypt_remain_continue(req, 0);  in gcm_hash_crypt_remain_done()
326 	aead_request_complete(req, err);  in gcm_hash_crypt_remain_done()
329 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_continue()  argument
331 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_crypt_continue()
337 		return gcm_hash_remain(req, remain,  in gcm_hash_crypt_continue()
339 		       gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
341 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
346 	struct aead_request *req = areq->data;  in gcm_hash_crypt_done()  local
351 	err = gcm_hash_crypt_continue(req, 0);  in gcm_hash_crypt_done()
356 	aead_request_complete(req, err);  in gcm_hash_crypt_done()
359 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_remain_continue()  argument
361 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_assoc_remain_continue()
365 		return gcm_hash_update(req, gcm_hash_crypt_done,  in gcm_hash_assoc_remain_continue()
367 		       gcm_hash_crypt_continue(req, flags);  in gcm_hash_assoc_remain_continue()
369 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_assoc_remain_continue()
375 	struct aead_request *req = areq->data;  in gcm_hash_assoc_remain_done()  local
380 	err = gcm_hash_assoc_remain_continue(req, 0);  in gcm_hash_assoc_remain_done()
385 	aead_request_complete(req, err);  in gcm_hash_assoc_remain_done()
388 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_continue()  argument
392 	remain = gcm_remain(req->assoclen);  in gcm_hash_assoc_continue()
394 		return gcm_hash_remain(req, remain,  in gcm_hash_assoc_continue()
396 		       gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
398 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
403 	struct aead_request *req = areq->data;  in gcm_hash_assoc_done()  local
408 	err = gcm_hash_assoc_continue(req, 0);  in gcm_hash_assoc_done()
413 	aead_request_complete(req, err);  in gcm_hash_assoc_done()
416 static int gcm_hash_init_continue(struct aead_request *req, u32 flags)  in gcm_hash_init_continue()  argument
418 	if (req->assoclen)  in gcm_hash_init_continue()
419 		return gcm_hash_update(req, gcm_hash_assoc_done,  in gcm_hash_init_continue()
420 				       req->src, req->assoclen, flags) ?:  in gcm_hash_init_continue()
421 		       gcm_hash_assoc_continue(req, flags);  in gcm_hash_init_continue()
423 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_init_continue()
428 	struct aead_request *req = areq->data;  in gcm_hash_init_done()  local
433 	err = gcm_hash_init_continue(req, 0);  in gcm_hash_init_done()
438 	aead_request_complete(req, err);  in gcm_hash_init_done()
441 static int gcm_hash(struct aead_request *req, u32 flags)  in gcm_hash()  argument
443 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash()
445 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in gcm_hash()
449 	ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);  in gcm_hash()
451 	       gcm_hash_init_continue(req, flags);  in gcm_hash()
454 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)  in gcm_enc_copy_hash()  argument
456 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_enc_copy_hash()
457 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in gcm_enc_copy_hash()
461 	scatterwalk_map_and_copy(auth_tag, req->dst,  in gcm_enc_copy_hash()
462 				 req->assoclen + req->cryptlen,  in gcm_enc_copy_hash()
467 static int gcm_encrypt_continue(struct aead_request *req, u32 flags)  in gcm_encrypt_continue()  argument
469 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_encrypt_continue()
472 	gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);  in gcm_encrypt_continue()
473 	gctx->cryptlen = req->cryptlen;  in gcm_encrypt_continue()
476 	return gcm_hash(req, flags);  in gcm_encrypt_continue()
481 	struct aead_request *req = areq->data;  in gcm_encrypt_done()  local
486 	err = gcm_encrypt_continue(req, 0);  in gcm_encrypt_done()
491 	aead_request_complete(req, err);  in gcm_encrypt_done()
494 static int crypto_gcm_encrypt(struct aead_request *req)  in crypto_gcm_encrypt()  argument
496 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_encrypt()
498 	u32 flags = aead_request_flags(req);  in crypto_gcm_encrypt()
500 	crypto_gcm_init_common(req);  in crypto_gcm_encrypt()
501 	crypto_gcm_init_crypt(req, req->cryptlen);  in crypto_gcm_encrypt()
502 	ablkcipher_request_set_callback(abreq, flags, gcm_encrypt_done, req);  in crypto_gcm_encrypt()
505 	       gcm_encrypt_continue(req, flags);  in crypto_gcm_encrypt()
508 static int crypto_gcm_verify(struct aead_request *req)  in crypto_gcm_verify()  argument
510 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_verify()
511 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_verify()
515 	unsigned int cryptlen = req->cryptlen - authsize;  in crypto_gcm_verify()
518 	scatterwalk_map_and_copy(iauth_tag, req->src,  in crypto_gcm_verify()
519 				 req->assoclen + cryptlen, authsize, 0);  in crypto_gcm_verify()
525 	struct aead_request *req = areq->data;  in gcm_decrypt_done()  local
528 		err = crypto_gcm_verify(req);  in gcm_decrypt_done()
530 	aead_request_complete(req, err);  in gcm_decrypt_done()
533 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)  in gcm_dec_hash_continue()  argument
535 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_dec_hash_continue()
539 	crypto_gcm_init_crypt(req, gctx->cryptlen);  in gcm_dec_hash_continue()
540 	ablkcipher_request_set_callback(abreq, flags, gcm_decrypt_done, req);  in gcm_dec_hash_continue()
541 	return crypto_ablkcipher_decrypt(abreq) ?: crypto_gcm_verify(req);  in gcm_dec_hash_continue()
544 static int crypto_gcm_decrypt(struct aead_request *req)  in crypto_gcm_decrypt()  argument
546 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_decrypt()
547 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_decrypt()
550 	unsigned int cryptlen = req->cryptlen;  in crypto_gcm_decrypt()
551 	u32 flags = aead_request_flags(req);  in crypto_gcm_decrypt()
555 	crypto_gcm_init_common(req);  in crypto_gcm_decrypt()
561 	return gcm_hash(req, flags);  in crypto_gcm_decrypt()
823 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)  in crypto_rfc4106_crypt()  argument
825 	struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4106_crypt()
826 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4106_crypt()
834 	scatterwalk_map_and_copy(iv + 12, req->src, 0, req->assoclen - 8, 0);  in crypto_rfc4106_crypt()
837 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4106_crypt()
840 	sg_set_buf(rctx->src, iv + 12, req->assoclen - 8);  in crypto_rfc4106_crypt()
841 	sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);  in crypto_rfc4106_crypt()
845 	if (req->src != req->dst) {  in crypto_rfc4106_crypt()
847 		sg_set_buf(rctx->dst, iv + 12, req->assoclen - 8);  in crypto_rfc4106_crypt()
848 		sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);  in crypto_rfc4106_crypt()
854 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,  in crypto_rfc4106_crypt()
855 				  req->base.data);  in crypto_rfc4106_crypt()
857 			       req->src == req->dst ? rctx->src : rctx->dst,  in crypto_rfc4106_crypt()
858 			       req->cryptlen, iv);  in crypto_rfc4106_crypt()
859 	aead_request_set_ad(subreq, req->assoclen - 8);  in crypto_rfc4106_crypt()
864 static int crypto_rfc4106_encrypt(struct aead_request *req)  in crypto_rfc4106_encrypt()  argument
866 	if (req->assoclen != 16 && req->assoclen != 20)  in crypto_rfc4106_encrypt()
869 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_encrypt()
871 	return crypto_aead_encrypt(req);  in crypto_rfc4106_encrypt()
874 static int crypto_rfc4106_decrypt(struct aead_request *req)  in crypto_rfc4106_decrypt()  argument
876 	if (req->assoclen != 16 && req->assoclen != 20)  in crypto_rfc4106_decrypt()
879 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_decrypt()
881 	return crypto_aead_decrypt(req);  in crypto_rfc4106_decrypt()
1049 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)  in crypto_rfc4543_crypt()  argument
1051 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_crypt()
1053 	struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4543_crypt()
1060 	if (req->src != req->dst) {  in crypto_rfc4543_crypt()
1061 		err = crypto_rfc4543_copy_src_to_dst(req, enc);  in crypto_rfc4543_crypt()
1067 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4543_crypt()
1070 	aead_request_set_callback(subreq, req->base.flags,  in crypto_rfc4543_crypt()
1071 				  req->base.complete, req->base.data);  in crypto_rfc4543_crypt()
1072 	aead_request_set_crypt(subreq, req->src, req->dst,  in crypto_rfc4543_crypt()
1074 	aead_request_set_ad(subreq, req->assoclen + req->cryptlen -  in crypto_rfc4543_crypt()
1080 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)  in crypto_rfc4543_copy_src_to_dst()  argument
1082 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_copy_src_to_dst()
1085 	unsigned int nbytes = req->assoclen + req->cryptlen -  in crypto_rfc4543_copy_src_to_dst()
1091 	return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);  in crypto_rfc4543_copy_src_to_dst()
1094 static int crypto_rfc4543_encrypt(struct aead_request *req)  in crypto_rfc4543_encrypt()  argument
1096 	return crypto_rfc4543_crypt(req, true);  in crypto_rfc4543_encrypt()
1099 static int crypto_rfc4543_decrypt(struct aead_request *req)  in crypto_rfc4543_decrypt()  argument
1101 	return crypto_rfc4543_crypt(req, false);  in crypto_rfc4543_decrypt()