1/* 2 This file is provided under a dual BSD/GPLv2 license. When using or 3 redistributing this file, you may do so under either license. 4 5 GPL LICENSE SUMMARY 6 Copyright(c) 2014 Intel Corporation. 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of version 2 of the GNU General Public License as 9 published by the Free Software Foundation. 10 11 This program is distributed in the hope that it will be useful, but 12 WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 General Public License for more details. 15 16 Contact Information: 17 qat-linux@intel.com 18 19 BSD LICENSE 20 Copyright(c) 2014 Intel Corporation. 21 Redistribution and use in source and binary forms, with or without 22 modification, are permitted provided that the following conditions 23 are met: 24 25 * Redistributions of source code must retain the above copyright 26 notice, this list of conditions and the following disclaimer. 27 * Redistributions in binary form must reproduce the above copyright 28 notice, this list of conditions and the following disclaimer in 29 the documentation and/or other materials provided with the 30 distribution. 31 * Neither the name of Intel Corporation nor the names of its 32 contributors may be used to endorse or promote products derived 33 from this software without specific prior written permission. 34 35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46*/ 47#include <linux/module.h> 48#include <linux/slab.h> 49#include <linux/crypto.h> 50#include <crypto/aead.h> 51#include <crypto/aes.h> 52#include <crypto/sha.h> 53#include <crypto/hash.h> 54#include <crypto/algapi.h> 55#include <crypto/authenc.h> 56#include <crypto/rng.h> 57#include <linux/dma-mapping.h> 58#include "adf_accel_devices.h" 59#include "adf_transport.h" 60#include "adf_common_drv.h" 61#include "qat_crypto.h" 62#include "icp_qat_hw.h" 63#include "icp_qat_fw.h" 64#include "icp_qat_fw_la.h" 65 66#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \ 67 ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ 68 ICP_QAT_HW_CIPHER_NO_CONVERT, \ 69 ICP_QAT_HW_CIPHER_ENCRYPT) 70 71#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \ 72 ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ 73 ICP_QAT_HW_CIPHER_KEY_CONVERT, \ 74 ICP_QAT_HW_CIPHER_DECRYPT) 75 76static DEFINE_MUTEX(algs_lock); 77static unsigned int active_devs; 78 79struct qat_alg_buf { 80 uint32_t len; 81 uint32_t resrvd; 82 uint64_t addr; 83} __packed; 84 85struct qat_alg_buf_list { 86 uint64_t resrvd; 87 uint32_t num_bufs; 88 uint32_t num_mapped_bufs; 89 struct qat_alg_buf bufers[]; 90} __packed __aligned(64); 91 92/* Common content descriptor */ 93struct qat_alg_cd { 94 union { 95 struct qat_enc { /* Encrypt content desc */ 96 struct icp_qat_hw_cipher_algo_blk cipher; 97 struct icp_qat_hw_auth_algo_blk hash; 98 } qat_enc_cd; 99 struct qat_dec { /* Decrytp content desc */ 100 struct icp_qat_hw_auth_algo_blk hash; 101 struct icp_qat_hw_cipher_algo_blk cipher; 102 } qat_dec_cd; 103 }; 104} __aligned(64); 105 106struct qat_alg_aead_ctx { 107 struct qat_alg_cd *enc_cd; 108 struct qat_alg_cd *dec_cd; 109 dma_addr_t enc_cd_paddr; 110 dma_addr_t dec_cd_paddr; 111 struct icp_qat_fw_la_bulk_req enc_fw_req; 112 struct icp_qat_fw_la_bulk_req dec_fw_req; 113 struct crypto_shash *hash_tfm; 114 enum icp_qat_hw_auth_algo qat_hash_alg; 115 struct qat_crypto_instance *inst; 116 struct crypto_tfm *tfm; 117 uint8_t salt[AES_BLOCK_SIZE]; 118 spinlock_t lock; /* protects qat_alg_aead_ctx struct */ 119}; 120 121struct qat_alg_ablkcipher_ctx { 122 struct icp_qat_hw_cipher_algo_blk *enc_cd; 123 struct icp_qat_hw_cipher_algo_blk *dec_cd; 124 dma_addr_t enc_cd_paddr; 125 dma_addr_t dec_cd_paddr; 126 struct icp_qat_fw_la_bulk_req enc_fw_req; 127 struct icp_qat_fw_la_bulk_req dec_fw_req; 128 struct qat_crypto_instance *inst; 129 struct crypto_tfm *tfm; 130 spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */ 131}; 132 133static int get_current_node(void) 134{ 135 return cpu_data(current_thread_info()->cpu).phys_proc_id; 136} 137 138static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) 139{ 140 switch (qat_hash_alg) { 141 case ICP_QAT_HW_AUTH_ALGO_SHA1: 142 return ICP_QAT_HW_SHA1_STATE1_SZ; 143 case ICP_QAT_HW_AUTH_ALGO_SHA256: 144 return ICP_QAT_HW_SHA256_STATE1_SZ; 145 case ICP_QAT_HW_AUTH_ALGO_SHA512: 146 return ICP_QAT_HW_SHA512_STATE1_SZ; 147 default: 148 return -EFAULT; 149 }; 150 return -EFAULT; 151} 152 153static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, 154 struct qat_alg_aead_ctx *ctx, 155 const uint8_t *auth_key, 156 unsigned int auth_keylen) 157{ 158 SHASH_DESC_ON_STACK(shash, ctx->hash_tfm); 159 struct sha1_state sha1; 160 struct sha256_state sha256; 161 struct sha512_state sha512; 162 int block_size = crypto_shash_blocksize(ctx->hash_tfm); 163 int digest_size = crypto_shash_digestsize(ctx->hash_tfm); 164 char ipad[block_size]; 165 char opad[block_size]; 166 __be32 *hash_state_out; 167 __be64 *hash512_state_out; 168 int i, offset; 169 170 memset(ipad, 0, block_size); 171 memset(opad, 0, block_size); 172 shash->tfm = ctx->hash_tfm; 173 shash->flags = 0x0; 174 175 if (auth_keylen > block_size) { 176 int ret = crypto_shash_digest(shash, auth_key, 177 auth_keylen, ipad); 178 if (ret) 179 return ret; 180 181 memcpy(opad, ipad, digest_size); 182 } else { 183 memcpy(ipad, auth_key, auth_keylen); 184 memcpy(opad, auth_key, auth_keylen); 185 } 186 187 for (i = 0; i < block_size; i++) { 188 char *ipad_ptr = ipad + i; 189 char *opad_ptr = opad + i; 190 *ipad_ptr ^= 0x36; 191 *opad_ptr ^= 0x5C; 192 } 193 194 if (crypto_shash_init(shash)) 195 return -EFAULT; 196 197 if (crypto_shash_update(shash, ipad, block_size)) 198 return -EFAULT; 199 200 hash_state_out = (__be32 *)hash->sha.state1; 201 hash512_state_out = (__be64 *)hash_state_out; 202 203 switch (ctx->qat_hash_alg) { 204 case ICP_QAT_HW_AUTH_ALGO_SHA1: 205 if (crypto_shash_export(shash, &sha1)) 206 return -EFAULT; 207 for (i = 0; i < digest_size >> 2; i++, hash_state_out++) 208 *hash_state_out = cpu_to_be32(*(sha1.state + i)); 209 break; 210 case ICP_QAT_HW_AUTH_ALGO_SHA256: 211 if (crypto_shash_export(shash, &sha256)) 212 return -EFAULT; 213 for (i = 0; i < digest_size >> 2; i++, hash_state_out++) 214 *hash_state_out = cpu_to_be32(*(sha256.state + i)); 215 break; 216 case ICP_QAT_HW_AUTH_ALGO_SHA512: 217 if (crypto_shash_export(shash, &sha512)) 218 return -EFAULT; 219 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) 220 *hash512_state_out = cpu_to_be64(*(sha512.state + i)); 221 break; 222 default: 223 return -EFAULT; 224 } 225 226 if (crypto_shash_init(shash)) 227 return -EFAULT; 228 229 if (crypto_shash_update(shash, opad, block_size)) 230 return -EFAULT; 231 232 offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8); 233 hash_state_out = (__be32 *)(hash->sha.state1 + offset); 234 hash512_state_out = (__be64 *)hash_state_out; 235 236 switch (ctx->qat_hash_alg) { 237 case ICP_QAT_HW_AUTH_ALGO_SHA1: 238 if (crypto_shash_export(shash, &sha1)) 239 return -EFAULT; 240 for (i = 0; i < digest_size >> 2; i++, hash_state_out++) 241 *hash_state_out = cpu_to_be32(*(sha1.state + i)); 242 break; 243 case ICP_QAT_HW_AUTH_ALGO_SHA256: 244 if (crypto_shash_export(shash, &sha256)) 245 return -EFAULT; 246 for (i = 0; i < digest_size >> 2; i++, hash_state_out++) 247 *hash_state_out = cpu_to_be32(*(sha256.state + i)); 248 break; 249 case ICP_QAT_HW_AUTH_ALGO_SHA512: 250 if (crypto_shash_export(shash, &sha512)) 251 return -EFAULT; 252 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) 253 *hash512_state_out = cpu_to_be64(*(sha512.state + i)); 254 break; 255 default: 256 return -EFAULT; 257 } 258 memzero_explicit(ipad, block_size); 259 memzero_explicit(opad, block_size); 260 return 0; 261} 262 263static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) 264{ 265 header->hdr_flags = 266 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); 267 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA; 268 header->comn_req_flags = 269 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, 270 QAT_COMN_PTR_TYPE_SGL); 271 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, 272 ICP_QAT_FW_LA_PARTIAL_NONE); 273 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, 274 ICP_QAT_FW_CIPH_IV_16BYTE_DATA); 275 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, 276 ICP_QAT_FW_LA_NO_PROTO); 277 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, 278 ICP_QAT_FW_LA_NO_UPDATE_STATE); 279} 280 281static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, 282 int alg, 283 struct crypto_authenc_keys *keys) 284{ 285 struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); 286 unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; 287 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd; 288 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher; 289 struct icp_qat_hw_auth_algo_blk *hash = 290 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx + 291 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen); 292 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req; 293 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; 294 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; 295 void *ptr = &req_tmpl->cd_ctrl; 296 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; 297 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; 298 299 /* CD setup */ 300 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); 301 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); 302 hash->sha.inner_setup.auth_config.config = 303 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, 304 ctx->qat_hash_alg, digestsize); 305 hash->sha.inner_setup.auth_counter.counter = 306 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); 307 308 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) 309 return -EFAULT; 310 311 /* Request setup */ 312 qat_alg_init_common_hdr(header); 313 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; 314 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, 315 ICP_QAT_FW_LA_DIGEST_IN_BUFFER); 316 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, 317 ICP_QAT_FW_LA_RET_AUTH_RES); 318 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, 319 ICP_QAT_FW_LA_NO_CMP_AUTH_RES); 320 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; 321 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; 322 323 /* Cipher CD config setup */ 324 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; 325 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; 326 cipher_cd_ctrl->cipher_cfg_offset = 0; 327 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); 328 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); 329 /* Auth CD config setup */ 330 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3; 331 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; 332 hash_cd_ctrl->inner_res_sz = digestsize; 333 hash_cd_ctrl->final_sz = digestsize; 334 335 switch (ctx->qat_hash_alg) { 336 case ICP_QAT_HW_AUTH_ALGO_SHA1: 337 hash_cd_ctrl->inner_state1_sz = 338 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); 339 hash_cd_ctrl->inner_state2_sz = 340 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); 341 break; 342 case ICP_QAT_HW_AUTH_ALGO_SHA256: 343 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; 344 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; 345 break; 346 case ICP_QAT_HW_AUTH_ALGO_SHA512: 347 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; 348 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; 349 break; 350 default: 351 break; 352 } 353 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + 354 ((sizeof(struct icp_qat_hw_auth_setup) + 355 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); 356 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); 357 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); 358 return 0; 359} 360 361static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx, 362 int alg, 363 struct crypto_authenc_keys *keys) 364{ 365 struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); 366 unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; 367 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd; 368 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash; 369 struct icp_qat_hw_cipher_algo_blk *cipher = 370 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + 371 sizeof(struct icp_qat_hw_auth_setup) + 372 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2); 373 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req; 374 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; 375 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; 376 void *ptr = &req_tmpl->cd_ctrl; 377 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; 378 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; 379 struct icp_qat_fw_la_auth_req_params *auth_param = 380 (struct icp_qat_fw_la_auth_req_params *) 381 ((char *)&req_tmpl->serv_specif_rqpars + 382 sizeof(struct icp_qat_fw_la_cipher_req_params)); 383 384 /* CD setup */ 385 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); 386 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); 387 hash->sha.inner_setup.auth_config.config = 388 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, 389 ctx->qat_hash_alg, 390 digestsize); 391 hash->sha.inner_setup.auth_counter.counter = 392 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); 393 394 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen)) 395 return -EFAULT; 396 397 /* Request setup */ 398 qat_alg_init_common_hdr(header); 399 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; 400 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, 401 ICP_QAT_FW_LA_DIGEST_IN_BUFFER); 402 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, 403 ICP_QAT_FW_LA_NO_RET_AUTH_RES); 404 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, 405 ICP_QAT_FW_LA_CMP_AUTH_RES); 406 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; 407 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; 408 409 /* Cipher CD config setup */ 410 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; 411 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; 412 cipher_cd_ctrl->cipher_cfg_offset = 413 (sizeof(struct icp_qat_hw_auth_setup) + 414 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3; 415 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); 416 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); 417 418 /* Auth CD config setup */ 419 hash_cd_ctrl->hash_cfg_offset = 0; 420 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; 421 hash_cd_ctrl->inner_res_sz = digestsize; 422 hash_cd_ctrl->final_sz = digestsize; 423 424 switch (ctx->qat_hash_alg) { 425 case ICP_QAT_HW_AUTH_ALGO_SHA1: 426 hash_cd_ctrl->inner_state1_sz = 427 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); 428 hash_cd_ctrl->inner_state2_sz = 429 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); 430 break; 431 case ICP_QAT_HW_AUTH_ALGO_SHA256: 432 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; 433 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; 434 break; 435 case ICP_QAT_HW_AUTH_ALGO_SHA512: 436 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; 437 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; 438 break; 439 default: 440 break; 441 } 442 443 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + 444 ((sizeof(struct icp_qat_hw_auth_setup) + 445 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); 446 auth_param->auth_res_sz = digestsize; 447 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); 448 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); 449 return 0; 450} 451 452static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx, 453 struct icp_qat_fw_la_bulk_req *req, 454 struct icp_qat_hw_cipher_algo_blk *cd, 455 const uint8_t *key, unsigned int keylen) 456{ 457 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; 458 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; 459 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; 460 461 memcpy(cd->aes.key, key, keylen); 462 qat_alg_init_common_hdr(header); 463 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; 464 cd_pars->u.s.content_desc_params_sz = 465 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; 466 /* Cipher CD config setup */ 467 cd_ctrl->cipher_key_sz = keylen >> 3; 468 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; 469 cd_ctrl->cipher_cfg_offset = 0; 470 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); 471 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); 472} 473 474static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx, 475 int alg, const uint8_t *key, 476 unsigned int keylen) 477{ 478 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd; 479 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req; 480 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; 481 482 qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen); 483 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; 484 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); 485} 486 487static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx, 488 int alg, const uint8_t *key, 489 unsigned int keylen) 490{ 491 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; 492 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; 493 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; 494 495 qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen); 496 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; 497 dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); 498} 499 500static int qat_alg_validate_key(int key_len, int *alg) 501{ 502 switch (key_len) { 503 case AES_KEYSIZE_128: 504 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; 505 break; 506 case AES_KEYSIZE_192: 507 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; 508 break; 509 case AES_KEYSIZE_256: 510 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; 511 break; 512 default: 513 return -EINVAL; 514 } 515 return 0; 516} 517 518static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx, 519 const uint8_t *key, unsigned int keylen) 520{ 521 struct crypto_authenc_keys keys; 522 int alg; 523 524 if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) 525 return -EFAULT; 526 527 if (crypto_authenc_extractkeys(&keys, key, keylen)) 528 goto bad_key; 529 530 if (qat_alg_validate_key(keys.enckeylen, &alg)) 531 goto bad_key; 532 533 if (qat_alg_aead_init_enc_session(ctx, alg, &keys)) 534 goto error; 535 536 if (qat_alg_aead_init_dec_session(ctx, alg, &keys)) 537 goto error; 538 539 return 0; 540bad_key: 541 crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 542 return -EINVAL; 543error: 544 return -EFAULT; 545} 546 547static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx, 548 const uint8_t *key, 549 unsigned int keylen) 550{ 551 int alg; 552 553 if (qat_alg_validate_key(keylen, &alg)) 554 goto bad_key; 555 556 qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen); 557 qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen); 558 return 0; 559bad_key: 560 crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 561 return -EINVAL; 562} 563 564static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, 565 unsigned int keylen) 566{ 567 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); 568 struct device *dev; 569 570 spin_lock(&ctx->lock); 571 if (ctx->enc_cd) { 572 /* rekeying */ 573 dev = &GET_DEV(ctx->inst->accel_dev); 574 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); 575 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); 576 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); 577 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); 578 } else { 579 /* new key */ 580 int node = get_current_node(); 581 struct qat_crypto_instance *inst = 582 qat_crypto_get_instance_node(node); 583 if (!inst) { 584 spin_unlock(&ctx->lock); 585 return -EINVAL; 586 } 587 588 dev = &GET_DEV(inst->accel_dev); 589 ctx->inst = inst; 590 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd), 591 &ctx->enc_cd_paddr, 592 GFP_ATOMIC); 593 if (!ctx->enc_cd) { 594 spin_unlock(&ctx->lock); 595 return -ENOMEM; 596 } 597 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), 598 &ctx->dec_cd_paddr, 599 GFP_ATOMIC); 600 if (!ctx->dec_cd) { 601 spin_unlock(&ctx->lock); 602 goto out_free_enc; 603 } 604 } 605 spin_unlock(&ctx->lock); 606 if (qat_alg_aead_init_sessions(ctx, key, keylen)) 607 goto out_free_all; 608 609 return 0; 610 611out_free_all: 612 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); 613 dma_free_coherent(dev, sizeof(struct qat_alg_cd), 614 ctx->dec_cd, ctx->dec_cd_paddr); 615 ctx->dec_cd = NULL; 616out_free_enc: 617 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); 618 dma_free_coherent(dev, sizeof(struct qat_alg_cd), 619 ctx->enc_cd, ctx->enc_cd_paddr); 620 ctx->enc_cd = NULL; 621 return -ENOMEM; 622} 623 624static void qat_alg_free_bufl(struct qat_crypto_instance *inst, 625 struct qat_crypto_request *qat_req) 626{ 627 struct device *dev = &GET_DEV(inst->accel_dev); 628 struct qat_alg_buf_list *bl = qat_req->buf.bl; 629 struct qat_alg_buf_list *blout = qat_req->buf.blout; 630 dma_addr_t blp = qat_req->buf.blp; 631 dma_addr_t blpout = qat_req->buf.bloutp; 632 size_t sz = qat_req->buf.sz; 633 size_t sz_out = qat_req->buf.sz_out; 634 int i; 635 636 for (i = 0; i < bl->num_bufs; i++) 637 dma_unmap_single(dev, bl->bufers[i].addr, 638 bl->bufers[i].len, DMA_BIDIRECTIONAL); 639 640 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); 641 kfree(bl); 642 if (blp != blpout) { 643 /* If out of place operation dma unmap only data */ 644 int bufless = blout->num_bufs - blout->num_mapped_bufs; 645 646 for (i = bufless; i < blout->num_bufs; i++) { 647 dma_unmap_single(dev, blout->bufers[i].addr, 648 blout->bufers[i].len, 649 DMA_BIDIRECTIONAL); 650 } 651 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); 652 kfree(blout); 653 } 654} 655 656static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, 657 struct scatterlist *assoc, 658 struct scatterlist *sgl, 659 struct scatterlist *sglout, uint8_t *iv, 660 uint8_t ivlen, 661 struct qat_crypto_request *qat_req) 662{ 663 struct device *dev = &GET_DEV(inst->accel_dev); 664 int i, bufs = 0, sg_nctr = 0; 665 int n = sg_nents(sgl), assoc_n = sg_nents(assoc); 666 struct qat_alg_buf_list *bufl; 667 struct qat_alg_buf_list *buflout = NULL; 668 dma_addr_t blp; 669 dma_addr_t bloutp = 0; 670 struct scatterlist *sg; 671 size_t sz_out, sz = sizeof(struct qat_alg_buf_list) + 672 ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); 673 674 if (unlikely(!n)) 675 return -EINVAL; 676 677 bufl = kzalloc_node(sz, GFP_ATOMIC, 678 dev_to_node(&GET_DEV(inst->accel_dev))); 679 if (unlikely(!bufl)) 680 return -ENOMEM; 681 682 blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE); 683 if (unlikely(dma_mapping_error(dev, blp))) 684 goto err; 685 686 for_each_sg(assoc, sg, assoc_n, i) { 687 if (!sg->length) 688 continue; 689 bufl->bufers[bufs].addr = dma_map_single(dev, 690 sg_virt(sg), 691 sg->length, 692 DMA_BIDIRECTIONAL); 693 bufl->bufers[bufs].len = sg->length; 694 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) 695 goto err; 696 bufs++; 697 } 698 if (ivlen) { 699 bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, 700 DMA_BIDIRECTIONAL); 701 bufl->bufers[bufs].len = ivlen; 702 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) 703 goto err; 704 bufs++; 705 } 706 707 for_each_sg(sgl, sg, n, i) { 708 int y = sg_nctr + bufs; 709 710 if (!sg->length) 711 continue; 712 713 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg), 714 sg->length, 715 DMA_BIDIRECTIONAL); 716 bufl->bufers[y].len = sg->length; 717 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr))) 718 goto err; 719 sg_nctr++; 720 } 721 bufl->num_bufs = sg_nctr + bufs; 722 qat_req->buf.bl = bufl; 723 qat_req->buf.blp = blp; 724 qat_req->buf.sz = sz; 725 /* Handle out of place operation */ 726 if (sgl != sglout) { 727 struct qat_alg_buf *bufers; 728 729 n = sg_nents(sglout); 730 sz_out = sizeof(struct qat_alg_buf_list) + 731 ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); 732 sg_nctr = 0; 733 buflout = kzalloc_node(sz_out, GFP_ATOMIC, 734 dev_to_node(&GET_DEV(inst->accel_dev))); 735 if (unlikely(!buflout)) 736 goto err; 737 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE); 738 if (unlikely(dma_mapping_error(dev, bloutp))) 739 goto err; 740 bufers = buflout->bufers; 741 /* For out of place operation dma map only data and 742 * reuse assoc mapping and iv */ 743 for (i = 0; i < bufs; i++) { 744 bufers[i].len = bufl->bufers[i].len; 745 bufers[i].addr = bufl->bufers[i].addr; 746 } 747 for_each_sg(sglout, sg, n, i) { 748 int y = sg_nctr + bufs; 749 750 if (!sg->length) 751 continue; 752 753 bufers[y].addr = dma_map_single(dev, sg_virt(sg), 754 sg->length, 755 DMA_BIDIRECTIONAL); 756 if (unlikely(dma_mapping_error(dev, bufers[y].addr))) 757 goto err; 758 bufers[y].len = sg->length; 759 sg_nctr++; 760 } 761 buflout->num_bufs = sg_nctr + bufs; 762 buflout->num_mapped_bufs = sg_nctr; 763 qat_req->buf.blout = buflout; 764 qat_req->buf.bloutp = bloutp; 765 qat_req->buf.sz_out = sz_out; 766 } else { 767 /* Otherwise set the src and dst to the same address */ 768 qat_req->buf.bloutp = qat_req->buf.blp; 769 qat_req->buf.sz_out = 0; 770 } 771 return 0; 772err: 773 dev_err(dev, "Failed to map buf for dma\n"); 774 sg_nctr = 0; 775 for (i = 0; i < n + bufs; i++) 776 if (!dma_mapping_error(dev, bufl->bufers[i].addr)) 777 dma_unmap_single(dev, bufl->bufers[i].addr, 778 bufl->bufers[i].len, 779 DMA_BIDIRECTIONAL); 780 781 if (!dma_mapping_error(dev, blp)) 782 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); 783 kfree(bufl); 784 if (sgl != sglout && buflout) { 785 n = sg_nents(sglout); 786 for (i = bufs; i < n + bufs; i++) 787 if (!dma_mapping_error(dev, buflout->bufers[i].addr)) 788 dma_unmap_single(dev, buflout->bufers[i].addr, 789 buflout->bufers[i].len, 790 DMA_BIDIRECTIONAL); 791 if (!dma_mapping_error(dev, bloutp)) 792 dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE); 793 kfree(buflout); 794 } 795 return -ENOMEM; 796} 797 798static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, 799 struct qat_crypto_request *qat_req) 800{ 801 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx; 802 struct qat_crypto_instance *inst = ctx->inst; 803 struct aead_request *areq = qat_req->aead_req; 804 uint8_t stat_filed = qat_resp->comn_resp.comn_status; 805 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); 806 807 qat_alg_free_bufl(inst, qat_req); 808 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) 809 res = -EBADMSG; 810 areq->base.complete(&areq->base, res); 811} 812 813static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, 814 struct qat_crypto_request *qat_req) 815{ 816 struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx; 817 struct qat_crypto_instance *inst = ctx->inst; 818 struct ablkcipher_request *areq = qat_req->ablkcipher_req; 819 uint8_t stat_filed = qat_resp->comn_resp.comn_status; 820 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); 821 822 qat_alg_free_bufl(inst, qat_req); 823 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) 824 res = -EINVAL; 825 areq->base.complete(&areq->base, res); 826} 827 828void qat_alg_callback(void *resp) 829{ 830 struct icp_qat_fw_la_resp *qat_resp = resp; 831 struct qat_crypto_request *qat_req = 832 (void *)(__force long)qat_resp->opaque_data; 833 834 qat_req->cb(qat_resp, qat_req); 835} 836 837static int qat_alg_aead_dec(struct aead_request *areq) 838{ 839 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); 840 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); 841 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); 842 struct qat_crypto_request *qat_req = aead_request_ctx(areq); 843 struct icp_qat_fw_la_cipher_req_params *cipher_param; 844 struct icp_qat_fw_la_auth_req_params *auth_param; 845 struct icp_qat_fw_la_bulk_req *msg; 846 int digst_size = crypto_aead_crt(aead_tfm)->authsize; 847 int ret, ctr = 0; 848 849 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst, 850 areq->iv, AES_BLOCK_SIZE, qat_req); 851 if (unlikely(ret)) 852 return ret; 853 854 msg = &qat_req->req; 855 *msg = ctx->dec_fw_req; 856 qat_req->aead_ctx = ctx; 857 qat_req->aead_req = areq; 858 qat_req->cb = qat_aead_alg_callback; 859 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; 860 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; 861 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; 862 cipher_param = (void *)&qat_req->req.serv_specif_rqpars; 863 cipher_param->cipher_length = areq->cryptlen - digst_size; 864 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; 865 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); 866 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); 867 auth_param->auth_off = 0; 868 auth_param->auth_len = areq->assoclen + 869 cipher_param->cipher_length + AES_BLOCK_SIZE; 870 do { 871 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); 872 } while (ret == -EAGAIN && ctr++ < 10); 873 874 if (ret == -EAGAIN) { 875 qat_alg_free_bufl(ctx->inst, qat_req); 876 return -EBUSY; 877 } 878 return -EINPROGRESS; 879} 880 881static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, 882 int enc_iv) 883{ 884 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); 885 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); 886 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); 887 struct qat_crypto_request *qat_req = aead_request_ctx(areq); 888 struct icp_qat_fw_la_cipher_req_params *cipher_param; 889 struct icp_qat_fw_la_auth_req_params *auth_param; 890 struct icp_qat_fw_la_bulk_req *msg; 891 int ret, ctr = 0; 892 893 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst, 894 iv, AES_BLOCK_SIZE, qat_req); 895 if (unlikely(ret)) 896 return ret; 897 898 msg = &qat_req->req; 899 *msg = ctx->enc_fw_req; 900 qat_req->aead_ctx = ctx; 901 qat_req->aead_req = areq; 902 qat_req->cb = qat_aead_alg_callback; 903 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; 904 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; 905 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; 906 cipher_param = (void *)&qat_req->req.serv_specif_rqpars; 907 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); 908 909 if (enc_iv) { 910 cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE; 911 cipher_param->cipher_offset = areq->assoclen; 912 } else { 913 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); 914 cipher_param->cipher_length = areq->cryptlen; 915 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; 916 } 917 auth_param->auth_off = 0; 918 auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE; 919 920 do { 921 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); 922 } while (ret == -EAGAIN && ctr++ < 10); 923 924 if (ret == -EAGAIN) { 925 qat_alg_free_bufl(ctx->inst, qat_req); 926 return -EBUSY; 927 } 928 return -EINPROGRESS; 929} 930 931static int qat_alg_aead_enc(struct aead_request *areq) 932{ 933 return qat_alg_aead_enc_internal(areq, areq->iv, 0); 934} 935 936static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req) 937{ 938 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq); 939 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); 940 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); 941 __be64 seq; 942 943 memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE); 944 seq = cpu_to_be64(req->seq); 945 memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t), 946 &seq, sizeof(uint64_t)); 947 return qat_alg_aead_enc_internal(&req->areq, req->giv, 1); 948} 949 950static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, 951 const uint8_t *key, 952 unsigned int keylen) 953{ 954 struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); 955 struct device *dev; 956 957 spin_lock(&ctx->lock); 958 if (ctx->enc_cd) { 959 /* rekeying */ 960 dev = &GET_DEV(ctx->inst->accel_dev); 961 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); 962 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); 963 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); 964 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); 965 } else { 966 /* new key */ 967 int node = get_current_node(); 968 struct qat_crypto_instance *inst = 969 qat_crypto_get_instance_node(node); 970 if (!inst) { 971 spin_unlock(&ctx->lock); 972 return -EINVAL; 973 } 974 975 dev = &GET_DEV(inst->accel_dev); 976 ctx->inst = inst; 977 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd), 978 &ctx->enc_cd_paddr, 979 GFP_ATOMIC); 980 if (!ctx->enc_cd) { 981 spin_unlock(&ctx->lock); 982 return -ENOMEM; 983 } 984 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), 985 &ctx->dec_cd_paddr, 986 GFP_ATOMIC); 987 if (!ctx->dec_cd) { 988 spin_unlock(&ctx->lock); 989 goto out_free_enc; 990 } 991 } 992 spin_unlock(&ctx->lock); 993 if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen)) 994 goto out_free_all; 995 996 return 0; 997 998out_free_all: 999 memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd)); 1000 dma_free_coherent(dev, sizeof(*ctx->enc_cd), 1001 ctx->dec_cd, ctx->dec_cd_paddr); 1002 ctx->dec_cd = NULL; 1003out_free_enc: 1004 memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd)); 1005 dma_free_coherent(dev, sizeof(*ctx->dec_cd), 1006 ctx->enc_cd, ctx->enc_cd_paddr); 1007 ctx->enc_cd = NULL; 1008 return -ENOMEM; 1009} 1010 1011static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) 1012{ 1013 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); 1014 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm); 1015 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); 1016 struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); 1017 struct icp_qat_fw_la_cipher_req_params *cipher_param; 1018 struct icp_qat_fw_la_bulk_req *msg; 1019 int ret, ctr = 0; 1020 1021 ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, 1022 NULL, 0, qat_req); 1023 if (unlikely(ret)) 1024 return ret; 1025 1026 msg = &qat_req->req; 1027 *msg = ctx->enc_fw_req; 1028 qat_req->ablkcipher_ctx = ctx; 1029 qat_req->ablkcipher_req = req; 1030 qat_req->cb = qat_ablkcipher_alg_callback; 1031 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; 1032 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; 1033 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; 1034 cipher_param = (void *)&qat_req->req.serv_specif_rqpars; 1035 cipher_param->cipher_length = req->nbytes; 1036 cipher_param->cipher_offset = 0; 1037 memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); 1038 do { 1039 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); 1040 } while (ret == -EAGAIN && ctr++ < 10); 1041 1042 if (ret == -EAGAIN) { 1043 qat_alg_free_bufl(ctx->inst, qat_req); 1044 return -EBUSY; 1045 } 1046 return -EINPROGRESS; 1047} 1048 1049static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) 1050{ 1051 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); 1052 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm); 1053 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); 1054 struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); 1055 struct icp_qat_fw_la_cipher_req_params *cipher_param; 1056 struct icp_qat_fw_la_bulk_req *msg; 1057 int ret, ctr = 0; 1058 1059 ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, 1060 NULL, 0, qat_req); 1061 if (unlikely(ret)) 1062 return ret; 1063 1064 msg = &qat_req->req; 1065 *msg = ctx->dec_fw_req; 1066 qat_req->ablkcipher_ctx = ctx; 1067 qat_req->ablkcipher_req = req; 1068 qat_req->cb = qat_ablkcipher_alg_callback; 1069 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; 1070 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; 1071 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; 1072 cipher_param = (void *)&qat_req->req.serv_specif_rqpars; 1073 cipher_param->cipher_length = req->nbytes; 1074 cipher_param->cipher_offset = 0; 1075 memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); 1076 do { 1077 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); 1078 } while (ret == -EAGAIN && ctr++ < 10); 1079 1080 if (ret == -EAGAIN) { 1081 qat_alg_free_bufl(ctx->inst, qat_req); 1082 return -EBUSY; 1083 } 1084 return -EINPROGRESS; 1085} 1086 1087static int qat_alg_aead_init(struct crypto_tfm *tfm, 1088 enum icp_qat_hw_auth_algo hash, 1089 const char *hash_name) 1090{ 1091 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); 1092 1093 ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 1094 if (IS_ERR(ctx->hash_tfm)) 1095 return -EFAULT; 1096 spin_lock_init(&ctx->lock); 1097 ctx->qat_hash_alg = hash; 1098 tfm->crt_aead.reqsize = sizeof(struct aead_request) + 1099 sizeof(struct qat_crypto_request); 1100 ctx->tfm = tfm; 1101 return 0; 1102} 1103 1104static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm) 1105{ 1106 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); 1107} 1108 1109static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm) 1110{ 1111 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); 1112} 1113 1114static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm) 1115{ 1116 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); 1117} 1118 1119static void qat_alg_aead_exit(struct crypto_tfm *tfm) 1120{ 1121 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); 1122 struct qat_crypto_instance *inst = ctx->inst; 1123 struct device *dev; 1124 1125 if (!IS_ERR(ctx->hash_tfm)) 1126 crypto_free_shash(ctx->hash_tfm); 1127 1128 if (!inst) 1129 return; 1130 1131 dev = &GET_DEV(inst->accel_dev); 1132 if (ctx->enc_cd) { 1133 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); 1134 dma_free_coherent(dev, sizeof(struct qat_alg_cd), 1135 ctx->enc_cd, ctx->enc_cd_paddr); 1136 } 1137 if (ctx->dec_cd) { 1138 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); 1139 dma_free_coherent(dev, sizeof(struct qat_alg_cd), 1140 ctx->dec_cd, ctx->dec_cd_paddr); 1141 } 1142 qat_crypto_put_instance(inst); 1143} 1144 1145static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm) 1146{ 1147 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); 1148 1149 spin_lock_init(&ctx->lock); 1150 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + 1151 sizeof(struct qat_crypto_request); 1152 ctx->tfm = tfm; 1153 return 0; 1154} 1155 1156static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm) 1157{ 1158 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); 1159 struct qat_crypto_instance *inst = ctx->inst; 1160 struct device *dev; 1161 1162 if (!inst) 1163 return; 1164 1165 dev = &GET_DEV(inst->accel_dev); 1166 if (ctx->enc_cd) { 1167 memset(ctx->enc_cd, 0, 1168 sizeof(struct icp_qat_hw_cipher_algo_blk)); 1169 dma_free_coherent(dev, 1170 sizeof(struct icp_qat_hw_cipher_algo_blk), 1171 ctx->enc_cd, ctx->enc_cd_paddr); 1172 } 1173 if (ctx->dec_cd) { 1174 memset(ctx->dec_cd, 0, 1175 sizeof(struct icp_qat_hw_cipher_algo_blk)); 1176 dma_free_coherent(dev, 1177 sizeof(struct icp_qat_hw_cipher_algo_blk), 1178 ctx->dec_cd, ctx->dec_cd_paddr); 1179 } 1180 qat_crypto_put_instance(inst); 1181} 1182 1183static struct crypto_alg qat_algs[] = { { 1184 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1185 .cra_driver_name = "qat_aes_cbc_hmac_sha1", 1186 .cra_priority = 4001, 1187 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1188 .cra_blocksize = AES_BLOCK_SIZE, 1189 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), 1190 .cra_alignmask = 0, 1191 .cra_type = &crypto_aead_type, 1192 .cra_module = THIS_MODULE, 1193 .cra_init = qat_alg_aead_sha1_init, 1194 .cra_exit = qat_alg_aead_exit, 1195 .cra_u = { 1196 .aead = { 1197 .setkey = qat_alg_aead_setkey, 1198 .decrypt = qat_alg_aead_dec, 1199 .encrypt = qat_alg_aead_enc, 1200 .givencrypt = qat_alg_aead_genivenc, 1201 .ivsize = AES_BLOCK_SIZE, 1202 .maxauthsize = SHA1_DIGEST_SIZE, 1203 }, 1204 }, 1205}, { 1206 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1207 .cra_driver_name = "qat_aes_cbc_hmac_sha256", 1208 .cra_priority = 4001, 1209 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1210 .cra_blocksize = AES_BLOCK_SIZE, 1211 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), 1212 .cra_alignmask = 0, 1213 .cra_type = &crypto_aead_type, 1214 .cra_module = THIS_MODULE, 1215 .cra_init = qat_alg_aead_sha256_init, 1216 .cra_exit = qat_alg_aead_exit, 1217 .cra_u = { 1218 .aead = { 1219 .setkey = qat_alg_aead_setkey, 1220 .decrypt = qat_alg_aead_dec, 1221 .encrypt = qat_alg_aead_enc, 1222 .givencrypt = qat_alg_aead_genivenc, 1223 .ivsize = AES_BLOCK_SIZE, 1224 .maxauthsize = SHA256_DIGEST_SIZE, 1225 }, 1226 }, 1227}, { 1228 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1229 .cra_driver_name = "qat_aes_cbc_hmac_sha512", 1230 .cra_priority = 4001, 1231 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1232 .cra_blocksize = AES_BLOCK_SIZE, 1233 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), 1234 .cra_alignmask = 0, 1235 .cra_type = &crypto_aead_type, 1236 .cra_module = THIS_MODULE, 1237 .cra_init = qat_alg_aead_sha512_init, 1238 .cra_exit = qat_alg_aead_exit, 1239 .cra_u = { 1240 .aead = { 1241 .setkey = qat_alg_aead_setkey, 1242 .decrypt = qat_alg_aead_dec, 1243 .encrypt = qat_alg_aead_enc, 1244 .givencrypt = qat_alg_aead_genivenc, 1245 .ivsize = AES_BLOCK_SIZE, 1246 .maxauthsize = SHA512_DIGEST_SIZE, 1247 }, 1248 }, 1249}, { 1250 .cra_name = "cbc(aes)", 1251 .cra_driver_name = "qat_aes_cbc", 1252 .cra_priority = 4001, 1253 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1254 .cra_blocksize = AES_BLOCK_SIZE, 1255 .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx), 1256 .cra_alignmask = 0, 1257 .cra_type = &crypto_ablkcipher_type, 1258 .cra_module = THIS_MODULE, 1259 .cra_init = qat_alg_ablkcipher_init, 1260 .cra_exit = qat_alg_ablkcipher_exit, 1261 .cra_u = { 1262 .ablkcipher = { 1263 .setkey = qat_alg_ablkcipher_setkey, 1264 .decrypt = qat_alg_ablkcipher_decrypt, 1265 .encrypt = qat_alg_ablkcipher_encrypt, 1266 .min_keysize = AES_MIN_KEY_SIZE, 1267 .max_keysize = AES_MAX_KEY_SIZE, 1268 .ivsize = AES_BLOCK_SIZE, 1269 }, 1270 }, 1271} }; 1272 1273int qat_algs_register(void) 1274{ 1275 int ret = 0; 1276 1277 mutex_lock(&algs_lock); 1278 if (++active_devs == 1) { 1279 int i; 1280 1281 for (i = 0; i < ARRAY_SIZE(qat_algs); i++) 1282 qat_algs[i].cra_flags = 1283 (qat_algs[i].cra_type == &crypto_aead_type) ? 1284 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC : 1285 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; 1286 1287 ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); 1288 } 1289 mutex_unlock(&algs_lock); 1290 return ret; 1291} 1292 1293int qat_algs_unregister(void) 1294{ 1295 int ret = 0; 1296 1297 mutex_lock(&algs_lock); 1298 if (--active_devs == 0) 1299 ret = crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); 1300 mutex_unlock(&algs_lock); 1301 return ret; 1302} 1303 1304int qat_algs_init(void) 1305{ 1306 crypto_get_default_rng(); 1307 return 0; 1308} 1309 1310void qat_algs_exit(void) 1311{ 1312 crypto_put_default_rng(); 1313} 1314