root/drivers/crypto/inside-secure/safexcel_cipher.c

DEFINITIONS

1. safexcel_cipher_token
2. safexcel_skcipher_token
3. safexcel_aead_token
4. safexcel_skcipher_aes_setkey
5. safexcel_aead_setkey
6. safexcel_context_control
7. safexcel_handle_req_result
8. safexcel_send_req
9. safexcel_handle_inv_result
10. safexcel_skcipher_handle_result
11. safexcel_aead_handle_result
12. safexcel_cipher_send_inv
13. safexcel_skcipher_send
14. safexcel_aead_send
15. safexcel_cipher_exit_inv
16. safexcel_skcipher_exit_inv
17. safexcel_aead_exit_inv
18. safexcel_queue_req
19. safexcel_encrypt
20. safexcel_decrypt
21. safexcel_skcipher_cra_init
22. safexcel_cipher_cra_exit
23. safexcel_skcipher_cra_exit
24. safexcel_aead_cra_exit
25. safexcel_skcipher_aes_ecb_cra_init
26. safexcel_skcipher_aes_cbc_cra_init
27. safexcel_skcipher_aes_cfb_cra_init
28. safexcel_skcipher_aes_ofb_cra_init
29. safexcel_skcipher_aesctr_setkey
30. safexcel_skcipher_aes_ctr_cra_init
31. safexcel_des_setkey
32. safexcel_skcipher_des_cbc_cra_init
33. safexcel_skcipher_des_ecb_cra_init
34. safexcel_des3_ede_setkey
35. safexcel_skcipher_des3_cbc_cra_init
36. safexcel_skcipher_des3_ecb_cra_init
37. safexcel_aead_encrypt
38. safexcel_aead_decrypt
39. safexcel_aead_cra_init
40. safexcel_aead_sha1_cra_init
41. safexcel_aead_sha256_cra_init
42. safexcel_aead_sha224_cra_init
43. safexcel_aead_sha512_cra_init
44. safexcel_aead_sha384_cra_init
45. safexcel_aead_sha1_des3_cra_init
46. safexcel_aead_sha1_ctr_cra_init
47. safexcel_aead_sha256_ctr_cra_init
48. safexcel_aead_sha224_ctr_cra_init
49. safexcel_aead_sha512_ctr_cra_init
50. safexcel_aead_sha384_ctr_cra_init
51. safexcel_skcipher_aesxts_setkey
52. safexcel_skcipher_aes_xts_cra_init
53. safexcel_encrypt_xts
54. safexcel_decrypt_xts
55. safexcel_aead_gcm_setkey
56. safexcel_aead_gcm_cra_init
57. safexcel_aead_gcm_cra_exit
58. safexcel_aead_gcm_setauthsize
59. safexcel_aead_ccm_setkey
60. safexcel_aead_ccm_cra_init
61. safexcel_aead_ccm_setauthsize
62. safexcel_ccm_encrypt
63. safexcel_ccm_decrypt

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <antoine.tenart@free-electrons.com>
   6  */
   7 
   8 #include <linux/device.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/dmapool.h>
  11 
  12 #include <crypto/aead.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/authenc.h>
  15 #include <crypto/ctr.h>
  16 #include <crypto/internal/des.h>
  17 #include <crypto/gcm.h>
  18 #include <crypto/ghash.h>
  19 #include <crypto/sha.h>
  20 #include <crypto/xts.h>
  21 #include <crypto/skcipher.h>
  22 #include <crypto/internal/aead.h>
  23 #include <crypto/internal/skcipher.h>
  24 
  25 #include "safexcel.h"
  26 
  27 enum safexcel_cipher_direction {
  28         SAFEXCEL_ENCRYPT,
  29         SAFEXCEL_DECRYPT,
  30 };
  31 
  32 enum safexcel_cipher_alg {
  33         SAFEXCEL_DES,
  34         SAFEXCEL_3DES,
  35         SAFEXCEL_AES,
  36 };
  37 
  38 struct safexcel_cipher_ctx {
  39         struct safexcel_context base;
  40         struct safexcel_crypto_priv *priv;
  41 
  42         u32 mode;
  43         enum safexcel_cipher_alg alg;
  44         bool aead;
  45         int  xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */
  46 
  47         __le32 key[16];
  48         u32 nonce;
  49         unsigned int key_len, xts;
  50 
  51         /* All the below is AEAD specific */
  52         u32 hash_alg;
  53         u32 state_sz;
  54         u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
  55         u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
  56 
  57         struct crypto_cipher *hkaes;
  58 };
  59 
  60 struct safexcel_cipher_req {
  61         enum safexcel_cipher_direction direction;
  62         /* Number of result descriptors associated to the request */
  63         unsigned int rdescs;
  64         bool needs_inv;
  65         int  nr_src, nr_dst;
  66 };
  67 
  68 static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
  69                                   struct safexcel_command_desc *cdesc)
  70 {
  71         u32 block_sz = 0;
  72 
  73         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
  74                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  75 
  76                 /* 32 bit nonce */
  77                 cdesc->control_data.token[0] = ctx->nonce;
  78                 /* 64 bit IV part */
  79                 memcpy(&cdesc->control_data.token[1], iv, 8);
  80                 /* 32 bit counter, start at 1 (big endian!) */
  81                 cdesc->control_data.token[3] = cpu_to_be32(1);
  82 
  83                 return;
  84         } else if (ctx->xcm == EIP197_XCM_MODE_GCM) {
  85                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  86 
  87                 /* 96 bit IV part */
  88                 memcpy(&cdesc->control_data.token[0], iv, 12);
  89                 /* 32 bit counter, start at 1 (big endian!) */
  90                 cdesc->control_data.token[3] = cpu_to_be32(1);
  91 
  92                 return;
  93         } else if (ctx->xcm == EIP197_XCM_MODE_CCM) {
  94                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  95 
  96                 /* Variable length IV part */
  97                 memcpy(&cdesc->control_data.token[0], iv, 15 - iv[0]);
  98                 /* Start variable length counter at 0 */
  99                 memset((u8 *)&cdesc->control_data.token[0] + 15 - iv[0],
 100                        0, iv[0] + 1);
 101 
 102                 return;
 103         }
 104 
 105         if (ctx->mode != CONTEXT_CONTROL_CRYPTO_MODE_ECB) {
 106                 switch (ctx->alg) {
 107                 case SAFEXCEL_DES:
 108                         block_sz = DES_BLOCK_SIZE;
 109                         cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
 110                         break;
 111                 case SAFEXCEL_3DES:
 112                         block_sz = DES3_EDE_BLOCK_SIZE;
 113                         cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
 114                         break;
 115                 case SAFEXCEL_AES:
 116                         block_sz = AES_BLOCK_SIZE;
 117                         cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
 118                         break;
 119                 }
 120                 memcpy(cdesc->control_data.token, iv, block_sz);
 121         }
 122 }
 123 
 124 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 125                                     struct safexcel_command_desc *cdesc,
 126                                     u32 length)
 127 {
 128         struct safexcel_token *token;
 129 
 130         safexcel_cipher_token(ctx, iv, cdesc);
 131 
 132         /* skip over worst case IV of 4 dwords, no need to be exact */
 133         token = (struct safexcel_token *)(cdesc->control_data.token + 4);
 134 
 135         token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 136         token[0].packet_length = length;
 137         token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET |
 138                         EIP197_TOKEN_STAT_LAST_HASH;
 139         token[0].instructions = EIP197_TOKEN_INS_LAST |
 140                                 EIP197_TOKEN_INS_TYPE_CRYPTO |
 141                                 EIP197_TOKEN_INS_TYPE_OUTPUT;
 142 }
 143 
 144 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 145                                 struct safexcel_command_desc *cdesc,
 146                                 enum safexcel_cipher_direction direction,
 147                                 u32 cryptlen, u32 assoclen, u32 digestsize)
 148 {
 149         struct safexcel_token *token;
 150 
 151         safexcel_cipher_token(ctx, iv, cdesc);
 152 
 153         if (direction == SAFEXCEL_ENCRYPT) {
 154                 /* align end of instruction sequence to end of token */
 155                 token = (struct safexcel_token *)(cdesc->control_data.token +
 156                          EIP197_MAX_TOKENS - 13);
 157 
 158                 token[12].opcode = EIP197_TOKEN_OPCODE_INSERT;
 159                 token[12].packet_length = digestsize;
 160                 token[12].stat = EIP197_TOKEN_STAT_LAST_HASH |
 161                                  EIP197_TOKEN_STAT_LAST_PACKET;
 162                 token[12].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 163                                          EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 164         } else {
 165                 cryptlen -= digestsize;
 166 
 167                 /* align end of instruction sequence to end of token */
 168                 token = (struct safexcel_token *)(cdesc->control_data.token +
 169                          EIP197_MAX_TOKENS - 14);
 170 
 171                 token[12].opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
 172                 token[12].packet_length = digestsize;
 173                 token[12].stat = EIP197_TOKEN_STAT_LAST_HASH |
 174                                  EIP197_TOKEN_STAT_LAST_PACKET;
 175                 token[12].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 176 
 177                 token[13].opcode = EIP197_TOKEN_OPCODE_VERIFY;
 178                 token[13].packet_length = digestsize |
 179                                           EIP197_TOKEN_HASH_RESULT_VERIFY;
 180                 token[13].stat = EIP197_TOKEN_STAT_LAST_HASH |
 181                                  EIP197_TOKEN_STAT_LAST_PACKET;
 182                 token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
 183         }
 184 
 185         token[6].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 186         token[6].packet_length = assoclen;
 187 
 188         if (likely(cryptlen)) {
 189                 token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH;
 190 
 191                 token[10].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 192                 token[10].packet_length = cryptlen;
 193                 token[10].stat = EIP197_TOKEN_STAT_LAST_HASH;
 194                 token[10].instructions = EIP197_TOKEN_INS_LAST |
 195                                          EIP197_TOKEN_INS_TYPE_CRYPTO |
 196                                          EIP197_TOKEN_INS_TYPE_HASH |
 197                                          EIP197_TOKEN_INS_TYPE_OUTPUT;
 198         } else if (ctx->xcm != EIP197_XCM_MODE_CCM) {
 199                 token[6].stat = EIP197_TOKEN_STAT_LAST_HASH;
 200                 token[6].instructions = EIP197_TOKEN_INS_LAST |
 201                                         EIP197_TOKEN_INS_TYPE_HASH;
 202         }
 203 
 204         if (!ctx->xcm)
 205                 return;
 206 
 207         token[8].opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES;
 208         token[8].packet_length = 0;
 209         token[8].instructions = AES_BLOCK_SIZE;
 210 
 211         token[9].opcode = EIP197_TOKEN_OPCODE_INSERT;
 212         token[9].packet_length = AES_BLOCK_SIZE;
 213         token[9].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 214                                 EIP197_TOKEN_INS_TYPE_CRYPTO;
 215 
 216         if (ctx->xcm == EIP197_XCM_MODE_GCM) {
 217                 token[6].instructions = EIP197_TOKEN_INS_LAST |
 218                                         EIP197_TOKEN_INS_TYPE_HASH;
 219         } else {
 220                 u8 *cbcmaciv = (u8 *)&token[1];
 221                 u32 *aadlen = (u32 *)&token[5];
 222 
 223                 /* Construct IV block B0 for the CBC-MAC */
 224                 token[0].opcode = EIP197_TOKEN_OPCODE_INSERT;
 225                 token[0].packet_length = AES_BLOCK_SIZE +
 226                                          ((assoclen > 0) << 1);
 227                 token[0].instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN |
 228                                         EIP197_TOKEN_INS_TYPE_HASH;
 229                 /* Variable length IV part */
 230                 memcpy(cbcmaciv, iv, 15 - iv[0]);
 231                 /* fixup flags byte */
 232                 cbcmaciv[0] |= ((assoclen > 0) << 6) | ((digestsize - 2) << 2);
 233                 /* Clear upper bytes of variable message length to 0 */
 234                 memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1);
 235                 /* insert lower 2 bytes of message length */
 236                 cbcmaciv[14] = cryptlen >> 8;
 237                 cbcmaciv[15] = cryptlen & 255;
 238 
 239                 if (assoclen) {
 240                         *aadlen = cpu_to_le32(cpu_to_be16(assoclen));
 241                         assoclen += 2;
 242                 }
 243 
 244                 token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH;
 245 
 246                 /* Align AAD data towards hash engine */
 247                 token[7].opcode = EIP197_TOKEN_OPCODE_INSERT;
 248                 assoclen &= 15;
 249                 token[7].packet_length = assoclen ? 16 - assoclen : 0;
 250 
 251                 if (likely(cryptlen)) {
 252                         token[7].instructions = EIP197_TOKEN_INS_TYPE_HASH;
 253 
 254                         /* Align crypto data towards hash engine */
 255                         token[10].stat = 0;
 256 
 257                         token[11].opcode = EIP197_TOKEN_OPCODE_INSERT;
 258                         cryptlen &= 15;
 259                         token[11].packet_length = cryptlen ? 16 - cryptlen : 0;
 260                         token[11].stat = EIP197_TOKEN_STAT_LAST_HASH;
 261                         token[11].instructions = EIP197_TOKEN_INS_TYPE_HASH;
 262                 } else {
 263                         token[7].stat = EIP197_TOKEN_STAT_LAST_HASH;
 264                         token[7].instructions = EIP197_TOKEN_INS_LAST |
 265                                                 EIP197_TOKEN_INS_TYPE_HASH;
 266                 }
 267         }
 268 }
 269 
 270 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
 271                                         const u8 *key, unsigned int len)
 272 {
 273         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 274         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 275         struct safexcel_crypto_priv *priv = ctx->priv;
 276         struct crypto_aes_ctx aes;
 277         int ret, i;
 278 
 279         ret = aes_expandkey(&aes, key, len);
 280         if (ret) {
 281                 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 282                 return ret;
 283         }
 284 
 285         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 286                 for (i = 0; i < len / sizeof(u32); i++) {
 287                         if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
 288                                 ctx->base.needs_inv = true;
 289                                 break;
 290                         }
 291                 }
 292         }
 293 
 294         for (i = 0; i < len / sizeof(u32); i++)
 295                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
 296 
 297         ctx->key_len = len;
 298 
 299         memzero_explicit(&aes, sizeof(aes));
 300         return 0;
 301 }
 302 
 303 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 304                                 unsigned int len)
 305 {
 306         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
 307         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 308         struct safexcel_ahash_export_state istate, ostate;
 309         struct safexcel_crypto_priv *priv = ctx->priv;
 310         struct crypto_authenc_keys keys;
 311         struct crypto_aes_ctx aes;
 312         int err = -EINVAL;
 313 
 314         if (crypto_authenc_extractkeys(&keys, key, len) != 0)
 315                 goto badkey;
 316 
 317         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
 318                 /* Minimum keysize is minimum AES key size + nonce size */
 319                 if (keys.enckeylen < (AES_MIN_KEY_SIZE +
 320                                       CTR_RFC3686_NONCE_SIZE))
 321                         goto badkey;
 322                 /* last 4 bytes of key are the nonce! */
 323                 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen -
 324                                       CTR_RFC3686_NONCE_SIZE);
 325                 /* exclude the nonce here */
 326                 keys.enckeylen -= CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
 327         }
 328 
 329         /* Encryption key */
 330         switch (ctx->alg) {
 331         case SAFEXCEL_3DES:
 332                 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen);
 333                 if (unlikely(err))
 334                         goto badkey_expflags;
 335                 break;
 336         case SAFEXCEL_AES:
 337                 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen);
 338                 if (unlikely(err))
 339                         goto badkey;
 340                 break;
 341         default:
 342                 dev_err(priv->dev, "aead: unsupported cipher algorithm\n");
 343                 goto badkey;
 344         }
 345 
 346         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
 347             memcmp(ctx->key, keys.enckey, keys.enckeylen))
 348                 ctx->base.needs_inv = true;
 349 
 350         /* Auth key */
 351         switch (ctx->hash_alg) {
 352         case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
 353                 if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
 354                                          keys.authkeylen, &istate, &ostate))
 355                         goto badkey;
 356                 break;
 357         case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
 358                 if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey,
 359                                          keys.authkeylen, &istate, &ostate))
 360                         goto badkey;
 361                 break;
 362         case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
 363                 if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey,
 364                                          keys.authkeylen, &istate, &ostate))
 365                         goto badkey;
 366                 break;
 367         case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
 368                 if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey,
 369                                          keys.authkeylen, &istate, &ostate))
 370                         goto badkey;
 371                 break;
 372         case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
 373                 if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey,
 374                                          keys.authkeylen, &istate, &ostate))
 375                         goto badkey;
 376                 break;
 377         default:
 378                 dev_err(priv->dev, "aead: unsupported hash algorithm\n");
 379                 goto badkey;
 380         }
 381 
 382         crypto_aead_set_flags(ctfm, crypto_aead_get_flags(ctfm) &
 383                                     CRYPTO_TFM_RES_MASK);
 384 
 385         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
 386             (memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
 387              memcmp(ctx->opad, ostate.state, ctx->state_sz)))
 388                 ctx->base.needs_inv = true;
 389 
 390         /* Now copy the keys into the context */
 391         memcpy(ctx->key, keys.enckey, keys.enckeylen);
 392         ctx->key_len = keys.enckeylen;
 393 
 394         memcpy(ctx->ipad, &istate.state, ctx->state_sz);
 395         memcpy(ctx->opad, &ostate.state, ctx->state_sz);
 396 
 397         memzero_explicit(&keys, sizeof(keys));
 398         return 0;
 399 
 400 badkey:
 401         crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 402 badkey_expflags:
 403         memzero_explicit(&keys, sizeof(keys));
 404         return err;
 405 }
 406 
 407 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
 408                                     struct crypto_async_request *async,
 409                                     struct safexcel_cipher_req *sreq,
 410                                     struct safexcel_command_desc *cdesc)
 411 {
 412         struct safexcel_crypto_priv *priv = ctx->priv;
 413         int ctrl_size = ctx->key_len / sizeof(u32);
 414 
 415         cdesc->control_data.control1 = ctx->mode;
 416 
 417         if (ctx->aead) {
 418                 /* Take in account the ipad+opad digests */
 419                 if (ctx->xcm) {
 420                         ctrl_size += ctx->state_sz / sizeof(u32);
 421                         cdesc->control_data.control0 =
 422                                 CONTEXT_CONTROL_KEY_EN |
 423                                 CONTEXT_CONTROL_DIGEST_XCM |
 424                                 ctx->hash_alg |
 425                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 426                 } else {
 427                         ctrl_size += ctx->state_sz / sizeof(u32) * 2;
 428                         cdesc->control_data.control0 =
 429                                 CONTEXT_CONTROL_KEY_EN |
 430                                 CONTEXT_CONTROL_DIGEST_HMAC |
 431                                 ctx->hash_alg |
 432                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 433                 }
 434                 if (sreq->direction == SAFEXCEL_ENCRYPT)
 435                         cdesc->control_data.control0 |=
 436                                 (ctx->xcm == EIP197_XCM_MODE_CCM) ?
 437                                         CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT :
 438                                         CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
 439 
 440                 else
 441                         cdesc->control_data.control0 |=
 442                                 (ctx->xcm == EIP197_XCM_MODE_CCM) ?
 443                                         CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN :
 444                                         CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
 445         } else {
 446                 if (sreq->direction == SAFEXCEL_ENCRYPT)
 447                         cdesc->control_data.control0 =
 448                                 CONTEXT_CONTROL_TYPE_CRYPTO_OUT |
 449                                 CONTEXT_CONTROL_KEY_EN |
 450                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 451                 else
 452                         cdesc->control_data.control0 =
 453                                 CONTEXT_CONTROL_TYPE_CRYPTO_IN |
 454                                 CONTEXT_CONTROL_KEY_EN |
 455                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 456         }
 457 
 458         if (ctx->alg == SAFEXCEL_DES) {
 459                 cdesc->control_data.control0 |=
 460                         CONTEXT_CONTROL_CRYPTO_ALG_DES;
 461         } else if (ctx->alg == SAFEXCEL_3DES) {
 462                 cdesc->control_data.control0 |=
 463                         CONTEXT_CONTROL_CRYPTO_ALG_3DES;
 464         } else if (ctx->alg == SAFEXCEL_AES) {
 465                 switch (ctx->key_len >> ctx->xts) {
 466                 case AES_KEYSIZE_128:
 467                         cdesc->control_data.control0 |=
 468                                 CONTEXT_CONTROL_CRYPTO_ALG_AES128;
 469                         break;
 470                 case AES_KEYSIZE_192:
 471                         cdesc->control_data.control0 |=
 472                                 CONTEXT_CONTROL_CRYPTO_ALG_AES192;
 473                         break;
 474                 case AES_KEYSIZE_256:
 475                         cdesc->control_data.control0 |=
 476                                 CONTEXT_CONTROL_CRYPTO_ALG_AES256;
 477                         break;
 478                 default:
 479                         dev_err(priv->dev, "aes keysize not supported: %u\n",
 480                                 ctx->key_len >> ctx->xts);
 481                         return -EINVAL;
 482                 }
 483         }
 484 
 485         return 0;
 486 }
 487 
 488 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
 489                                       struct crypto_async_request *async,
 490                                       struct scatterlist *src,
 491                                       struct scatterlist *dst,
 492                                       unsigned int cryptlen,
 493                                       struct safexcel_cipher_req *sreq,
 494                                       bool *should_complete, int *ret)
 495 {
 496         struct skcipher_request *areq = skcipher_request_cast(async);
 497         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 498         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher);
 499         struct safexcel_result_desc *rdesc;
 500         int ndesc = 0;
 501 
 502         *ret = 0;
 503 
 504         if (unlikely(!sreq->rdescs))
 505                 return 0;
 506 
 507         while (sreq->rdescs--) {
 508                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 509                 if (IS_ERR(rdesc)) {
 510                         dev_err(priv->dev,
 511                                 "cipher: result: could not retrieve the result descriptor\n");
 512                         *ret = PTR_ERR(rdesc);
 513                         break;
 514                 }
 515 
 516                 if (likely(!*ret))
 517                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 518 
 519                 ndesc++;
 520         }
 521 
 522         safexcel_complete(priv, ring);
 523 
 524         if (src == dst) {
 525                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
 526         } else {
 527                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 528                 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
 529         }
 530 
 531         /*
 532          * Update IV in req from last crypto output word for CBC modes
 533          */
 534         if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 535             (sreq->direction == SAFEXCEL_ENCRYPT)) {
 536                 /* For encrypt take the last output word */
 537                 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv,
 538                                    crypto_skcipher_ivsize(skcipher),
 539                                    (cryptlen -
 540                                     crypto_skcipher_ivsize(skcipher)));
 541         }
 542 
 543         *should_complete = true;
 544 
 545         return ndesc;
 546 }
 547 
 548 static int safexcel_send_req(struct crypto_async_request *base, int ring,
 549                              struct safexcel_cipher_req *sreq,
 550                              struct scatterlist *src, struct scatterlist *dst,
 551                              unsigned int cryptlen, unsigned int assoclen,
 552                              unsigned int digestsize, u8 *iv, int *commands,
 553                              int *results)
 554 {
 555         struct skcipher_request *areq = skcipher_request_cast(base);
 556         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 557         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 558         struct safexcel_crypto_priv *priv = ctx->priv;
 559         struct safexcel_command_desc *cdesc;
 560         struct safexcel_command_desc *first_cdesc = NULL;
 561         struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
 562         struct scatterlist *sg;
 563         unsigned int totlen;
 564         unsigned int totlen_src = cryptlen + assoclen;
 565         unsigned int totlen_dst = totlen_src;
 566         int n_cdesc = 0, n_rdesc = 0;
 567         int queued, i, ret = 0;
 568         bool first = true;
 569 
 570         sreq->nr_src = sg_nents_for_len(src, totlen_src);
 571 
 572         if (ctx->aead) {
 573                 /*
 574                  * AEAD has auth tag appended to output for encrypt and
 575                  * removed from the output for decrypt!
 576                  */
 577                 if (sreq->direction == SAFEXCEL_DECRYPT)
 578                         totlen_dst -= digestsize;
 579                 else
 580                         totlen_dst += digestsize;
 581 
 582                 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
 583                        ctx->ipad, ctx->state_sz);
 584                 if (!ctx->xcm)
 585                         memcpy(ctx->base.ctxr->data + (ctx->key_len +
 586                                ctx->state_sz) / sizeof(u32), ctx->opad,
 587                                ctx->state_sz);
 588         } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 589                    (sreq->direction == SAFEXCEL_DECRYPT)) {
 590                 /*
 591                  * Save IV from last crypto input word for CBC modes in decrypt
 592                  * direction. Need to do this first in case of inplace operation
 593                  * as it will be overwritten.
 594                  */
 595                 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv,
 596                                    crypto_skcipher_ivsize(skcipher),
 597                                    (totlen_src -
 598                                     crypto_skcipher_ivsize(skcipher)));
 599         }
 600 
 601         sreq->nr_dst = sg_nents_for_len(dst, totlen_dst);
 602 
 603         /*
 604          * Remember actual input length, source buffer length may be
 605          * updated in case of inline operation below.
 606          */
 607         totlen = totlen_src;
 608         queued = totlen_src;
 609 
 610         if (src == dst) {
 611                 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst);
 612                 sreq->nr_dst = sreq->nr_src;
 613                 if (unlikely((totlen_src || totlen_dst) &&
 614                     (sreq->nr_src <= 0))) {
 615                         dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!",
 616                                 max(totlen_src, totlen_dst));
 617                         return -EINVAL;
 618                 }
 619                 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
 620         } else {
 621                 if (unlikely(totlen_src && (sreq->nr_src <= 0))) {
 622                         dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!",
 623                                 totlen_src);
 624                         return -EINVAL;
 625                 }
 626                 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 627 
 628                 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) {
 629                         dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!",
 630                                 totlen_dst);
 631                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 632                                      DMA_TO_DEVICE);
 633                         return -EINVAL;
 634                 }
 635                 dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
 636         }
 637 
 638         memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
 639 
 640         /* The EIP cannot deal with zero length input packets! */
 641         if (totlen == 0)
 642                 totlen = 1;
 643 
 644         /* command descriptors */
 645         for_each_sg(src, sg, sreq->nr_src, i) {
 646                 int len = sg_dma_len(sg);
 647 
 648                 /* Do not overflow the request */
 649                 if (queued - len < 0)
 650                         len = queued;
 651 
 652                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 653                                            !(queued - len),
 654                                            sg_dma_address(sg), len, totlen,
 655                                            ctx->base.ctxr_dma);
 656                 if (IS_ERR(cdesc)) {
 657                         /* No space left in the command descriptor ring */
 658                         ret = PTR_ERR(cdesc);
 659                         goto cdesc_rollback;
 660                 }
 661                 n_cdesc++;
 662 
 663                 if (n_cdesc == 1) {
 664                         first_cdesc = cdesc;
 665                 }
 666 
 667                 queued -= len;
 668                 if (!queued)
 669                         break;
 670         }
 671 
 672         if (unlikely(!n_cdesc)) {
 673                 /*
 674                  * Special case: zero length input buffer.
 675                  * The engine always needs the 1st command descriptor, however!
 676                  */
 677                 first_cdesc = safexcel_add_cdesc(priv, ring, 1, 1, 0, 0, totlen,
 678                                                  ctx->base.ctxr_dma);
 679                 n_cdesc = 1;
 680         }
 681 
 682         /* Add context control words and token to first command descriptor */
 683         safexcel_context_control(ctx, base, sreq, first_cdesc);
 684         if (ctx->aead)
 685                 safexcel_aead_token(ctx, iv, first_cdesc,
 686                                     sreq->direction, cryptlen,
 687                                     assoclen, digestsize);
 688         else
 689                 safexcel_skcipher_token(ctx, iv, first_cdesc,
 690                                         cryptlen);
 691 
 692         /* result descriptors */
 693         for_each_sg(dst, sg, sreq->nr_dst, i) {
 694                 bool last = (i == sreq->nr_dst - 1);
 695                 u32 len = sg_dma_len(sg);
 696 
 697                 /* only allow the part of the buffer we know we need */
 698                 if (len > totlen_dst)
 699                         len = totlen_dst;
 700                 if (unlikely(!len))
 701                         break;
 702                 totlen_dst -= len;
 703 
 704                 /* skip over AAD space in buffer - not written */
 705                 if (assoclen) {
 706                         if (assoclen >= len) {
 707                                 assoclen -= len;
 708                                 continue;
 709                         }
 710                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 711                                                    sg_dma_address(sg) +
 712                                                    assoclen,
 713                                                    len - assoclen);
 714                         assoclen = 0;
 715                 } else {
 716                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 717                                                    sg_dma_address(sg),
 718                                                    len);
 719                 }
 720                 if (IS_ERR(rdesc)) {
 721                         /* No space left in the result descriptor ring */
 722                         ret = PTR_ERR(rdesc);
 723                         goto rdesc_rollback;
 724                 }
 725                 if (first) {
 726                         first_rdesc = rdesc;
 727                         first = false;
 728                 }
 729                 n_rdesc++;
 730         }
 731 
 732         if (unlikely(first)) {
 733                 /*
 734                  * Special case: AEAD decrypt with only AAD data.
 735                  * In this case there is NO output data from the engine,
 736                  * but the engine still needs a result descriptor!
 737                  * Create a dummy one just for catching the result token.
 738                  */
 739                 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
 740                 if (IS_ERR(rdesc)) {
 741                         /* No space left in the result descriptor ring */
 742                         ret = PTR_ERR(rdesc);
 743                         goto rdesc_rollback;
 744                 }
 745                 first_rdesc = rdesc;
 746                 n_rdesc = 1;
 747         }
 748 
 749         safexcel_rdr_req_set(priv, ring, first_rdesc, base);
 750 
 751         *commands = n_cdesc;
 752         *results = n_rdesc;
 753         return 0;
 754 
 755 rdesc_rollback:
 756         for (i = 0; i < n_rdesc; i++)
 757                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
 758 cdesc_rollback:
 759         for (i = 0; i < n_cdesc; i++)
 760                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 761 
 762         if (src == dst) {
 763                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
 764         } else {
 765                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 766                 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
 767         }
 768 
 769         return ret;
 770 }
 771 
 772 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 773                                       int ring,
 774                                       struct crypto_async_request *base,
 775                                       struct safexcel_cipher_req *sreq,
 776                                       bool *should_complete, int *ret)
 777 {
 778         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 779         struct safexcel_result_desc *rdesc;
 780         int ndesc = 0, enq_ret;
 781 
 782         *ret = 0;
 783 
 784         if (unlikely(!sreq->rdescs))
 785                 return 0;
 786 
 787         while (sreq->rdescs--) {
 788                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 789                 if (IS_ERR(rdesc)) {
 790                         dev_err(priv->dev,
 791                                 "cipher: invalidate: could not retrieve the result descriptor\n");
 792                         *ret = PTR_ERR(rdesc);
 793                         break;
 794                 }
 795 
 796                 if (likely(!*ret))
 797                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 798 
 799                 ndesc++;
 800         }
 801 
 802         safexcel_complete(priv, ring);
 803 
 804         if (ctx->base.exit_inv) {
 805                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 806                               ctx->base.ctxr_dma);
 807 
 808                 *should_complete = true;
 809 
 810                 return ndesc;
 811         }
 812 
 813         ring = safexcel_select_ring(priv);
 814         ctx->base.ring = ring;
 815 
 816         spin_lock_bh(&priv->ring[ring].queue_lock);
 817         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 818         spin_unlock_bh(&priv->ring[ring].queue_lock);
 819 
 820         if (enq_ret != -EINPROGRESS)
 821                 *ret = enq_ret;
 822 
 823         queue_work(priv->ring[ring].workqueue,
 824                    &priv->ring[ring].work_data.work);
 825 
 826         *should_complete = false;
 827 
 828         return ndesc;
 829 }
 830 
 831 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
 832                                            int ring,
 833                                            struct crypto_async_request *async,
 834                                            bool *should_complete, int *ret)
 835 {
 836         struct skcipher_request *req = skcipher_request_cast(async);
 837         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 838         int err;
 839 
 840         if (sreq->needs_inv) {
 841                 sreq->needs_inv = false;
 842                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
 843                                                  should_complete, ret);
 844         } else {
 845                 err = safexcel_handle_req_result(priv, ring, async, req->src,
 846                                                  req->dst, req->cryptlen, sreq,
 847                                                  should_complete, ret);
 848         }
 849 
 850         return err;
 851 }
 852 
 853 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
 854                                        int ring,
 855                                        struct crypto_async_request *async,
 856                                        bool *should_complete, int *ret)
 857 {
 858         struct aead_request *req = aead_request_cast(async);
 859         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 860         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
 861         int err;
 862 
 863         if (sreq->needs_inv) {
 864                 sreq->needs_inv = false;
 865                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
 866                                                  should_complete, ret);
 867         } else {
 868                 err = safexcel_handle_req_result(priv, ring, async, req->src,
 869                                                  req->dst,
 870                                                  req->cryptlen + crypto_aead_authsize(tfm),
 871                                                  sreq, should_complete, ret);
 872         }
 873 
 874         return err;
 875 }
 876 
 877 static int safexcel_cipher_send_inv(struct crypto_async_request *base,
 878                                     int ring, int *commands, int *results)
 879 {
 880         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 881         struct safexcel_crypto_priv *priv = ctx->priv;
 882         int ret;
 883 
 884         ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
 885         if (unlikely(ret))
 886                 return ret;
 887 
 888         *commands = 1;
 889         *results = 1;
 890 
 891         return 0;
 892 }
 893 
 894 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
 895                                   int *commands, int *results)
 896 {
 897         struct skcipher_request *req = skcipher_request_cast(async);
 898         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 899         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 900         struct safexcel_crypto_priv *priv = ctx->priv;
 901         int ret;
 902 
 903         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
 904 
 905         if (sreq->needs_inv) {
 906                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
 907         } else {
 908                 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
 909                 u8 input_iv[AES_BLOCK_SIZE];
 910 
 911                 /*
 912                  * Save input IV in case of CBC decrypt mode
 913                  * Will be overwritten with output IV prior to use!
 914                  */
 915                 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher));
 916 
 917                 ret = safexcel_send_req(async, ring, sreq, req->src,
 918                                         req->dst, req->cryptlen, 0, 0, input_iv,
 919                                         commands, results);
 920         }
 921 
 922         sreq->rdescs = *results;
 923         return ret;
 924 }
 925 
 926 static int safexcel_aead_send(struct crypto_async_request *async, int ring,
 927                               int *commands, int *results)
 928 {
 929         struct aead_request *req = aead_request_cast(async);
 930         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 931         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 932         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
 933         struct safexcel_crypto_priv *priv = ctx->priv;
 934         int ret;
 935 
 936         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
 937 
 938         if (sreq->needs_inv)
 939                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
 940         else
 941                 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst,
 942                                         req->cryptlen, req->assoclen,
 943                                         crypto_aead_authsize(tfm), req->iv,
 944                                         commands, results);
 945         sreq->rdescs = *results;
 946         return ret;
 947 }
 948 
 949 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
 950                                     struct crypto_async_request *base,
 951                                     struct safexcel_cipher_req *sreq,
 952                                     struct safexcel_inv_result *result)
 953 {
 954         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 955         struct safexcel_crypto_priv *priv = ctx->priv;
 956         int ring = ctx->base.ring;
 957 
 958         init_completion(&result->completion);
 959 
 960         ctx = crypto_tfm_ctx(base->tfm);
 961         ctx->base.exit_inv = true;
 962         sreq->needs_inv = true;
 963 
 964         spin_lock_bh(&priv->ring[ring].queue_lock);
 965         crypto_enqueue_request(&priv->ring[ring].queue, base);
 966         spin_unlock_bh(&priv->ring[ring].queue_lock);
 967 
 968         queue_work(priv->ring[ring].workqueue,
 969                    &priv->ring[ring].work_data.work);
 970 
 971         wait_for_completion(&result->completion);
 972 
 973         if (result->error) {
 974                 dev_warn(priv->dev,
 975                         "cipher: sync: invalidate: completion error %d\n",
 976                          result->error);
 977                 return result->error;
 978         }
 979 
 980         return 0;
 981 }
 982 
 983 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
 984 {
 985         EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
 986         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 987         struct safexcel_inv_result result = {};
 988 
 989         memset(req, 0, sizeof(struct skcipher_request));
 990 
 991         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 992                                       safexcel_inv_complete, &result);
 993         skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
 994 
 995         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
 996 }
 997 
 998 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
 999 {
1000         EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
1001         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1002         struct safexcel_inv_result result = {};
1003 
1004         memset(req, 0, sizeof(struct aead_request));
1005 
1006         aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1007                                   safexcel_inv_complete, &result);
1008         aead_request_set_tfm(req, __crypto_aead_cast(tfm));
1009 
1010         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
1011 }
1012 
1013 static int safexcel_queue_req(struct crypto_async_request *base,
1014                         struct safexcel_cipher_req *sreq,
1015                         enum safexcel_cipher_direction dir)
1016 {
1017         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
1018         struct safexcel_crypto_priv *priv = ctx->priv;
1019         int ret, ring;
1020 
1021         sreq->needs_inv = false;
1022         sreq->direction = dir;
1023 
1024         if (ctx->base.ctxr) {
1025                 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
1026                         sreq->needs_inv = true;
1027                         ctx->base.needs_inv = false;
1028                 }
1029         } else {
1030                 ctx->base.ring = safexcel_select_ring(priv);
1031                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
1032                                                  EIP197_GFP_FLAGS(*base),
1033                                                  &ctx->base.ctxr_dma);
1034                 if (!ctx->base.ctxr)
1035                         return -ENOMEM;
1036         }
1037 
1038         ring = ctx->base.ring;
1039 
1040         spin_lock_bh(&priv->ring[ring].queue_lock);
1041         ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
1042         spin_unlock_bh(&priv->ring[ring].queue_lock);
1043 
1044         queue_work(priv->ring[ring].workqueue,
1045                    &priv->ring[ring].work_data.work);
1046 
1047         return ret;
1048 }
1049 
1050 static int safexcel_encrypt(struct skcipher_request *req)
1051 {
1052         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1053                         SAFEXCEL_ENCRYPT);
1054 }
1055 
1056 static int safexcel_decrypt(struct skcipher_request *req)
1057 {
1058         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1059                         SAFEXCEL_DECRYPT);
1060 }
1061 
1062 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
1063 {
1064         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1065         struct safexcel_alg_template *tmpl =
1066                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1067                              alg.skcipher.base);
1068 
1069         crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
1070                                     sizeof(struct safexcel_cipher_req));
1071 
1072         ctx->priv = tmpl->priv;
1073 
1074         ctx->base.send = safexcel_skcipher_send;
1075         ctx->base.handle_result = safexcel_skcipher_handle_result;
1076         return 0;
1077 }
1078 
1079 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
1080 {
1081         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1082 
1083         memzero_explicit(ctx->key, sizeof(ctx->key));
1084 
1085         /* context not allocated, skip invalidation */
1086         if (!ctx->base.ctxr)
1087                 return -ENOMEM;
1088 
1089         memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
1090         return 0;
1091 }
1092 
1093 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
1094 {
1095         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1096         struct safexcel_crypto_priv *priv = ctx->priv;
1097         int ret;
1098 
1099         if (safexcel_cipher_cra_exit(tfm))
1100                 return;
1101 
1102         if (priv->flags & EIP197_TRC_CACHE) {
1103                 ret = safexcel_skcipher_exit_inv(tfm);
1104                 if (ret)
1105                         dev_warn(priv->dev, "skcipher: invalidation error %d\n",
1106                                  ret);
1107         } else {
1108                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1109                               ctx->base.ctxr_dma);
1110         }
1111 }
1112 
1113 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
1114 {
1115         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1116         struct safexcel_crypto_priv *priv = ctx->priv;
1117         int ret;
1118 
1119         if (safexcel_cipher_cra_exit(tfm))
1120                 return;
1121 
1122         if (priv->flags & EIP197_TRC_CACHE) {
1123                 ret = safexcel_aead_exit_inv(tfm);
1124                 if (ret)
1125                         dev_warn(priv->dev, "aead: invalidation error %d\n",
1126                                  ret);
1127         } else {
1128                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1129                               ctx->base.ctxr_dma);
1130         }
1131 }
1132 
1133 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm)
1134 {
1135         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1136 
1137         safexcel_skcipher_cra_init(tfm);
1138         ctx->alg  = SAFEXCEL_AES;
1139         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1140         return 0;
1141 }
1142 
1143 struct safexcel_alg_template safexcel_alg_ecb_aes = {
1144         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1145         .algo_mask = SAFEXCEL_ALG_AES,
1146         .alg.skcipher = {
1147                 .setkey = safexcel_skcipher_aes_setkey,
1148                 .encrypt = safexcel_encrypt,
1149                 .decrypt = safexcel_decrypt,
1150                 .min_keysize = AES_MIN_KEY_SIZE,
1151                 .max_keysize = AES_MAX_KEY_SIZE,
1152                 .base = {
1153                         .cra_name = "ecb(aes)",
1154                         .cra_driver_name = "safexcel-ecb-aes",
1155                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1156                         .cra_flags = CRYPTO_ALG_ASYNC |
1157                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1158                         .cra_blocksize = AES_BLOCK_SIZE,
1159                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1160                         .cra_alignmask = 0,
1161                         .cra_init = safexcel_skcipher_aes_ecb_cra_init,
1162                         .cra_exit = safexcel_skcipher_cra_exit,
1163                         .cra_module = THIS_MODULE,
1164                 },
1165         },
1166 };
1167 
1168 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm)
1169 {
1170         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1171 
1172         safexcel_skcipher_cra_init(tfm);
1173         ctx->alg  = SAFEXCEL_AES;
1174         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1175         return 0;
1176 }
1177 
1178 struct safexcel_alg_template safexcel_alg_cbc_aes = {
1179         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1180         .algo_mask = SAFEXCEL_ALG_AES,
1181         .alg.skcipher = {
1182                 .setkey = safexcel_skcipher_aes_setkey,
1183                 .encrypt = safexcel_encrypt,
1184                 .decrypt = safexcel_decrypt,
1185                 .min_keysize = AES_MIN_KEY_SIZE,
1186                 .max_keysize = AES_MAX_KEY_SIZE,
1187                 .ivsize = AES_BLOCK_SIZE,
1188                 .base = {
1189                         .cra_name = "cbc(aes)",
1190                         .cra_driver_name = "safexcel-cbc-aes",
1191                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1192                         .cra_flags = CRYPTO_ALG_ASYNC |
1193                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1194                         .cra_blocksize = AES_BLOCK_SIZE,
1195                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1196                         .cra_alignmask = 0,
1197                         .cra_init = safexcel_skcipher_aes_cbc_cra_init,
1198                         .cra_exit = safexcel_skcipher_cra_exit,
1199                         .cra_module = THIS_MODULE,
1200                 },
1201         },
1202 };
1203 
1204 static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm)
1205 {
1206         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1207 
1208         safexcel_skcipher_cra_init(tfm);
1209         ctx->alg  = SAFEXCEL_AES;
1210         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
1211         return 0;
1212 }
1213 
1214 struct safexcel_alg_template safexcel_alg_cfb_aes = {
1215         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1216         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1217         .alg.skcipher = {
1218                 .setkey = safexcel_skcipher_aes_setkey,
1219                 .encrypt = safexcel_encrypt,
1220                 .decrypt = safexcel_decrypt,
1221                 .min_keysize = AES_MIN_KEY_SIZE,
1222                 .max_keysize = AES_MAX_KEY_SIZE,
1223                 .ivsize = AES_BLOCK_SIZE,
1224                 .base = {
1225                         .cra_name = "cfb(aes)",
1226                         .cra_driver_name = "safexcel-cfb-aes",
1227                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1228                         .cra_flags = CRYPTO_ALG_ASYNC |
1229                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1230                         .cra_blocksize = 1,
1231                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1232                         .cra_alignmask = 0,
1233                         .cra_init = safexcel_skcipher_aes_cfb_cra_init,
1234                         .cra_exit = safexcel_skcipher_cra_exit,
1235                         .cra_module = THIS_MODULE,
1236                 },
1237         },
1238 };
1239 
1240 static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm)
1241 {
1242         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1243 
1244         safexcel_skcipher_cra_init(tfm);
1245         ctx->alg  = SAFEXCEL_AES;
1246         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
1247         return 0;
1248 }
1249 
1250 struct safexcel_alg_template safexcel_alg_ofb_aes = {
1251         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1252         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1253         .alg.skcipher = {
1254                 .setkey = safexcel_skcipher_aes_setkey,
1255                 .encrypt = safexcel_encrypt,
1256                 .decrypt = safexcel_decrypt,
1257                 .min_keysize = AES_MIN_KEY_SIZE,
1258                 .max_keysize = AES_MAX_KEY_SIZE,
1259                 .ivsize = AES_BLOCK_SIZE,
1260                 .base = {
1261                         .cra_name = "ofb(aes)",
1262                         .cra_driver_name = "safexcel-ofb-aes",
1263                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1264                         .cra_flags = CRYPTO_ALG_ASYNC |
1265                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1266                         .cra_blocksize = 1,
1267                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1268                         .cra_alignmask = 0,
1269                         .cra_init = safexcel_skcipher_aes_ofb_cra_init,
1270                         .cra_exit = safexcel_skcipher_cra_exit,
1271                         .cra_module = THIS_MODULE,
1272                 },
1273         },
1274 };
1275 
1276 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm,
1277                                            const u8 *key, unsigned int len)
1278 {
1279         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
1280         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1281         struct safexcel_crypto_priv *priv = ctx->priv;
1282         struct crypto_aes_ctx aes;
1283         int ret, i;
1284         unsigned int keylen;
1285 
1286         /* last 4 bytes of key are the nonce! */
1287         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
1288         /* exclude the nonce here */
1289         keylen = len - CTR_RFC3686_NONCE_SIZE;
1290         ret = aes_expandkey(&aes, key, keylen);
1291         if (ret) {
1292                 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1293                 return ret;
1294         }
1295 
1296         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
1297                 for (i = 0; i < keylen / sizeof(u32); i++) {
1298                         if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
1299                                 ctx->base.needs_inv = true;
1300                                 break;
1301                         }
1302                 }
1303         }
1304 
1305         for (i = 0; i < keylen / sizeof(u32); i++)
1306                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
1307 
1308         ctx->key_len = keylen;
1309 
1310         memzero_explicit(&aes, sizeof(aes));
1311         return 0;
1312 }
1313 
1314 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm)
1315 {
1316         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1317 
1318         safexcel_skcipher_cra_init(tfm);
1319         ctx->alg  = SAFEXCEL_AES;
1320         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
1321         return 0;
1322 }
1323 
1324 struct safexcel_alg_template safexcel_alg_ctr_aes = {
1325         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1326         .algo_mask = SAFEXCEL_ALG_AES,
1327         .alg.skcipher = {
1328                 .setkey = safexcel_skcipher_aesctr_setkey,
1329                 .encrypt = safexcel_encrypt,
1330                 .decrypt = safexcel_decrypt,
1331                 /* Add nonce size */
1332                 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1333                 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1334                 .ivsize = CTR_RFC3686_IV_SIZE,
1335                 .base = {
1336                         .cra_name = "rfc3686(ctr(aes))",
1337                         .cra_driver_name = "safexcel-ctr-aes",
1338                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1339                         .cra_flags = CRYPTO_ALG_ASYNC |
1340                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1341                         .cra_blocksize = 1,
1342                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1343                         .cra_alignmask = 0,
1344                         .cra_init = safexcel_skcipher_aes_ctr_cra_init,
1345                         .cra_exit = safexcel_skcipher_cra_exit,
1346                         .cra_module = THIS_MODULE,
1347                 },
1348         },
1349 };
1350 
1351 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
1352                                unsigned int len)
1353 {
1354         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1355         int ret;
1356 
1357         ret = verify_skcipher_des_key(ctfm, key);
1358         if (ret)
1359                 return ret;
1360 
1361         /* if context exits and key changed, need to invalidate it */
1362         if (ctx->base.ctxr_dma)
1363                 if (memcmp(ctx->key, key, len))
1364                         ctx->base.needs_inv = true;
1365 
1366         memcpy(ctx->key, key, len);
1367         ctx->key_len = len;
1368 
1369         return 0;
1370 }
1371 
1372 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm)
1373 {
1374         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1375 
1376         safexcel_skcipher_cra_init(tfm);
1377         ctx->alg  = SAFEXCEL_DES;
1378         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1379         return 0;
1380 }
1381 
1382 struct safexcel_alg_template safexcel_alg_cbc_des = {
1383         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1384         .algo_mask = SAFEXCEL_ALG_DES,
1385         .alg.skcipher = {
1386                 .setkey = safexcel_des_setkey,
1387                 .encrypt = safexcel_encrypt,
1388                 .decrypt = safexcel_decrypt,
1389                 .min_keysize = DES_KEY_SIZE,
1390                 .max_keysize = DES_KEY_SIZE,
1391                 .ivsize = DES_BLOCK_SIZE,
1392                 .base = {
1393                         .cra_name = "cbc(des)",
1394                         .cra_driver_name = "safexcel-cbc-des",
1395                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1396                         .cra_flags = CRYPTO_ALG_ASYNC |
1397                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1398                         .cra_blocksize = DES_BLOCK_SIZE,
1399                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1400                         .cra_alignmask = 0,
1401                         .cra_init = safexcel_skcipher_des_cbc_cra_init,
1402                         .cra_exit = safexcel_skcipher_cra_exit,
1403                         .cra_module = THIS_MODULE,
1404                 },
1405         },
1406 };
1407 
1408 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm)
1409 {
1410         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1411 
1412         safexcel_skcipher_cra_init(tfm);
1413         ctx->alg  = SAFEXCEL_DES;
1414         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1415         return 0;
1416 }
1417 
1418 struct safexcel_alg_template safexcel_alg_ecb_des = {
1419         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1420         .algo_mask = SAFEXCEL_ALG_DES,
1421         .alg.skcipher = {
1422                 .setkey = safexcel_des_setkey,
1423                 .encrypt = safexcel_encrypt,
1424                 .decrypt = safexcel_decrypt,
1425                 .min_keysize = DES_KEY_SIZE,
1426                 .max_keysize = DES_KEY_SIZE,
1427                 .base = {
1428                         .cra_name = "ecb(des)",
1429                         .cra_driver_name = "safexcel-ecb-des",
1430                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1431                         .cra_flags = CRYPTO_ALG_ASYNC |
1432                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1433                         .cra_blocksize = DES_BLOCK_SIZE,
1434                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1435                         .cra_alignmask = 0,
1436                         .cra_init = safexcel_skcipher_des_ecb_cra_init,
1437                         .cra_exit = safexcel_skcipher_cra_exit,
1438                         .cra_module = THIS_MODULE,
1439                 },
1440         },
1441 };
1442 
1443 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
1444                                    const u8 *key, unsigned int len)
1445 {
1446         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1447         int err;
1448 
1449         err = verify_skcipher_des3_key(ctfm, key);
1450         if (err)
1451                 return err;
1452 
1453         /* if context exits and key changed, need to invalidate it */
1454         if (ctx->base.ctxr_dma) {
1455                 if (memcmp(ctx->key, key, len))
1456                         ctx->base.needs_inv = true;
1457         }
1458 
1459         memcpy(ctx->key, key, len);
1460 
1461         ctx->key_len = len;
1462 
1463         return 0;
1464 }
1465 
1466 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm)
1467 {
1468         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1469 
1470         safexcel_skcipher_cra_init(tfm);
1471         ctx->alg  = SAFEXCEL_3DES;
1472         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1473         return 0;
1474 }
1475 
1476 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
1477         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1478         .algo_mask = SAFEXCEL_ALG_DES,
1479         .alg.skcipher = {
1480                 .setkey = safexcel_des3_ede_setkey,
1481                 .encrypt = safexcel_encrypt,
1482                 .decrypt = safexcel_decrypt,
1483                 .min_keysize = DES3_EDE_KEY_SIZE,
1484                 .max_keysize = DES3_EDE_KEY_SIZE,
1485                 .ivsize = DES3_EDE_BLOCK_SIZE,
1486                 .base = {
1487                         .cra_name = "cbc(des3_ede)",
1488                         .cra_driver_name = "safexcel-cbc-des3_ede",
1489                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1490                         .cra_flags = CRYPTO_ALG_ASYNC |
1491                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1492                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1493                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1494                         .cra_alignmask = 0,
1495                         .cra_init = safexcel_skcipher_des3_cbc_cra_init,
1496                         .cra_exit = safexcel_skcipher_cra_exit,
1497                         .cra_module = THIS_MODULE,
1498                 },
1499         },
1500 };
1501 
1502 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm)
1503 {
1504         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1505 
1506         safexcel_skcipher_cra_init(tfm);
1507         ctx->alg  = SAFEXCEL_3DES;
1508         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1509         return 0;
1510 }
1511 
1512 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
1513         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1514         .algo_mask = SAFEXCEL_ALG_DES,
1515         .alg.skcipher = {
1516                 .setkey = safexcel_des3_ede_setkey,
1517                 .encrypt = safexcel_encrypt,
1518                 .decrypt = safexcel_decrypt,
1519                 .min_keysize = DES3_EDE_KEY_SIZE,
1520                 .max_keysize = DES3_EDE_KEY_SIZE,
1521                 .base = {
1522                         .cra_name = "ecb(des3_ede)",
1523                         .cra_driver_name = "safexcel-ecb-des3_ede",
1524                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1525                         .cra_flags = CRYPTO_ALG_ASYNC |
1526                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1527                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1528                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1529                         .cra_alignmask = 0,
1530                         .cra_init = safexcel_skcipher_des3_ecb_cra_init,
1531                         .cra_exit = safexcel_skcipher_cra_exit,
1532                         .cra_module = THIS_MODULE,
1533                 },
1534         },
1535 };
1536 
1537 static int safexcel_aead_encrypt(struct aead_request *req)
1538 {
1539         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1540 
1541         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
1542 }
1543 
1544 static int safexcel_aead_decrypt(struct aead_request *req)
1545 {
1546         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1547 
1548         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
1549 }
1550 
1551 static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
1552 {
1553         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1554         struct safexcel_alg_template *tmpl =
1555                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1556                              alg.aead.base);
1557 
1558         crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1559                                 sizeof(struct safexcel_cipher_req));
1560 
1561         ctx->priv = tmpl->priv;
1562 
1563         ctx->alg  = SAFEXCEL_AES; /* default */
1564         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */
1565         ctx->aead = true;
1566         ctx->base.send = safexcel_aead_send;
1567         ctx->base.handle_result = safexcel_aead_handle_result;
1568         return 0;
1569 }
1570 
1571 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
1572 {
1573         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1574 
1575         safexcel_aead_cra_init(tfm);
1576         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1577         ctx->state_sz = SHA1_DIGEST_SIZE;
1578         return 0;
1579 }
1580 
1581 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
1582         .type = SAFEXCEL_ALG_TYPE_AEAD,
1583         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
1584         .alg.aead = {
1585                 .setkey = safexcel_aead_setkey,
1586                 .encrypt = safexcel_aead_encrypt,
1587                 .decrypt = safexcel_aead_decrypt,
1588                 .ivsize = AES_BLOCK_SIZE,
1589                 .maxauthsize = SHA1_DIGEST_SIZE,
1590                 .base = {
1591                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1592                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
1593                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1594                         .cra_flags = CRYPTO_ALG_ASYNC |
1595                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1596                         .cra_blocksize = AES_BLOCK_SIZE,
1597                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1598                         .cra_alignmask = 0,
1599                         .cra_init = safexcel_aead_sha1_cra_init,
1600                         .cra_exit = safexcel_aead_cra_exit,
1601                         .cra_module = THIS_MODULE,
1602                 },
1603         },
1604 };
1605 
1606 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
1607 {
1608         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1609 
1610         safexcel_aead_cra_init(tfm);
1611         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1612         ctx->state_sz = SHA256_DIGEST_SIZE;
1613         return 0;
1614 }
1615 
1616 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
1617         .type = SAFEXCEL_ALG_TYPE_AEAD,
1618         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1619         .alg.aead = {
1620                 .setkey = safexcel_aead_setkey,
1621                 .encrypt = safexcel_aead_encrypt,
1622                 .decrypt = safexcel_aead_decrypt,
1623                 .ivsize = AES_BLOCK_SIZE,
1624                 .maxauthsize = SHA256_DIGEST_SIZE,
1625                 .base = {
1626                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1627                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
1628                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1629                         .cra_flags = CRYPTO_ALG_ASYNC |
1630                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1631                         .cra_blocksize = AES_BLOCK_SIZE,
1632                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1633                         .cra_alignmask = 0,
1634                         .cra_init = safexcel_aead_sha256_cra_init,
1635                         .cra_exit = safexcel_aead_cra_exit,
1636                         .cra_module = THIS_MODULE,
1637                 },
1638         },
1639 };
1640 
1641 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
1642 {
1643         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1644 
1645         safexcel_aead_cra_init(tfm);
1646         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1647         ctx->state_sz = SHA256_DIGEST_SIZE;
1648         return 0;
1649 }
1650 
1651 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
1652         .type = SAFEXCEL_ALG_TYPE_AEAD,
1653         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1654         .alg.aead = {
1655                 .setkey = safexcel_aead_setkey,
1656                 .encrypt = safexcel_aead_encrypt,
1657                 .decrypt = safexcel_aead_decrypt,
1658                 .ivsize = AES_BLOCK_SIZE,
1659                 .maxauthsize = SHA224_DIGEST_SIZE,
1660                 .base = {
1661                         .cra_name = "authenc(hmac(sha224),cbc(aes))",
1662                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
1663                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1664                         .cra_flags = CRYPTO_ALG_ASYNC |
1665                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1666                         .cra_blocksize = AES_BLOCK_SIZE,
1667                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1668                         .cra_alignmask = 0,
1669                         .cra_init = safexcel_aead_sha224_cra_init,
1670                         .cra_exit = safexcel_aead_cra_exit,
1671                         .cra_module = THIS_MODULE,
1672                 },
1673         },
1674 };
1675 
1676 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
1677 {
1678         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1679 
1680         safexcel_aead_cra_init(tfm);
1681         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1682         ctx->state_sz = SHA512_DIGEST_SIZE;
1683         return 0;
1684 }
1685 
1686 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
1687         .type = SAFEXCEL_ALG_TYPE_AEAD,
1688         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1689         .alg.aead = {
1690                 .setkey = safexcel_aead_setkey,
1691                 .encrypt = safexcel_aead_encrypt,
1692                 .decrypt = safexcel_aead_decrypt,
1693                 .ivsize = AES_BLOCK_SIZE,
1694                 .maxauthsize = SHA512_DIGEST_SIZE,
1695                 .base = {
1696                         .cra_name = "authenc(hmac(sha512),cbc(aes))",
1697                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
1698                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1699                         .cra_flags = CRYPTO_ALG_ASYNC |
1700                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1701                         .cra_blocksize = AES_BLOCK_SIZE,
1702                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1703                         .cra_alignmask = 0,
1704                         .cra_init = safexcel_aead_sha512_cra_init,
1705                         .cra_exit = safexcel_aead_cra_exit,
1706                         .cra_module = THIS_MODULE,
1707                 },
1708         },
1709 };
1710 
1711 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
1712 {
1713         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1714 
1715         safexcel_aead_cra_init(tfm);
1716         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1717         ctx->state_sz = SHA512_DIGEST_SIZE;
1718         return 0;
1719 }
1720 
1721 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
1722         .type = SAFEXCEL_ALG_TYPE_AEAD,
1723         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1724         .alg.aead = {
1725                 .setkey = safexcel_aead_setkey,
1726                 .encrypt = safexcel_aead_encrypt,
1727                 .decrypt = safexcel_aead_decrypt,
1728                 .ivsize = AES_BLOCK_SIZE,
1729                 .maxauthsize = SHA384_DIGEST_SIZE,
1730                 .base = {
1731                         .cra_name = "authenc(hmac(sha384),cbc(aes))",
1732                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
1733                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1734                         .cra_flags = CRYPTO_ALG_ASYNC |
1735                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1736                         .cra_blocksize = AES_BLOCK_SIZE,
1737                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1738                         .cra_alignmask = 0,
1739                         .cra_init = safexcel_aead_sha384_cra_init,
1740                         .cra_exit = safexcel_aead_cra_exit,
1741                         .cra_module = THIS_MODULE,
1742                 },
1743         },
1744 };
1745 
1746 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm)
1747 {
1748         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1749 
1750         safexcel_aead_sha1_cra_init(tfm);
1751         ctx->alg = SAFEXCEL_3DES; /* override default */
1752         return 0;
1753 }
1754 
1755 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = {
1756         .type = SAFEXCEL_ALG_TYPE_AEAD,
1757         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
1758         .alg.aead = {
1759                 .setkey = safexcel_aead_setkey,
1760                 .encrypt = safexcel_aead_encrypt,
1761                 .decrypt = safexcel_aead_decrypt,
1762                 .ivsize = DES3_EDE_BLOCK_SIZE,
1763                 .maxauthsize = SHA1_DIGEST_SIZE,
1764                 .base = {
1765                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1766                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede",
1767                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1768                         .cra_flags = CRYPTO_ALG_ASYNC |
1769                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1770                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1771                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1772                         .cra_alignmask = 0,
1773                         .cra_init = safexcel_aead_sha1_des3_cra_init,
1774                         .cra_exit = safexcel_aead_cra_exit,
1775                         .cra_module = THIS_MODULE,
1776                 },
1777         },
1778 };
1779 
1780 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm)
1781 {
1782         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1783 
1784         safexcel_aead_sha1_cra_init(tfm);
1785         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
1786         return 0;
1787 }
1788 
1789 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = {
1790         .type = SAFEXCEL_ALG_TYPE_AEAD,
1791         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
1792         .alg.aead = {
1793                 .setkey = safexcel_aead_setkey,
1794                 .encrypt = safexcel_aead_encrypt,
1795                 .decrypt = safexcel_aead_decrypt,
1796                 .ivsize = CTR_RFC3686_IV_SIZE,
1797                 .maxauthsize = SHA1_DIGEST_SIZE,
1798                 .base = {
1799                         .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
1800                         .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes",
1801                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1802                         .cra_flags = CRYPTO_ALG_ASYNC |
1803                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1804                         .cra_blocksize = 1,
1805                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1806                         .cra_alignmask = 0,
1807                         .cra_init = safexcel_aead_sha1_ctr_cra_init,
1808                         .cra_exit = safexcel_aead_cra_exit,
1809                         .cra_module = THIS_MODULE,
1810                 },
1811         },
1812 };
1813 
1814 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm)
1815 {
1816         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1817 
1818         safexcel_aead_sha256_cra_init(tfm);
1819         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
1820         return 0;
1821 }
1822 
1823 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = {
1824         .type = SAFEXCEL_ALG_TYPE_AEAD,
1825         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1826         .alg.aead = {
1827                 .setkey = safexcel_aead_setkey,
1828                 .encrypt = safexcel_aead_encrypt,
1829                 .decrypt = safexcel_aead_decrypt,
1830                 .ivsize = CTR_RFC3686_IV_SIZE,
1831                 .maxauthsize = SHA256_DIGEST_SIZE,
1832                 .base = {
1833                         .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
1834                         .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes",
1835                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1836                         .cra_flags = CRYPTO_ALG_ASYNC |
1837                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1838                         .cra_blocksize = 1,
1839                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1840                         .cra_alignmask = 0,
1841                         .cra_init = safexcel_aead_sha256_ctr_cra_init,
1842                         .cra_exit = safexcel_aead_cra_exit,
1843                         .cra_module = THIS_MODULE,
1844                 },
1845         },
1846 };
1847 
1848 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm)
1849 {
1850         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1851 
1852         safexcel_aead_sha224_cra_init(tfm);
1853         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
1854         return 0;
1855 }
1856 
1857 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = {
1858         .type = SAFEXCEL_ALG_TYPE_AEAD,
1859         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1860         .alg.aead = {
1861                 .setkey = safexcel_aead_setkey,
1862                 .encrypt = safexcel_aead_encrypt,
1863                 .decrypt = safexcel_aead_decrypt,
1864                 .ivsize = CTR_RFC3686_IV_SIZE,
1865                 .maxauthsize = SHA224_DIGEST_SIZE,
1866                 .base = {
1867                         .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
1868                         .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes",
1869                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1870                         .cra_flags = CRYPTO_ALG_ASYNC |
1871                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1872                         .cra_blocksize = 1,
1873                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1874                         .cra_alignmask = 0,
1875                         .cra_init = safexcel_aead_sha224_ctr_cra_init,
1876                         .cra_exit = safexcel_aead_cra_exit,
1877                         .cra_module = THIS_MODULE,
1878                 },
1879         },
1880 };
1881 
1882 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm)
1883 {
1884         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1885 
1886         safexcel_aead_sha512_cra_init(tfm);
1887         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
1888         return 0;
1889 }
1890 
1891 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = {
1892         .type = SAFEXCEL_ALG_TYPE_AEAD,
1893         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1894         .alg.aead = {
1895                 .setkey = safexcel_aead_setkey,
1896                 .encrypt = safexcel_aead_encrypt,
1897                 .decrypt = safexcel_aead_decrypt,
1898                 .ivsize = CTR_RFC3686_IV_SIZE,
1899                 .maxauthsize = SHA512_DIGEST_SIZE,
1900                 .base = {
1901                         .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
1902                         .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes",
1903                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1904                         .cra_flags = CRYPTO_ALG_ASYNC |
1905                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1906                         .cra_blocksize = 1,
1907                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1908                         .cra_alignmask = 0,
1909                         .cra_init = safexcel_aead_sha512_ctr_cra_init,
1910                         .cra_exit = safexcel_aead_cra_exit,
1911                         .cra_module = THIS_MODULE,
1912                 },
1913         },
1914 };
1915 
1916 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm)
1917 {
1918         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1919 
1920         safexcel_aead_sha384_cra_init(tfm);
1921         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
1922         return 0;
1923 }
1924 
1925 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = {
1926         .type = SAFEXCEL_ALG_TYPE_AEAD,
1927         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1928         .alg.aead = {
1929                 .setkey = safexcel_aead_setkey,
1930                 .encrypt = safexcel_aead_encrypt,
1931                 .decrypt = safexcel_aead_decrypt,
1932                 .ivsize = CTR_RFC3686_IV_SIZE,
1933                 .maxauthsize = SHA384_DIGEST_SIZE,
1934                 .base = {
1935                         .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
1936                         .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes",
1937                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1938                         .cra_flags = CRYPTO_ALG_ASYNC |
1939                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1940                         .cra_blocksize = 1,
1941                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1942                         .cra_alignmask = 0,
1943                         .cra_init = safexcel_aead_sha384_ctr_cra_init,
1944                         .cra_exit = safexcel_aead_cra_exit,
1945                         .cra_module = THIS_MODULE,
1946                 },
1947         },
1948 };
1949 
1950 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm,
1951                                            const u8 *key, unsigned int len)
1952 {
1953         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
1954         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1955         struct safexcel_crypto_priv *priv = ctx->priv;
1956         struct crypto_aes_ctx aes;
1957         int ret, i;
1958         unsigned int keylen;
1959 
1960         /* Check for illegal XTS keys */
1961         ret = xts_verify_key(ctfm, key, len);
1962         if (ret)
1963                 return ret;
1964 
1965         /* Only half of the key data is cipher key */
1966         keylen = (len >> 1);
1967         ret = aes_expandkey(&aes, key, keylen);
1968         if (ret) {
1969                 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1970                 return ret;
1971         }
1972 
1973         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
1974                 for (i = 0; i < keylen / sizeof(u32); i++) {
1975                         if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
1976                                 ctx->base.needs_inv = true;
1977                                 break;
1978                         }
1979                 }
1980         }
1981 
1982         for (i = 0; i < keylen / sizeof(u32); i++)
1983                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
1984 
1985         /* The other half is the tweak key */
1986         ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen);
1987         if (ret) {
1988                 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1989                 return ret;
1990         }
1991 
1992         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
1993                 for (i = 0; i < keylen / sizeof(u32); i++) {
1994                         if (ctx->key[i + keylen / sizeof(u32)] !=
1995                             cpu_to_le32(aes.key_enc[i])) {
1996                                 ctx->base.needs_inv = true;
1997                                 break;
1998                         }
1999                 }
2000         }
2001 
2002         for (i = 0; i < keylen / sizeof(u32); i++)
2003                 ctx->key[i + keylen / sizeof(u32)] =
2004                         cpu_to_le32(aes.key_enc[i]);
2005 
2006         ctx->key_len = keylen << 1;
2007 
2008         memzero_explicit(&aes, sizeof(aes));
2009         return 0;
2010 }
2011 
2012 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm)
2013 {
2014         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2015 
2016         safexcel_skcipher_cra_init(tfm);
2017         ctx->alg  = SAFEXCEL_AES;
2018         ctx->xts  = 1;
2019         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS;
2020         return 0;
2021 }
2022 
2023 static int safexcel_encrypt_xts(struct skcipher_request *req)
2024 {
2025         if (req->cryptlen < XTS_BLOCK_SIZE)
2026                 return -EINVAL;
2027         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2028                                   SAFEXCEL_ENCRYPT);
2029 }
2030 
2031 static int safexcel_decrypt_xts(struct skcipher_request *req)
2032 {
2033         if (req->cryptlen < XTS_BLOCK_SIZE)
2034                 return -EINVAL;
2035         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2036                                   SAFEXCEL_DECRYPT);
2037 }
2038 
2039 struct safexcel_alg_template safexcel_alg_xts_aes = {
2040         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
2041         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS,
2042         .alg.skcipher = {
2043                 .setkey = safexcel_skcipher_aesxts_setkey,
2044                 .encrypt = safexcel_encrypt_xts,
2045                 .decrypt = safexcel_decrypt_xts,
2046                 /* XTS actually uses 2 AES keys glued together */
2047                 .min_keysize = AES_MIN_KEY_SIZE * 2,
2048                 .max_keysize = AES_MAX_KEY_SIZE * 2,
2049                 .ivsize = XTS_BLOCK_SIZE,
2050                 .base = {
2051                         .cra_name = "xts(aes)",
2052                         .cra_driver_name = "safexcel-xts-aes",
2053                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2054                         .cra_flags = CRYPTO_ALG_ASYNC |
2055                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2056                         .cra_blocksize = XTS_BLOCK_SIZE,
2057                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2058                         .cra_alignmask = 0,
2059                         .cra_init = safexcel_skcipher_aes_xts_cra_init,
2060                         .cra_exit = safexcel_skcipher_cra_exit,
2061                         .cra_module = THIS_MODULE,
2062                 },
2063         },
2064 };
2065 
2066 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
2067                                     unsigned int len)
2068 {
2069         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2070         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2071         struct safexcel_crypto_priv *priv = ctx->priv;
2072         struct crypto_aes_ctx aes;
2073         u32 hashkey[AES_BLOCK_SIZE >> 2];
2074         int ret, i;
2075 
2076         ret = aes_expandkey(&aes, key, len);
2077         if (ret) {
2078                 crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2079                 memzero_explicit(&aes, sizeof(aes));
2080                 return ret;
2081         }
2082 
2083         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2084                 for (i = 0; i < len / sizeof(u32); i++) {
2085                         if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
2086                                 ctx->base.needs_inv = true;
2087                                 break;
2088                         }
2089                 }
2090         }
2091 
2092         for (i = 0; i < len / sizeof(u32); i++)
2093                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2094 
2095         ctx->key_len = len;
2096 
2097         /* Compute hash key by encrypting zeroes with cipher key */
2098         crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK);
2099         crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) &
2100                                 CRYPTO_TFM_REQ_MASK);
2101         ret = crypto_cipher_setkey(ctx->hkaes, key, len);
2102         crypto_aead_set_flags(ctfm, crypto_cipher_get_flags(ctx->hkaes) &
2103                               CRYPTO_TFM_RES_MASK);
2104         if (ret)
2105                 return ret;
2106 
2107         memset(hashkey, 0, AES_BLOCK_SIZE);
2108         crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey);
2109 
2110         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2111                 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
2112                         if (ctx->ipad[i] != cpu_to_be32(hashkey[i])) {
2113                                 ctx->base.needs_inv = true;
2114                                 break;
2115                         }
2116                 }
2117         }
2118 
2119         for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
2120                 ctx->ipad[i] = cpu_to_be32(hashkey[i]);
2121 
2122         memzero_explicit(hashkey, AES_BLOCK_SIZE);
2123         memzero_explicit(&aes, sizeof(aes));
2124         return 0;
2125 }
2126 
2127 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm)
2128 {
2129         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2130 
2131         safexcel_aead_cra_init(tfm);
2132         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH;
2133         ctx->state_sz = GHASH_BLOCK_SIZE;
2134         ctx->xcm = EIP197_XCM_MODE_GCM;
2135         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2136 
2137         ctx->hkaes = crypto_alloc_cipher("aes", 0, 0);
2138         if (IS_ERR(ctx->hkaes))
2139                 return PTR_ERR(ctx->hkaes);
2140 
2141         return 0;
2142 }
2143 
2144 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm)
2145 {
2146         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2147 
2148         crypto_free_cipher(ctx->hkaes);
2149         safexcel_aead_cra_exit(tfm);
2150 }
2151 
2152 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm,
2153                                          unsigned int authsize)
2154 {
2155         return crypto_gcm_check_authsize(authsize);
2156 }
2157 
2158 struct safexcel_alg_template safexcel_alg_gcm = {
2159         .type = SAFEXCEL_ALG_TYPE_AEAD,
2160         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
2161         .alg.aead = {
2162                 .setkey = safexcel_aead_gcm_setkey,
2163                 .setauthsize = safexcel_aead_gcm_setauthsize,
2164                 .encrypt = safexcel_aead_encrypt,
2165                 .decrypt = safexcel_aead_decrypt,
2166                 .ivsize = GCM_AES_IV_SIZE,
2167                 .maxauthsize = GHASH_DIGEST_SIZE,
2168                 .base = {
2169                         .cra_name = "gcm(aes)",
2170                         .cra_driver_name = "safexcel-gcm-aes",
2171                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2172                         .cra_flags = CRYPTO_ALG_ASYNC |
2173                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2174                         .cra_blocksize = 1,
2175                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2176                         .cra_alignmask = 0,
2177                         .cra_init = safexcel_aead_gcm_cra_init,
2178                         .cra_exit = safexcel_aead_gcm_cra_exit,
2179                         .cra_module = THIS_MODULE,
2180                 },
2181         },
2182 };
2183 
2184 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
2185                                     unsigned int len)
2186 {
2187         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2188         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2189         struct safexcel_crypto_priv *priv = ctx->priv;
2190         struct crypto_aes_ctx aes;
2191         int ret, i;
2192 
2193         ret = aes_expandkey(&aes, key, len);
2194         if (ret) {
2195                 crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2196                 memzero_explicit(&aes, sizeof(aes));
2197                 return ret;
2198         }
2199 
2200         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2201                 for (i = 0; i < len / sizeof(u32); i++) {
2202                         if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
2203                                 ctx->base.needs_inv = true;
2204                                 break;
2205                         }
2206                 }
2207         }
2208 
2209         for (i = 0; i < len / sizeof(u32); i++) {
2210                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2211                 ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] =
2212                         cpu_to_be32(aes.key_enc[i]);
2213         }
2214 
2215         ctx->key_len = len;
2216         ctx->state_sz = 2 * AES_BLOCK_SIZE + len;
2217 
2218         if (len == AES_KEYSIZE_192)
2219                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2220         else if (len == AES_KEYSIZE_256)
2221                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2222         else
2223                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2224 
2225         memzero_explicit(&aes, sizeof(aes));
2226         return 0;
2227 }
2228 
2229 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm)
2230 {
2231         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2232 
2233         safexcel_aead_cra_init(tfm);
2234         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2235         ctx->state_sz = 3 * AES_BLOCK_SIZE;
2236         ctx->xcm = EIP197_XCM_MODE_CCM;
2237         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2238         return 0;
2239 }
2240 
2241 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm,
2242                                          unsigned int authsize)
2243 {
2244         /* Borrowed from crypto/ccm.c */
2245         switch (authsize) {
2246         case 4:
2247         case 6:
2248         case 8:
2249         case 10:
2250         case 12:
2251         case 14:
2252         case 16:
2253                 break;
2254         default:
2255                 return -EINVAL;
2256         }
2257 
2258         return 0;
2259 }
2260 
2261 static int safexcel_ccm_encrypt(struct aead_request *req)
2262 {
2263         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2264 
2265         if (req->iv[0] < 1 || req->iv[0] > 7)
2266                 return -EINVAL;
2267 
2268         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
2269 }
2270 
2271 static int safexcel_ccm_decrypt(struct aead_request *req)
2272 {
2273         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2274 
2275         if (req->iv[0] < 1 || req->iv[0] > 7)
2276                 return -EINVAL;
2277 
2278         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
2279 }
2280 
2281 struct safexcel_alg_template safexcel_alg_ccm = {
2282         .type = SAFEXCEL_ALG_TYPE_AEAD,
2283         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
2284         .alg.aead = {
2285                 .setkey = safexcel_aead_ccm_setkey,
2286                 .setauthsize = safexcel_aead_ccm_setauthsize,
2287                 .encrypt = safexcel_ccm_encrypt,
2288                 .decrypt = safexcel_ccm_decrypt,
2289                 .ivsize = AES_BLOCK_SIZE,
2290                 .maxauthsize = AES_BLOCK_SIZE,
2291                 .base = {
2292                         .cra_name = "ccm(aes)",
2293                         .cra_driver_name = "safexcel-ccm-aes",
2294                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2295                         .cra_flags = CRYPTO_ALG_ASYNC |
2296                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2297                         .cra_blocksize = 1,
2298                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2299                         .cra_alignmask = 0,
2300                         .cra_init = safexcel_aead_ccm_cra_init,
2301                         .cra_exit = safexcel_aead_cra_exit,
2302                         .cra_module = THIS_MODULE,
2303                 },
2304         },
2305 };