drivers/crypto/ccree/cc

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
valid_assoclen
cc_aead_exit
cc_get_aead_hash_len
cc_aead_init
cc_aead_complete
xcbc_setkey
hmac_setkey
validate_keys_sizes
cc_get_plain_hmac_key
cc_aead_setkey
cc_des3_aead_setkey
cc_rfc4309_ccm_setkey
cc_aead_setauthsize
cc_rfc4309_ccm_setauthsize
cc_ccm_setauthsize
cc_set_assoc_desc
cc_proc_authen_desc
cc_proc_cipher_desc
cc_proc_digest_desc
cc_set_cipher_desc
cc_proc_cipher
cc_set_hmac_desc
cc_set_xcbc_desc
cc_proc_header_desc
cc_proc_scheme_desc
cc_mlli_to_sram
cc_get_data_flow
cc_hmac_authenc
cc_xcbc_authenc
validate_data_size
format_ccm_a0
set_msg_len
cc_ccm
config_ccm_adata
cc_proc_rfc4309_ccm
cc_set_ghash_desc
cc_set_gctr_desc
cc_proc_gcm_result
cc_gcm
config_gcm_context
cc_proc_rfc4_gcm
cc_proc_aead
cc_aead_encrypt
cc_rfc4309_ccm_encrypt
cc_aead_decrypt
cc_rfc4309_ccm_decrypt
cc_rfc4106_gcm_setkey
cc_rfc4543_gcm_setkey
cc_gcm_setauthsize
cc_rfc4106_gcm_setauthsize
cc_rfc4543_gcm_setauthsize
cc_rfc4106_gcm_encrypt
cc_rfc4543_gcm_encrypt
cc_rfc4106_gcm_decrypt
cc_rfc4543_gcm_decrypt
cc_create_aead_alg
cc_aead_free
cc_aead_alloc
   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
   3 
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <crypto/algapi.h>
   7 #include <crypto/internal/aead.h>
   8 #include <crypto/authenc.h>
   9 #include <crypto/internal/des.h>
  10 #include <linux/rtnetlink.h>
  11 #include "cc_driver.h"
  12 #include "cc_buffer_mgr.h"
  13 #include "cc_aead.h"
  14 #include "cc_request_mgr.h"
  15 #include "cc_hash.h"
  16 #include "cc_sram_mgr.h"
  17 
  18 #define template_aead   template_u.aead
  19 
  20 #define MAX_AEAD_SETKEY_SEQ 12
  21 #define MAX_AEAD_PROCESS_SEQ 23
  22 
  23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)
  24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)
  25 
  26 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE
  27 
  28 struct cc_aead_handle {
  29         cc_sram_addr_t sram_workspace_addr;
  30         struct list_head aead_list;
  31 };
  32 
  33 struct cc_hmac_s {
  34         u8 *padded_authkey;
  35         u8 *ipad_opad; /* IPAD, OPAD*/
  36         dma_addr_t padded_authkey_dma_addr;
  37         dma_addr_t ipad_opad_dma_addr;
  38 };
  39 
  40 struct cc_xcbc_s {
  41         u8 *xcbc_keys; /* K1,K2,K3 */
  42         dma_addr_t xcbc_keys_dma_addr;
  43 };
  44 
  45 struct cc_aead_ctx {
  46         struct cc_drvdata *drvdata;
  47         u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */
  48         u8 *enckey;
  49         dma_addr_t enckey_dma_addr;
  50         union {
  51                 struct cc_hmac_s hmac;
  52                 struct cc_xcbc_s xcbc;
  53         } auth_state;
  54         unsigned int enc_keylen;
  55         unsigned int auth_keylen;
  56         unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */
  57         unsigned int hash_len;
  58         enum drv_cipher_mode cipher_mode;
  59         enum cc_flow_mode flow_mode;
  60         enum drv_hash_mode auth_mode;
  61 };
  62 
  63 static inline bool valid_assoclen(struct aead_request *req)
  64 {
  65         return ((req->assoclen == 16) || (req->assoclen == 20));
  66 }
  67 
  68 static void cc_aead_exit(struct crypto_aead *tfm)
  69 {
  70         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
  71         struct device *dev = drvdata_to_dev(ctx->drvdata);
  72 
  73         dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),
  74                 crypto_tfm_alg_name(&tfm->base));
  75 
  76         /* Unmap enckey buffer */
  77         if (ctx->enckey) {
  78                 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,
  79                                   ctx->enckey_dma_addr);
  80                 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",
  81                         &ctx->enckey_dma_addr);
  82                 ctx->enckey_dma_addr = 0;
  83                 ctx->enckey = NULL;
  84         }
  85 
  86         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
  87                 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
  88 
  89                 if (xcbc->xcbc_keys) {
  90                         dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
  91                                           xcbc->xcbc_keys,
  92                                           xcbc->xcbc_keys_dma_addr);
  93                 }
  94                 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
  95                         &xcbc->xcbc_keys_dma_addr);
  96                 xcbc->xcbc_keys_dma_addr = 0;
  97                 xcbc->xcbc_keys = NULL;
  98         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */
  99                 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 100 
 101                 if (hmac->ipad_opad) {
 102                         dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
 103                                           hmac->ipad_opad,
 104                                           hmac->ipad_opad_dma_addr);
 105                         dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
 106                                 &hmac->ipad_opad_dma_addr);
 107                         hmac->ipad_opad_dma_addr = 0;
 108                         hmac->ipad_opad = NULL;
 109                 }
 110                 if (hmac->padded_authkey) {
 111                         dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
 112                                           hmac->padded_authkey,
 113                                           hmac->padded_authkey_dma_addr);
 114                         dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
 115                                 &hmac->padded_authkey_dma_addr);
 116                         hmac->padded_authkey_dma_addr = 0;
 117                         hmac->padded_authkey = NULL;
 118                 }
 119         }
 120 }
 121 
 122 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm)
 123 {
 124         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 125 
 126         return cc_get_default_hash_len(ctx->drvdata);
 127 }
 128 
 129 static int cc_aead_init(struct crypto_aead *tfm)
 130 {
 131         struct aead_alg *alg = crypto_aead_alg(tfm);
 132         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 133         struct cc_crypto_alg *cc_alg =
 134                         container_of(alg, struct cc_crypto_alg, aead_alg);
 135         struct device *dev = drvdata_to_dev(cc_alg->drvdata);
 136 
 137         dev_dbg(dev, "Initializing context @%p for %s\n", ctx,
 138                 crypto_tfm_alg_name(&tfm->base));
 139 
 140         /* Initialize modes in instance */
 141         ctx->cipher_mode = cc_alg->cipher_mode;
 142         ctx->flow_mode = cc_alg->flow_mode;
 143         ctx->auth_mode = cc_alg->auth_mode;
 144         ctx->drvdata = cc_alg->drvdata;
 145         crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));
 146 
 147         /* Allocate key buffer, cache line aligned */
 148         ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
 149                                          &ctx->enckey_dma_addr, GFP_KERNEL);
 150         if (!ctx->enckey) {
 151                 dev_err(dev, "Failed allocating key buffer\n");
 152                 goto init_failed;
 153         }
 154         dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",
 155                 ctx->enckey);
 156 
 157         /* Set default authlen value */
 158 
 159         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
 160                 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
 161                 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
 162 
 163                 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */
 164                 /* (and temporary for user key - up to 256b) */
 165                 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
 166                                                      &xcbc->xcbc_keys_dma_addr,
 167                                                      GFP_KERNEL);
 168                 if (!xcbc->xcbc_keys) {
 169                         dev_err(dev, "Failed allocating buffer for XCBC keys\n");
 170                         goto init_failed;
 171                 }
 172         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */
 173                 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 174                 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
 175                 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
 176 
 177                 /* Allocate dma-coherent buffer for IPAD + OPAD */
 178                 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
 179                                                      &hmac->ipad_opad_dma_addr,
 180                                                      GFP_KERNEL);
 181 
 182                 if (!hmac->ipad_opad) {
 183                         dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");
 184                         goto init_failed;
 185                 }
 186 
 187                 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",
 188                         hmac->ipad_opad);
 189 
 190                 hmac->padded_authkey = dma_alloc_coherent(dev,
 191                                                           MAX_HMAC_BLOCK_SIZE,
 192                                                           pkey_dma,
 193                                                           GFP_KERNEL);
 194 
 195                 if (!hmac->padded_authkey) {
 196                         dev_err(dev, "failed to allocate padded_authkey\n");
 197                         goto init_failed;
 198                 }
 199         } else {
 200                 ctx->auth_state.hmac.ipad_opad = NULL;
 201                 ctx->auth_state.hmac.padded_authkey = NULL;
 202         }
 203         ctx->hash_len = cc_get_aead_hash_len(tfm);
 204 
 205         return 0;
 206 
 207 init_failed:
 208         cc_aead_exit(tfm);
 209         return -ENOMEM;
 210 }
 211 
 212 static void cc_aead_complete(struct device *dev, void *cc_req, int err)
 213 {
 214         struct aead_request *areq = (struct aead_request *)cc_req;
 215         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 216         struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
 217         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 218 
 219         /* BACKLOG notification */
 220         if (err == -EINPROGRESS)
 221                 goto done;
 222 
 223         cc_unmap_aead_request(dev, areq);
 224 
 225         /* Restore ordinary iv pointer */
 226         areq->iv = areq_ctx->backup_iv;
 227 
 228         if (err)
 229                 goto done;
 230 
 231         if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
 232                 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
 233                            ctx->authsize) != 0) {
 234                         dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
 235                                 ctx->authsize, ctx->cipher_mode);
 236                         /* In case of payload authentication failure, MUST NOT
 237                          * revealed the decrypted message --> zero its memory.
 238                          */
 239                         sg_zero_buffer(areq->dst, sg_nents(areq->dst),
 240                                        areq->cryptlen, areq->assoclen);
 241                         err = -EBADMSG;
 242                 }
 243         /*ENCRYPT*/
 244         } else if (areq_ctx->is_icv_fragmented) {
 245                 u32 skip = areq->cryptlen + areq_ctx->dst_offset;
 246 
 247                 cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl,
 248                                    skip, (skip + ctx->authsize),
 249                                    CC_SG_FROM_BUF);
 250         }
 251 done:
 252         aead_request_complete(areq, err);
 253 }
 254 
 255 static unsigned int xcbc_setkey(struct cc_hw_desc *desc,
 256                                 struct cc_aead_ctx *ctx)
 257 {
 258         /* Load the AES key */
 259         hw_desc_init(&desc[0]);
 260         /* We are using for the source/user key the same buffer
 261          * as for the output keys, * because after this key loading it
 262          * is not needed anymore
 263          */
 264         set_din_type(&desc[0], DMA_DLLI,
 265                      ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,
 266                      NS_BIT);
 267         set_cipher_mode(&desc[0], DRV_CIPHER_ECB);
 268         set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);
 269         set_key_size_aes(&desc[0], ctx->auth_keylen);
 270         set_flow_mode(&desc[0], S_DIN_to_AES);
 271         set_setup_mode(&desc[0], SETUP_LOAD_KEY0);
 272 
 273         hw_desc_init(&desc[1]);
 274         set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
 275         set_flow_mode(&desc[1], DIN_AES_DOUT);
 276         set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,
 277                       AES_KEYSIZE_128, NS_BIT, 0);
 278 
 279         hw_desc_init(&desc[2]);
 280         set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
 281         set_flow_mode(&desc[2], DIN_AES_DOUT);
 282         set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
 283                                          + AES_KEYSIZE_128),
 284                               AES_KEYSIZE_128, NS_BIT, 0);
 285 
 286         hw_desc_init(&desc[3]);
 287         set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
 288         set_flow_mode(&desc[3], DIN_AES_DOUT);
 289         set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
 290                                           + 2 * AES_KEYSIZE_128),
 291                               AES_KEYSIZE_128, NS_BIT, 0);
 292 
 293         return 4;
 294 }
 295 
 296 static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx)
 297 {
 298         unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
 299         unsigned int digest_ofs = 0;
 300         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 301                         DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 302         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
 303                         CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
 304         struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 305 
 306         unsigned int idx = 0;
 307         int i;
 308 
 309         /* calc derived HMAC key */
 310         for (i = 0; i < 2; i++) {
 311                 /* Load hash initial state */
 312                 hw_desc_init(&desc[idx]);
 313                 set_cipher_mode(&desc[idx], hash_mode);
 314                 set_din_sram(&desc[idx],
 315                              cc_larval_digest_addr(ctx->drvdata,
 316                                                    ctx->auth_mode),
 317                              digest_size);
 318                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 319                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 320                 idx++;
 321 
 322                 /* Load the hash current length*/
 323                 hw_desc_init(&desc[idx]);
 324                 set_cipher_mode(&desc[idx], hash_mode);
 325                 set_din_const(&desc[idx], 0, ctx->hash_len);
 326                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 327                 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 328                 idx++;
 329 
 330                 /* Prepare ipad key */
 331                 hw_desc_init(&desc[idx]);
 332                 set_xor_val(&desc[idx], hmac_pad_const[i]);
 333                 set_cipher_mode(&desc[idx], hash_mode);
 334                 set_flow_mode(&desc[idx], S_DIN_to_HASH);
 335                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
 336                 idx++;
 337 
 338                 /* Perform HASH update */
 339                 hw_desc_init(&desc[idx]);
 340                 set_din_type(&desc[idx], DMA_DLLI,
 341                              hmac->padded_authkey_dma_addr,
 342                              SHA256_BLOCK_SIZE, NS_BIT);
 343                 set_cipher_mode(&desc[idx], hash_mode);
 344                 set_xor_active(&desc[idx]);
 345                 set_flow_mode(&desc[idx], DIN_HASH);
 346                 idx++;
 347 
 348                 /* Get the digset */
 349                 hw_desc_init(&desc[idx]);
 350                 set_cipher_mode(&desc[idx], hash_mode);
 351                 set_dout_dlli(&desc[idx],
 352                               (hmac->ipad_opad_dma_addr + digest_ofs),
 353                               digest_size, NS_BIT, 0);
 354                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 355                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 356                 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 357                 idx++;
 358 
 359                 digest_ofs += digest_size;
 360         }
 361 
 362         return idx;
 363 }
 364 
 365 static int validate_keys_sizes(struct cc_aead_ctx *ctx)
 366 {
 367         struct device *dev = drvdata_to_dev(ctx->drvdata);
 368 
 369         dev_dbg(dev, "enc_keylen=%u  authkeylen=%u\n",
 370                 ctx->enc_keylen, ctx->auth_keylen);
 371 
 372         switch (ctx->auth_mode) {
 373         case DRV_HASH_SHA1:
 374         case DRV_HASH_SHA256:
 375                 break;
 376         case DRV_HASH_XCBC_MAC:
 377                 if (ctx->auth_keylen != AES_KEYSIZE_128 &&
 378                     ctx->auth_keylen != AES_KEYSIZE_192 &&
 379                     ctx->auth_keylen != AES_KEYSIZE_256)
 380                         return -ENOTSUPP;
 381                 break;
 382         case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */
 383                 if (ctx->auth_keylen > 0)
 384                         return -EINVAL;
 385                 break;
 386         default:
 387                 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);
 388                 return -EINVAL;
 389         }
 390         /* Check cipher key size */
 391         if (ctx->flow_mode == S_DIN_to_DES) {
 392                 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
 393                         dev_err(dev, "Invalid cipher(3DES) key size: %u\n",
 394                                 ctx->enc_keylen);
 395                         return -EINVAL;
 396                 }
 397         } else { /* Default assumed to be AES ciphers */
 398                 if (ctx->enc_keylen != AES_KEYSIZE_128 &&
 399                     ctx->enc_keylen != AES_KEYSIZE_192 &&
 400                     ctx->enc_keylen != AES_KEYSIZE_256) {
 401                         dev_err(dev, "Invalid cipher(AES) key size: %u\n",
 402                                 ctx->enc_keylen);
 403                         return -EINVAL;
 404                 }
 405         }
 406 
 407         return 0; /* All tests of keys sizes passed */
 408 }
 409 
 410 /* This function prepers the user key so it can pass to the hmac processing
 411  * (copy to intenral buffer or hash in case of key longer than block
 412  */
 413 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey,
 414                                  unsigned int keylen)
 415 {
 416         dma_addr_t key_dma_addr = 0;
 417         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 418         struct device *dev = drvdata_to_dev(ctx->drvdata);
 419         u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode);
 420         struct cc_crypto_req cc_req = {};
 421         unsigned int blocksize;
 422         unsigned int digestsize;
 423         unsigned int hashmode;
 424         unsigned int idx = 0;
 425         int rc = 0;
 426         u8 *key = NULL;
 427         struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
 428         dma_addr_t padded_authkey_dma_addr =
 429                 ctx->auth_state.hmac.padded_authkey_dma_addr;
 430 
 431         switch (ctx->auth_mode) { /* auth_key required and >0 */
 432         case DRV_HASH_SHA1:
 433                 blocksize = SHA1_BLOCK_SIZE;
 434                 digestsize = SHA1_DIGEST_SIZE;
 435                 hashmode = DRV_HASH_HW_SHA1;
 436                 break;
 437         case DRV_HASH_SHA256:
 438         default:
 439                 blocksize = SHA256_BLOCK_SIZE;
 440                 digestsize = SHA256_DIGEST_SIZE;
 441                 hashmode = DRV_HASH_HW_SHA256;
 442         }
 443 
 444         if (keylen != 0) {
 445 
 446                 key = kmemdup(authkey, keylen, GFP_KERNEL);
 447                 if (!key)
 448                         return -ENOMEM;
 449 
 450                 key_dma_addr = dma_map_single(dev, (void *)key, keylen,
 451                                               DMA_TO_DEVICE);
 452                 if (dma_mapping_error(dev, key_dma_addr)) {
 453                         dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
 454                                 key, keylen);
 455                         kzfree(key);
 456                         return -ENOMEM;
 457                 }
 458                 if (keylen > blocksize) {
 459                         /* Load hash initial state */
 460                         hw_desc_init(&desc[idx]);
 461                         set_cipher_mode(&desc[idx], hashmode);
 462                         set_din_sram(&desc[idx], larval_addr, digestsize);
 463                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 464                         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 465                         idx++;
 466 
 467                         /* Load the hash current length*/
 468                         hw_desc_init(&desc[idx]);
 469                         set_cipher_mode(&desc[idx], hashmode);
 470                         set_din_const(&desc[idx], 0, ctx->hash_len);
 471                         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
 472                         set_flow_mode(&desc[idx], S_DIN_to_HASH);
 473                         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 474                         idx++;
 475 
 476                         hw_desc_init(&desc[idx]);
 477                         set_din_type(&desc[idx], DMA_DLLI,
 478                                      key_dma_addr, keylen, NS_BIT);
 479                         set_flow_mode(&desc[idx], DIN_HASH);
 480                         idx++;
 481 
 482                         /* Get hashed key */
 483                         hw_desc_init(&desc[idx]);
 484                         set_cipher_mode(&desc[idx], hashmode);
 485                         set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 486                                       digestsize, NS_BIT, 0);
 487                         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 488                         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 489                         set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 490                         set_cipher_config0(&desc[idx],
 491                                            HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 492                         idx++;
 493 
 494                         hw_desc_init(&desc[idx]);
 495                         set_din_const(&desc[idx], 0, (blocksize - digestsize));
 496                         set_flow_mode(&desc[idx], BYPASS);
 497                         set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +
 498                                       digestsize), (blocksize - digestsize),
 499                                       NS_BIT, 0);
 500                         idx++;
 501                 } else {
 502                         hw_desc_init(&desc[idx]);
 503                         set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
 504                                      keylen, NS_BIT);
 505                         set_flow_mode(&desc[idx], BYPASS);
 506                         set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 507                                       keylen, NS_BIT, 0);
 508                         idx++;
 509 
 510                         if ((blocksize - keylen) != 0) {
 511                                 hw_desc_init(&desc[idx]);
 512                                 set_din_const(&desc[idx], 0,
 513                                               (blocksize - keylen));
 514                                 set_flow_mode(&desc[idx], BYPASS);
 515                                 set_dout_dlli(&desc[idx],
 516                                               (padded_authkey_dma_addr +
 517                                                keylen),
 518                                               (blocksize - keylen), NS_BIT, 0);
 519                                 idx++;
 520                         }
 521                 }
 522         } else {
 523                 hw_desc_init(&desc[idx]);
 524                 set_din_const(&desc[idx], 0, (blocksize - keylen));
 525                 set_flow_mode(&desc[idx], BYPASS);
 526                 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
 527                               blocksize, NS_BIT, 0);
 528                 idx++;
 529         }
 530 
 531         rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
 532         if (rc)
 533                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 534 
 535         if (key_dma_addr)
 536                 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);
 537 
 538         kzfree(key);
 539 
 540         return rc;
 541 }
 542 
 543 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 544                           unsigned int keylen)
 545 {
 546         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 547         struct cc_crypto_req cc_req = {};
 548         struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
 549         unsigned int seq_len = 0;
 550         struct device *dev = drvdata_to_dev(ctx->drvdata);
 551         const u8 *enckey, *authkey;
 552         int rc;
 553 
 554         dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",
 555                 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
 556 
 557         /* STAT_PHASE_0: Init and sanity checks */
 558 
 559         if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */
 560                 struct crypto_authenc_keys keys;
 561 
 562                 rc = crypto_authenc_extractkeys(&keys, key, keylen);
 563                 if (rc)
 564                         goto badkey;
 565                 enckey = keys.enckey;
 566                 authkey = keys.authkey;
 567                 ctx->enc_keylen = keys.enckeylen;
 568                 ctx->auth_keylen = keys.authkeylen;
 569 
 570                 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
 571                         /* the nonce is stored in bytes at end of key */
 572                         rc = -EINVAL;
 573                         if (ctx->enc_keylen <
 574                             (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))
 575                                 goto badkey;
 576                         /* Copy nonce from last 4 bytes in CTR key to
 577                          *  first 4 bytes in CTR IV
 578                          */
 579                         memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen -
 580                                CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE);
 581                         /* Set CTR key size */
 582                         ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;
 583                 }
 584         } else { /* non-authenc - has just one key */
 585                 enckey = key;
 586                 authkey = NULL;
 587                 ctx->enc_keylen = keylen;
 588                 ctx->auth_keylen = 0;
 589         }
 590 
 591         rc = validate_keys_sizes(ctx);
 592         if (rc)
 593                 goto badkey;
 594 
 595         /* STAT_PHASE_1: Copy key to ctx */
 596 
 597         /* Get key material */
 598         memcpy(ctx->enckey, enckey, ctx->enc_keylen);
 599         if (ctx->enc_keylen == 24)
 600                 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);
 601         if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 602                 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey,
 603                        ctx->auth_keylen);
 604         } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */
 605                 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen);
 606                 if (rc)
 607                         goto badkey;
 608         }
 609 
 610         /* STAT_PHASE_2: Create sequence */
 611 
 612         switch (ctx->auth_mode) {
 613         case DRV_HASH_SHA1:
 614         case DRV_HASH_SHA256:
 615                 seq_len = hmac_setkey(desc, ctx);
 616                 break;
 617         case DRV_HASH_XCBC_MAC:
 618                 seq_len = xcbc_setkey(desc, ctx);
 619                 break;
 620         case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */
 621                 break; /* No auth. key setup */
 622         default:
 623                 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
 624                 rc = -ENOTSUPP;
 625                 goto badkey;
 626         }
 627 
 628         /* STAT_PHASE_3: Submit sequence to HW */
 629 
 630         if (seq_len > 0) { /* For CCM there is no sequence to setup the key */
 631                 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);
 632                 if (rc) {
 633                         dev_err(dev, "send_request() failed (rc=%d)\n", rc);
 634                         goto setkey_error;
 635                 }
 636         }
 637 
 638         /* Update STAT_PHASE_3 */
 639         return rc;
 640 
 641 badkey:
 642         crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 643 
 644 setkey_error:
 645         return rc;
 646 }
 647 
 648 static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 649                                unsigned int keylen)
 650 {
 651         struct crypto_authenc_keys keys;
 652         int err;
 653 
 654         err = crypto_authenc_extractkeys(&keys, key, keylen);
 655         if (unlikely(err))
 656                 return err;
 657 
 658         err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
 659               cc_aead_setkey(aead, key, keylen);
 660 
 661         memzero_explicit(&keys, sizeof(keys));
 662         return err;
 663 }
 664 
 665 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
 666                                  unsigned int keylen)
 667 {
 668         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 669 
 670         if (keylen < 3)
 671                 return -EINVAL;
 672 
 673         keylen -= 3;
 674         memcpy(ctx->ctr_nonce, key + keylen, 3);
 675 
 676         return cc_aead_setkey(tfm, key, keylen);
 677 }
 678 
 679 static int cc_aead_setauthsize(struct crypto_aead *authenc,
 680                                unsigned int authsize)
 681 {
 682         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
 683         struct device *dev = drvdata_to_dev(ctx->drvdata);
 684 
 685         /* Unsupported auth. sizes */
 686         if (authsize == 0 ||
 687             authsize > crypto_aead_maxauthsize(authenc)) {
 688                 return -ENOTSUPP;
 689         }
 690 
 691         ctx->authsize = authsize;
 692         dev_dbg(dev, "authlen=%d\n", ctx->authsize);
 693 
 694         return 0;
 695 }
 696 
 697 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
 698                                       unsigned int authsize)
 699 {
 700         switch (authsize) {
 701         case 8:
 702         case 12:
 703         case 16:
 704                 break;
 705         default:
 706                 return -EINVAL;
 707         }
 708 
 709         return cc_aead_setauthsize(authenc, authsize);
 710 }
 711 
 712 static int cc_ccm_setauthsize(struct crypto_aead *authenc,
 713                               unsigned int authsize)
 714 {
 715         switch (authsize) {
 716         case 4:
 717         case 6:
 718         case 8:
 719         case 10:
 720         case 12:
 721         case 14:
 722         case 16:
 723                 break;
 724         default:
 725                 return -EINVAL;
 726         }
 727 
 728         return cc_aead_setauthsize(authenc, authsize);
 729 }
 730 
 731 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
 732                               struct cc_hw_desc desc[], unsigned int *seq_size)
 733 {
 734         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 735         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 736         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 737         enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
 738         unsigned int idx = *seq_size;
 739         struct device *dev = drvdata_to_dev(ctx->drvdata);
 740 
 741         switch (assoc_dma_type) {
 742         case CC_DMA_BUF_DLLI:
 743                 dev_dbg(dev, "ASSOC buffer type DLLI\n");
 744                 hw_desc_init(&desc[idx]);
 745                 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),
 746                              areq_ctx->assoclen, NS_BIT);
 747                 set_flow_mode(&desc[idx], flow_mode);
 748                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
 749                     areq_ctx->cryptlen > 0)
 750                         set_din_not_last_indication(&desc[idx]);
 751                 break;
 752         case CC_DMA_BUF_MLLI:
 753                 dev_dbg(dev, "ASSOC buffer type MLLI\n");
 754                 hw_desc_init(&desc[idx]);
 755                 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,
 756                              areq_ctx->assoc.mlli_nents, NS_BIT);
 757                 set_flow_mode(&desc[idx], flow_mode);
 758                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
 759                     areq_ctx->cryptlen > 0)
 760                         set_din_not_last_indication(&desc[idx]);
 761                 break;
 762         case CC_DMA_BUF_NULL:
 763         default:
 764                 dev_err(dev, "Invalid ASSOC buffer type\n");
 765         }
 766 
 767         *seq_size = (++idx);
 768 }
 769 
 770 static void cc_proc_authen_desc(struct aead_request *areq,
 771                                 unsigned int flow_mode,
 772                                 struct cc_hw_desc desc[],
 773                                 unsigned int *seq_size, int direct)
 774 {
 775         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 776         enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
 777         unsigned int idx = *seq_size;
 778         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 779         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 780         struct device *dev = drvdata_to_dev(ctx->drvdata);
 781 
 782         switch (data_dma_type) {
 783         case CC_DMA_BUF_DLLI:
 784         {
 785                 struct scatterlist *cipher =
 786                         (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
 787                         areq_ctx->dst_sgl : areq_ctx->src_sgl;
 788 
 789                 unsigned int offset =
 790                         (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
 791                         areq_ctx->dst_offset : areq_ctx->src_offset;
 792                 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");
 793                 hw_desc_init(&desc[idx]);
 794                 set_din_type(&desc[idx], DMA_DLLI,
 795                              (sg_dma_address(cipher) + offset),
 796                              areq_ctx->cryptlen, NS_BIT);
 797                 set_flow_mode(&desc[idx], flow_mode);
 798                 break;
 799         }
 800         case CC_DMA_BUF_MLLI:
 801         {
 802                 /* DOUBLE-PASS flow (as default)
 803                  * assoc. + iv + data -compact in one table
 804                  * if assoclen is ZERO only IV perform
 805                  */
 806                 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr;
 807                 u32 mlli_nents = areq_ctx->assoc.mlli_nents;
 808 
 809                 if (areq_ctx->is_single_pass) {
 810                         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 811                                 mlli_addr = areq_ctx->dst.sram_addr;
 812                                 mlli_nents = areq_ctx->dst.mlli_nents;
 813                         } else {
 814                                 mlli_addr = areq_ctx->src.sram_addr;
 815                                 mlli_nents = areq_ctx->src.mlli_nents;
 816                         }
 817                 }
 818 
 819                 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");
 820                 hw_desc_init(&desc[idx]);
 821                 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,
 822                              NS_BIT);
 823                 set_flow_mode(&desc[idx], flow_mode);
 824                 break;
 825         }
 826         case CC_DMA_BUF_NULL:
 827         default:
 828                 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");
 829         }
 830 
 831         *seq_size = (++idx);
 832 }
 833 
 834 static void cc_proc_cipher_desc(struct aead_request *areq,
 835                                 unsigned int flow_mode,
 836                                 struct cc_hw_desc desc[],
 837                                 unsigned int *seq_size)
 838 {
 839         unsigned int idx = *seq_size;
 840         struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
 841         enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
 842         struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
 843         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 844         struct device *dev = drvdata_to_dev(ctx->drvdata);
 845 
 846         if (areq_ctx->cryptlen == 0)
 847                 return; /*null processing*/
 848 
 849         switch (data_dma_type) {
 850         case CC_DMA_BUF_DLLI:
 851                 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");
 852                 hw_desc_init(&desc[idx]);
 853                 set_din_type(&desc[idx], DMA_DLLI,
 854                              (sg_dma_address(areq_ctx->src_sgl) +
 855                               areq_ctx->src_offset), areq_ctx->cryptlen,
 856                               NS_BIT);
 857                 set_dout_dlli(&desc[idx],
 858                               (sg_dma_address(areq_ctx->dst_sgl) +
 859                                areq_ctx->dst_offset),
 860                               areq_ctx->cryptlen, NS_BIT, 0);
 861                 set_flow_mode(&desc[idx], flow_mode);
 862                 break;
 863         case CC_DMA_BUF_MLLI:
 864                 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");
 865                 hw_desc_init(&desc[idx]);
 866                 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,
 867                              areq_ctx->src.mlli_nents, NS_BIT);
 868                 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,
 869                               areq_ctx->dst.mlli_nents, NS_BIT, 0);
 870                 set_flow_mode(&desc[idx], flow_mode);
 871                 break;
 872         case CC_DMA_BUF_NULL:
 873         default:
 874                 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");
 875         }
 876 
 877         *seq_size = (++idx);
 878 }
 879 
 880 static void cc_proc_digest_desc(struct aead_request *req,
 881                                 struct cc_hw_desc desc[],
 882                                 unsigned int *seq_size)
 883 {
 884         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 885         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 886         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 887         unsigned int idx = *seq_size;
 888         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 889                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 890         int direct = req_ctx->gen_ctx.op_type;
 891 
 892         /* Get final ICV result */
 893         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 894                 hw_desc_init(&desc[idx]);
 895                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 896                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 897                 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,
 898                               NS_BIT, 1);
 899                 set_queue_last_ind(ctx->drvdata, &desc[idx]);
 900                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 901                         set_aes_not_hash_mode(&desc[idx]);
 902                         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
 903                 } else {
 904                         set_cipher_config0(&desc[idx],
 905                                            HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 906                         set_cipher_mode(&desc[idx], hash_mode);
 907                 }
 908         } else { /*Decrypt*/
 909                 /* Get ICV out from hardware */
 910                 hw_desc_init(&desc[idx]);
 911                 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 912                 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 913                 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,
 914                               ctx->authsize, NS_BIT, 1);
 915                 set_queue_last_ind(ctx->drvdata, &desc[idx]);
 916                 set_cipher_config0(&desc[idx],
 917                                    HASH_DIGEST_RESULT_LITTLE_ENDIAN);
 918                 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
 919                 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
 920                         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
 921                         set_aes_not_hash_mode(&desc[idx]);
 922                 } else {
 923                         set_cipher_mode(&desc[idx], hash_mode);
 924                 }
 925         }
 926 
 927         *seq_size = (++idx);
 928 }
 929 
 930 static void cc_set_cipher_desc(struct aead_request *req,
 931                                struct cc_hw_desc desc[],
 932                                unsigned int *seq_size)
 933 {
 934         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 935         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
 936         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 937         unsigned int hw_iv_size = req_ctx->hw_iv_size;
 938         unsigned int idx = *seq_size;
 939         int direct = req_ctx->gen_ctx.op_type;
 940 
 941         /* Setup cipher state */
 942         hw_desc_init(&desc[idx]);
 943         set_cipher_config0(&desc[idx], direct);
 944         set_flow_mode(&desc[idx], ctx->flow_mode);
 945         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,
 946                      hw_iv_size, NS_BIT);
 947         if (ctx->cipher_mode == DRV_CIPHER_CTR)
 948                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
 949         else
 950                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
 951         set_cipher_mode(&desc[idx], ctx->cipher_mode);
 952         idx++;
 953 
 954         /* Setup enc. key */
 955         hw_desc_init(&desc[idx]);
 956         set_cipher_config0(&desc[idx], direct);
 957         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 958         set_flow_mode(&desc[idx], ctx->flow_mode);
 959         if (ctx->flow_mode == S_DIN_to_AES) {
 960                 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
 961                              ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
 962                               ctx->enc_keylen), NS_BIT);
 963                 set_key_size_aes(&desc[idx], ctx->enc_keylen);
 964         } else {
 965                 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
 966                              ctx->enc_keylen, NS_BIT);
 967                 set_key_size_des(&desc[idx], ctx->enc_keylen);
 968         }
 969         set_cipher_mode(&desc[idx], ctx->cipher_mode);
 970         idx++;
 971 
 972         *seq_size = idx;
 973 }
 974 
 975 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
 976                            unsigned int *seq_size, unsigned int data_flow_mode)
 977 {
 978         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
 979         int direct = req_ctx->gen_ctx.op_type;
 980         unsigned int idx = *seq_size;
 981 
 982         if (req_ctx->cryptlen == 0)
 983                 return; /*null processing*/
 984 
 985         cc_set_cipher_desc(req, desc, &idx);
 986         cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);
 987         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
 988                 /* We must wait for DMA to write all cipher */
 989                 hw_desc_init(&desc[idx]);
 990                 set_din_no_dma(&desc[idx], 0, 0xfffff0);
 991                 set_dout_no_dma(&desc[idx], 0, 0, 1);
 992                 idx++;
 993         }
 994 
 995         *seq_size = idx;
 996 }
 997 
 998 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],
 999                              unsigned int *seq_size)
1000 {
1001         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1002         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1003         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1004                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1005         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1006                                 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1007         unsigned int idx = *seq_size;
1008 
1009         /* Loading hash ipad xor key state */
1010         hw_desc_init(&desc[idx]);
1011         set_cipher_mode(&desc[idx], hash_mode);
1012         set_din_type(&desc[idx], DMA_DLLI,
1013                      ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,
1014                      NS_BIT);
1015         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1016         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1017         idx++;
1018 
1019         /* Load init. digest len (64 bytes) */
1020         hw_desc_init(&desc[idx]);
1021         set_cipher_mode(&desc[idx], hash_mode);
1022         set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1023                      ctx->hash_len);
1024         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1025         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1026         idx++;
1027 
1028         *seq_size = idx;
1029 }
1030 
1031 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],
1032                              unsigned int *seq_size)
1033 {
1034         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1035         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1036         unsigned int idx = *seq_size;
1037 
1038         /* Loading MAC state */
1039         hw_desc_init(&desc[idx]);
1040         set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);
1041         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1042         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1043         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1044         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1045         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1046         set_aes_not_hash_mode(&desc[idx]);
1047         idx++;
1048 
1049         /* Setup XCBC MAC K1 */
1050         hw_desc_init(&desc[idx]);
1051         set_din_type(&desc[idx], DMA_DLLI,
1052                      ctx->auth_state.xcbc.xcbc_keys_dma_addr,
1053                      AES_KEYSIZE_128, NS_BIT);
1054         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1055         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1056         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1057         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1058         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1059         set_aes_not_hash_mode(&desc[idx]);
1060         idx++;
1061 
1062         /* Setup XCBC MAC K2 */
1063         hw_desc_init(&desc[idx]);
1064         set_din_type(&desc[idx], DMA_DLLI,
1065                      (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1066                       AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1067         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1068         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1069         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1070         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1071         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1072         set_aes_not_hash_mode(&desc[idx]);
1073         idx++;
1074 
1075         /* Setup XCBC MAC K3 */
1076         hw_desc_init(&desc[idx]);
1077         set_din_type(&desc[idx], DMA_DLLI,
1078                      (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1079                       2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1080         set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
1081         set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1082         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1083         set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1084         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1085         set_aes_not_hash_mode(&desc[idx]);
1086         idx++;
1087 
1088         *seq_size = idx;
1089 }
1090 
1091 static void cc_proc_header_desc(struct aead_request *req,
1092                                 struct cc_hw_desc desc[],
1093                                 unsigned int *seq_size)
1094 {
1095         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1096         unsigned int idx = *seq_size;
1097 
1098         /* Hash associated data */
1099         if (areq_ctx->assoclen > 0)
1100                 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1101 
1102         /* Hash IV */
1103         *seq_size = idx;
1104 }
1105 
1106 static void cc_proc_scheme_desc(struct aead_request *req,
1107                                 struct cc_hw_desc desc[],
1108                                 unsigned int *seq_size)
1109 {
1110         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1111         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1112         struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;
1113         unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1114                                 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1115         unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1116                                 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1117         unsigned int idx = *seq_size;
1118 
1119         hw_desc_init(&desc[idx]);
1120         set_cipher_mode(&desc[idx], hash_mode);
1121         set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1122                       ctx->hash_len);
1123         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1124         set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
1125         set_cipher_do(&desc[idx], DO_PAD);
1126         idx++;
1127 
1128         /* Get final ICV result */
1129         hw_desc_init(&desc[idx]);
1130         set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1131                       digest_size);
1132         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1133         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1134         set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1135         set_cipher_mode(&desc[idx], hash_mode);
1136         idx++;
1137 
1138         /* Loading hash opad xor key state */
1139         hw_desc_init(&desc[idx]);
1140         set_cipher_mode(&desc[idx], hash_mode);
1141         set_din_type(&desc[idx], DMA_DLLI,
1142                      (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),
1143                      digest_size, NS_BIT);
1144         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1145         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1146         idx++;
1147 
1148         /* Load init. digest len (64 bytes) */
1149         hw_desc_init(&desc[idx]);
1150         set_cipher_mode(&desc[idx], hash_mode);
1151         set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1152                      ctx->hash_len);
1153         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1154         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1155         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1156         idx++;
1157 
1158         /* Perform HASH update */
1159         hw_desc_init(&desc[idx]);
1160         set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,
1161                      digest_size);
1162         set_flow_mode(&desc[idx], DIN_HASH);
1163         idx++;
1164 
1165         *seq_size = idx;
1166 }
1167 
1168 static void cc_mlli_to_sram(struct aead_request *req,
1169                             struct cc_hw_desc desc[], unsigned int *seq_size)
1170 {
1171         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1172         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1173         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1174         struct device *dev = drvdata_to_dev(ctx->drvdata);
1175 
1176         if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1177             req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||
1178             !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) {
1179                 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
1180                         (unsigned int)ctx->drvdata->mlli_sram_addr,
1181                         req_ctx->mlli_params.mlli_len);
1182                 /* Copy MLLI table host-to-sram */
1183                 hw_desc_init(&desc[*seq_size]);
1184                 set_din_type(&desc[*seq_size], DMA_DLLI,
1185                              req_ctx->mlli_params.mlli_dma_addr,
1186                              req_ctx->mlli_params.mlli_len, NS_BIT);
1187                 set_dout_sram(&desc[*seq_size],
1188                               ctx->drvdata->mlli_sram_addr,
1189                               req_ctx->mlli_params.mlli_len);
1190                 set_flow_mode(&desc[*seq_size], BYPASS);
1191                 (*seq_size)++;
1192         }
1193 }
1194 
1195 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,
1196                                           enum cc_flow_mode setup_flow_mode,
1197                                           bool is_single_pass)
1198 {
1199         enum cc_flow_mode data_flow_mode;
1200 
1201         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1202                 if (setup_flow_mode == S_DIN_to_AES)
1203                         data_flow_mode = is_single_pass ?
1204                                 AES_to_HASH_and_DOUT : DIN_AES_DOUT;
1205                 else
1206                         data_flow_mode = is_single_pass ?
1207                                 DES_to_HASH_and_DOUT : DIN_DES_DOUT;
1208         } else { /* Decrypt */
1209                 if (setup_flow_mode == S_DIN_to_AES)
1210                         data_flow_mode = is_single_pass ?
1211                                 AES_and_HASH : DIN_AES_DOUT;
1212                 else
1213                         data_flow_mode = is_single_pass ?
1214                                 DES_and_HASH : DIN_DES_DOUT;
1215         }
1216 
1217         return data_flow_mode;
1218 }
1219 
1220 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1221                             unsigned int *seq_size)
1222 {
1223         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1224         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1225         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1226         int direct = req_ctx->gen_ctx.op_type;
1227         unsigned int data_flow_mode =
1228                 cc_get_data_flow(direct, ctx->flow_mode,
1229                                  req_ctx->is_single_pass);
1230 
1231         if (req_ctx->is_single_pass) {
1232                 /**
1233                  * Single-pass flow
1234                  */
1235                 cc_set_hmac_desc(req, desc, seq_size);
1236                 cc_set_cipher_desc(req, desc, seq_size);
1237                 cc_proc_header_desc(req, desc, seq_size);
1238                 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1239                 cc_proc_scheme_desc(req, desc, seq_size);
1240                 cc_proc_digest_desc(req, desc, seq_size);
1241                 return;
1242         }
1243 
1244         /**
1245          * Double-pass flow
1246          * Fallback for unsupported single-pass modes,
1247          * i.e. using assoc. data of non-word-multiple
1248          */
1249         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1250                 /* encrypt first.. */
1251                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1252                 /* authenc after..*/
1253                 cc_set_hmac_desc(req, desc, seq_size);
1254                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1255                 cc_proc_scheme_desc(req, desc, seq_size);
1256                 cc_proc_digest_desc(req, desc, seq_size);
1257 
1258         } else { /*DECRYPT*/
1259                 /* authenc first..*/
1260                 cc_set_hmac_desc(req, desc, seq_size);
1261                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1262                 cc_proc_scheme_desc(req, desc, seq_size);
1263                 /* decrypt after.. */
1264                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1265                 /* read the digest result with setting the completion bit
1266                  * must be after the cipher operation
1267                  */
1268                 cc_proc_digest_desc(req, desc, seq_size);
1269         }
1270 }
1271 
1272 static void
1273 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1274                 unsigned int *seq_size)
1275 {
1276         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1277         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1278         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1279         int direct = req_ctx->gen_ctx.op_type;
1280         unsigned int data_flow_mode =
1281                 cc_get_data_flow(direct, ctx->flow_mode,
1282                                  req_ctx->is_single_pass);
1283 
1284         if (req_ctx->is_single_pass) {
1285                 /**
1286                  * Single-pass flow
1287                  */
1288                 cc_set_xcbc_desc(req, desc, seq_size);
1289                 cc_set_cipher_desc(req, desc, seq_size);
1290                 cc_proc_header_desc(req, desc, seq_size);
1291                 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1292                 cc_proc_digest_desc(req, desc, seq_size);
1293                 return;
1294         }
1295 
1296         /**
1297          * Double-pass flow
1298          * Fallback for unsupported single-pass modes,
1299          * i.e. using assoc. data of non-word-multiple
1300          */
1301         if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1302                 /* encrypt first.. */
1303                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1304                 /* authenc after.. */
1305                 cc_set_xcbc_desc(req, desc, seq_size);
1306                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1307                 cc_proc_digest_desc(req, desc, seq_size);
1308         } else { /*DECRYPT*/
1309                 /* authenc first.. */
1310                 cc_set_xcbc_desc(req, desc, seq_size);
1311                 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1312                 /* decrypt after..*/
1313                 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1314                 /* read the digest result with setting the completion bit
1315                  * must be after the cipher operation
1316                  */
1317                 cc_proc_digest_desc(req, desc, seq_size);
1318         }
1319 }
1320 
1321 static int validate_data_size(struct cc_aead_ctx *ctx,
1322                               enum drv_crypto_direction direct,
1323                               struct aead_request *req)
1324 {
1325         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1326         struct device *dev = drvdata_to_dev(ctx->drvdata);
1327         unsigned int assoclen = areq_ctx->assoclen;
1328         unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
1329                         (req->cryptlen - ctx->authsize) : req->cryptlen;
1330 
1331         if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&
1332             req->cryptlen < ctx->authsize)
1333                 goto data_size_err;
1334 
1335         areq_ctx->is_single_pass = true; /*defaulted to fast flow*/
1336 
1337         switch (ctx->flow_mode) {
1338         case S_DIN_to_AES:
1339                 if (ctx->cipher_mode == DRV_CIPHER_CBC &&
1340                     !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))
1341                         goto data_size_err;
1342                 if (ctx->cipher_mode == DRV_CIPHER_CCM)
1343                         break;
1344                 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1345                         if (areq_ctx->plaintext_authenticate_only)
1346                                 areq_ctx->is_single_pass = false;
1347                         break;
1348                 }
1349 
1350                 if (!IS_ALIGNED(assoclen, sizeof(u32)))
1351                         areq_ctx->is_single_pass = false;
1352 
1353                 if (ctx->cipher_mode == DRV_CIPHER_CTR &&
1354                     !IS_ALIGNED(cipherlen, sizeof(u32)))
1355                         areq_ctx->is_single_pass = false;
1356 
1357                 break;
1358         case S_DIN_to_DES:
1359                 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))
1360                         goto data_size_err;
1361                 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))
1362                         areq_ctx->is_single_pass = false;
1363                 break;
1364         default:
1365                 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);
1366                 goto data_size_err;
1367         }
1368 
1369         return 0;
1370 
1371 data_size_err:
1372         return -EINVAL;
1373 }
1374 
1375 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
1376 {
1377         unsigned int len = 0;
1378 
1379         if (header_size == 0)
1380                 return 0;
1381 
1382         if (header_size < ((1UL << 16) - (1UL << 8))) {
1383                 len = 2;
1384 
1385                 pa0_buff[0] = (header_size >> 8) & 0xFF;
1386                 pa0_buff[1] = header_size & 0xFF;
1387         } else {
1388                 len = 6;
1389 
1390                 pa0_buff[0] = 0xFF;
1391                 pa0_buff[1] = 0xFE;
1392                 pa0_buff[2] = (header_size >> 24) & 0xFF;
1393                 pa0_buff[3] = (header_size >> 16) & 0xFF;
1394                 pa0_buff[4] = (header_size >> 8) & 0xFF;
1395                 pa0_buff[5] = header_size & 0xFF;
1396         }
1397 
1398         return len;
1399 }
1400 
1401 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)
1402 {
1403         __be32 data;
1404 
1405         memset(block, 0, csize);
1406         block += csize;
1407 
1408         if (csize >= 4)
1409                 csize = 4;
1410         else if (msglen > (1 << (8 * csize)))
1411                 return -EOVERFLOW;
1412 
1413         data = cpu_to_be32(msglen);
1414         memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1415 
1416         return 0;
1417 }
1418 
1419 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
1420                   unsigned int *seq_size)
1421 {
1422         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1423         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1424         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1425         unsigned int idx = *seq_size;
1426         unsigned int cipher_flow_mode;
1427         dma_addr_t mac_result;
1428 
1429         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1430                 cipher_flow_mode = AES_to_HASH_and_DOUT;
1431                 mac_result = req_ctx->mac_buf_dma_addr;
1432         } else { /* Encrypt */
1433                 cipher_flow_mode = AES_and_HASH;
1434                 mac_result = req_ctx->icv_dma_addr;
1435         }
1436 
1437         /* load key */
1438         hw_desc_init(&desc[idx]);
1439         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1440         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1441                      ((ctx->enc_keylen == 24) ?  CC_AES_KEY_SIZE_MAX :
1442                       ctx->enc_keylen), NS_BIT);
1443         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1444         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1445         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1446         set_flow_mode(&desc[idx], S_DIN_to_AES);
1447         idx++;
1448 
1449         /* load ctr state */
1450         hw_desc_init(&desc[idx]);
1451         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1452         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1453         set_din_type(&desc[idx], DMA_DLLI,
1454                      req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);
1455         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1456         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1457         set_flow_mode(&desc[idx], S_DIN_to_AES);
1458         idx++;
1459 
1460         /* load MAC key */
1461         hw_desc_init(&desc[idx]);
1462         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1463         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1464                      ((ctx->enc_keylen == 24) ?  CC_AES_KEY_SIZE_MAX :
1465                       ctx->enc_keylen), NS_BIT);
1466         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1467         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1468         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1469         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1470         set_aes_not_hash_mode(&desc[idx]);
1471         idx++;
1472 
1473         /* load MAC state */
1474         hw_desc_init(&desc[idx]);
1475         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1476         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1477         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1478                      AES_BLOCK_SIZE, NS_BIT);
1479         set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1480         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1481         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1482         set_aes_not_hash_mode(&desc[idx]);
1483         idx++;
1484 
1485         /* process assoc data */
1486         if (req_ctx->assoclen > 0) {
1487                 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1488         } else {
1489                 hw_desc_init(&desc[idx]);
1490                 set_din_type(&desc[idx], DMA_DLLI,
1491                              sg_dma_address(&req_ctx->ccm_adata_sg),
1492                              AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);
1493                 set_flow_mode(&desc[idx], DIN_HASH);
1494                 idx++;
1495         }
1496 
1497         /* process the cipher */
1498         if (req_ctx->cryptlen)
1499                 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);
1500 
1501         /* Read temporal MAC */
1502         hw_desc_init(&desc[idx]);
1503         set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1504         set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,
1505                       NS_BIT, 0);
1506         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1507         set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1508         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1509         set_aes_not_hash_mode(&desc[idx]);
1510         idx++;
1511 
1512         /* load AES-CTR state (for last MAC calculation)*/
1513         hw_desc_init(&desc[idx]);
1514         set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1515         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1516         set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,
1517                      AES_BLOCK_SIZE, NS_BIT);
1518         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1519         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1520         set_flow_mode(&desc[idx], S_DIN_to_AES);
1521         idx++;
1522 
1523         hw_desc_init(&desc[idx]);
1524         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1525         set_dout_no_dma(&desc[idx], 0, 0, 1);
1526         idx++;
1527 
1528         /* encrypt the "T" value and store MAC in mac_state */
1529         hw_desc_init(&desc[idx]);
1530         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1531                      ctx->authsize, NS_BIT);
1532         set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1533         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1534         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1535         idx++;
1536 
1537         *seq_size = idx;
1538         return 0;
1539 }
1540 
1541 static int config_ccm_adata(struct aead_request *req)
1542 {
1543         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1544         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1545         struct device *dev = drvdata_to_dev(ctx->drvdata);
1546         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1547         //unsigned int size_of_a = 0, rem_a_size = 0;
1548         unsigned int lp = req->iv[0];
1549         /* Note: The code assume that req->iv[0] already contains the value
1550          * of L' of RFC3610
1551          */
1552         unsigned int l = lp + 1;  /* This is L' of RFC 3610. */
1553         unsigned int m = ctx->authsize;  /* This is M' of RFC 3610. */
1554         u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;
1555         u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;
1556         u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1557         unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1558                                  DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1559                                 req->cryptlen :
1560                                 (req->cryptlen - ctx->authsize);
1561         int rc;
1562 
1563         memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1564         memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);
1565 
1566         /* taken from crypto/ccm.c */
1567         /* 2 <= L <= 8, so 1 <= L' <= 7. */
1568         if (l < 2 || l > 8) {
1569                 dev_err(dev, "illegal iv value %X\n", req->iv[0]);
1570                 return -EINVAL;
1571         }
1572         memcpy(b0, req->iv, AES_BLOCK_SIZE);
1573 
1574         /* format control info per RFC 3610 and
1575          * NIST Special Publication 800-38C
1576          */
1577         *b0 |= (8 * ((m - 2) / 2));
1578         if (req_ctx->assoclen > 0)
1579                 *b0 |= 64;  /* Enable bit 6 if Adata exists. */
1580 
1581         rc = set_msg_len(b0 + 16 - l, cryptlen, l);  /* Write L'. */
1582         if (rc) {
1583                 dev_err(dev, "message len overflow detected");
1584                 return rc;
1585         }
1586          /* END of "taken from crypto/ccm.c" */
1587 
1588         /* l(a) - size of associated data. */
1589         req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen);
1590 
1591         memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);
1592         req->iv[15] = 1;
1593 
1594         memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);
1595         ctr_count_0[15] = 0;
1596 
1597         return 0;
1598 }
1599 
1600 static void cc_proc_rfc4309_ccm(struct aead_request *req)
1601 {
1602         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1603         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1604         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1605 
1606         /* L' */
1607         memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
1608         /* For RFC 4309, always use 4 bytes for message length
1609          * (at most 2^32-1 bytes).
1610          */
1611         areq_ctx->ctr_iv[0] = 3;
1612 
1613         /* In RFC 4309 there is an 11-bytes nonce+IV part,
1614          * that we build here.
1615          */
1616         memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,
1617                CCM_BLOCK_NONCE_SIZE);
1618         memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,
1619                CCM_BLOCK_IV_SIZE);
1620         req->iv = areq_ctx->ctr_iv;
1621         areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE;
1622 }
1623 
1624 static void cc_set_ghash_desc(struct aead_request *req,
1625                               struct cc_hw_desc desc[], unsigned int *seq_size)
1626 {
1627         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1628         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1629         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1630         unsigned int idx = *seq_size;
1631 
1632         /* load key to AES*/
1633         hw_desc_init(&desc[idx]);
1634         set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1635         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1636         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1637                      ctx->enc_keylen, NS_BIT);
1638         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1639         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1640         set_flow_mode(&desc[idx], S_DIN_to_AES);
1641         idx++;
1642 
1643         /* process one zero block to generate hkey */
1644         hw_desc_init(&desc[idx]);
1645         set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1646         set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,
1647                       NS_BIT, 0);
1648         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1649         idx++;
1650 
1651         /* Memory Barrier */
1652         hw_desc_init(&desc[idx]);
1653         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1654         set_dout_no_dma(&desc[idx], 0, 0, 1);
1655         idx++;
1656 
1657         /* Load GHASH subkey */
1658         hw_desc_init(&desc[idx]);
1659         set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,
1660                      AES_BLOCK_SIZE, NS_BIT);
1661         set_dout_no_dma(&desc[idx], 0, 0, 1);
1662         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1663         set_aes_not_hash_mode(&desc[idx]);
1664         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1665         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1666         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1667         idx++;
1668 
1669         /* Configure Hash Engine to work with GHASH.
1670          * Since it was not possible to extend HASH submodes to add GHASH,
1671          * The following command is necessary in order to
1672          * select GHASH (according to HW designers)
1673          */
1674         hw_desc_init(&desc[idx]);
1675         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1676         set_dout_no_dma(&desc[idx], 0, 0, 1);
1677         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1678         set_aes_not_hash_mode(&desc[idx]);
1679         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1680         set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK
1681         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1682         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1683         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1684         idx++;
1685 
1686         /* Load GHASH initial STATE (which is 0). (for any hash there is an
1687          * initial state)
1688          */
1689         hw_desc_init(&desc[idx]);
1690         set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1691         set_dout_no_dma(&desc[idx], 0, 0, 1);
1692         set_flow_mode(&desc[idx], S_DIN_to_HASH);
1693         set_aes_not_hash_mode(&desc[idx]);
1694         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1695         set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1696         set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1697         idx++;
1698 
1699         *seq_size = idx;
1700 }
1701 
1702 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
1703                              unsigned int *seq_size)
1704 {
1705         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1706         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1707         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1708         unsigned int idx = *seq_size;
1709 
1710         /* load key to AES*/
1711         hw_desc_init(&desc[idx]);
1712         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1713         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1714         set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1715                      ctx->enc_keylen, NS_BIT);
1716         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1717         set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1718         set_flow_mode(&desc[idx], S_DIN_to_AES);
1719         idx++;
1720 
1721         if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {
1722                 /* load AES/CTR initial CTR value inc by 2*/
1723                 hw_desc_init(&desc[idx]);
1724                 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1725                 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1726                 set_din_type(&desc[idx], DMA_DLLI,
1727                              req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,
1728                              NS_BIT);
1729                 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1730                 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1731                 set_flow_mode(&desc[idx], S_DIN_to_AES);
1732                 idx++;
1733         }
1734 
1735         *seq_size = idx;
1736 }
1737 
1738 static void cc_proc_gcm_result(struct aead_request *req,
1739                                struct cc_hw_desc desc[],
1740                                unsigned int *seq_size)
1741 {
1742         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1743         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1744         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1745         dma_addr_t mac_result;
1746         unsigned int idx = *seq_size;
1747 
1748         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1749                 mac_result = req_ctx->mac_buf_dma_addr;
1750         } else { /* Encrypt */
1751                 mac_result = req_ctx->icv_dma_addr;
1752         }
1753 
1754         /* process(ghash) gcm_block_len */
1755         hw_desc_init(&desc[idx]);
1756         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,
1757                      AES_BLOCK_SIZE, NS_BIT);
1758         set_flow_mode(&desc[idx], DIN_HASH);
1759         idx++;
1760 
1761         /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */
1762         hw_desc_init(&desc[idx]);
1763         set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1764         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1765         set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,
1766                       NS_BIT, 0);
1767         set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1768         set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1769         set_aes_not_hash_mode(&desc[idx]);
1770 
1771         idx++;
1772 
1773         /* load AES/CTR initial CTR value inc by 1*/
1774         hw_desc_init(&desc[idx]);
1775         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1776         set_key_size_aes(&desc[idx], ctx->enc_keylen);
1777         set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,
1778                      AES_BLOCK_SIZE, NS_BIT);
1779         set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1780         set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1781         set_flow_mode(&desc[idx], S_DIN_to_AES);
1782         idx++;
1783 
1784         /* Memory Barrier */
1785         hw_desc_init(&desc[idx]);
1786         set_din_no_dma(&desc[idx], 0, 0xfffff0);
1787         set_dout_no_dma(&desc[idx], 0, 0, 1);
1788         idx++;
1789 
1790         /* process GCTR on stored GHASH and store MAC in mac_state*/
1791         hw_desc_init(&desc[idx]);
1792         set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1793         set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1794                      AES_BLOCK_SIZE, NS_BIT);
1795         set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1796         set_queue_last_ind(ctx->drvdata, &desc[idx]);
1797         set_flow_mode(&desc[idx], DIN_AES_DOUT);
1798         idx++;
1799 
1800         *seq_size = idx;
1801 }
1802 
1803 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
1804                   unsigned int *seq_size)
1805 {
1806         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1807         unsigned int cipher_flow_mode;
1808 
1809         if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1810                 cipher_flow_mode = AES_and_HASH;
1811         } else { /* Encrypt */
1812                 cipher_flow_mode = AES_to_HASH_and_DOUT;
1813         }
1814 
1815         //in RFC4543 no data to encrypt. just copy data from src to dest.
1816         if (req_ctx->plaintext_authenticate_only) {
1817                 cc_proc_cipher_desc(req, BYPASS, desc, seq_size);
1818                 cc_set_ghash_desc(req, desc, seq_size);
1819                 /* process(ghash) assoc data */
1820                 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1821                 cc_set_gctr_desc(req, desc, seq_size);
1822                 cc_proc_gcm_result(req, desc, seq_size);
1823                 return 0;
1824         }
1825 
1826         // for gcm and rfc4106.
1827         cc_set_ghash_desc(req, desc, seq_size);
1828         /* process(ghash) assoc data */
1829         if (req_ctx->assoclen > 0)
1830                 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1831         cc_set_gctr_desc(req, desc, seq_size);
1832         /* process(gctr+ghash) */
1833         if (req_ctx->cryptlen)
1834                 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);
1835         cc_proc_gcm_result(req, desc, seq_size);
1836 
1837         return 0;
1838 }
1839 
1840 static int config_gcm_context(struct aead_request *req)
1841 {
1842         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1843         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1844         struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1845         struct device *dev = drvdata_to_dev(ctx->drvdata);
1846 
1847         unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1848                                  DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1849                                 req->cryptlen :
1850                                 (req->cryptlen - ctx->authsize);
1851         __be32 counter = cpu_to_be32(2);
1852 
1853         dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n",
1854                 __func__, cryptlen, req_ctx->assoclen, ctx->authsize);
1855 
1856         memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
1857 
1858         memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1859 
1860         memcpy(req->iv + 12, &counter, 4);
1861         memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);
1862 
1863         counter = cpu_to_be32(1);
1864         memcpy(req->iv + 12, &counter, 4);
1865         memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);
1866 
1867         if (!req_ctx->plaintext_authenticate_only) {
1868                 __be64 temp64;
1869 
1870                 temp64 = cpu_to_be64(req_ctx->assoclen * 8);
1871                 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1872                 temp64 = cpu_to_be64(cryptlen * 8);
1873                 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1874         } else {
1875                 /* rfc4543=>  all data(AAD,IV,Plain) are considered additional
1876                  * data that is nothing is encrypted.
1877                  */
1878                 __be64 temp64;
1879 
1880                 temp64 = cpu_to_be64((req_ctx->assoclen +
1881                                       GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8);
1882                 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1883                 temp64 = 0;
1884                 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1885         }
1886 
1887         return 0;
1888 }
1889 
1890 static void cc_proc_rfc4_gcm(struct aead_request *req)
1891 {
1892         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1893         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1894         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1895 
1896         memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
1897                ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
1898         memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,
1899                GCM_BLOCK_RFC4_IV_SIZE);
1900         req->iv = areq_ctx->ctr_iv;
1901         areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE;
1902 }
1903 
1904 static int cc_proc_aead(struct aead_request *req,
1905                         enum drv_crypto_direction direct)
1906 {
1907         int rc = 0;
1908         int seq_len = 0;
1909         struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
1910         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1911         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1912         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1913         struct device *dev = drvdata_to_dev(ctx->drvdata);
1914         struct cc_crypto_req cc_req = {};
1915 
1916         dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
1917                 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),
1918                 ctx, req, req->iv, sg_virt(req->src), req->src->offset,
1919                 sg_virt(req->dst), req->dst->offset, req->cryptlen);
1920 
1921         /* STAT_PHASE_0: Init and sanity checks */
1922 
1923         /* Check data length according to mode */
1924         if (validate_data_size(ctx, direct, req)) {
1925                 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",
1926                         req->cryptlen, areq_ctx->assoclen);
1927                 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
1928                 return -EINVAL;
1929         }
1930 
1931         /* Setup request structure */
1932         cc_req.user_cb = (void *)cc_aead_complete;
1933         cc_req.user_arg = (void *)req;
1934 
1935         /* Setup request context */
1936         areq_ctx->gen_ctx.op_type = direct;
1937         areq_ctx->req_authsize = ctx->authsize;
1938         areq_ctx->cipher_mode = ctx->cipher_mode;
1939 
1940         /* STAT_PHASE_1: Map buffers */
1941 
1942         if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1943                 /* Build CTR IV - Copy nonce from last 4 bytes in
1944                  * CTR key to first 4 bytes in CTR IV
1945                  */
1946                 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
1947                        CTR_RFC3686_NONCE_SIZE);
1948                 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv,
1949                        CTR_RFC3686_IV_SIZE);
1950                 /* Initialize counter portion of counter block */
1951                 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
1952                             CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1953 
1954                 /* Replace with counter iv */
1955                 req->iv = areq_ctx->ctr_iv;
1956                 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;
1957         } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||
1958                    (ctx->cipher_mode == DRV_CIPHER_GCTR)) {
1959                 areq_ctx->hw_iv_size = AES_BLOCK_SIZE;
1960                 if (areq_ctx->ctr_iv != req->iv) {
1961                         memcpy(areq_ctx->ctr_iv, req->iv,
1962                                crypto_aead_ivsize(tfm));
1963                         req->iv = areq_ctx->ctr_iv;
1964                 }
1965         }  else {
1966                 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);
1967         }
1968 
1969         if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1970                 rc = config_ccm_adata(req);
1971                 if (rc) {
1972                         dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",
1973                                 rc);
1974                         goto exit;
1975                 }
1976         } else {
1977                 areq_ctx->ccm_hdr_size = ccm_header_size_null;
1978         }
1979 
1980         if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1981                 rc = config_gcm_context(req);
1982                 if (rc) {
1983                         dev_dbg(dev, "config_gcm_context() returned with a failure %d!",
1984                                 rc);
1985                         goto exit;
1986                 }
1987         }
1988 
1989         rc = cc_map_aead_request(ctx->drvdata, req);
1990         if (rc) {
1991                 dev_err(dev, "map_request() failed\n");
1992                 goto exit;
1993         }
1994 
1995         /* STAT_PHASE_2: Create sequence */
1996 
1997         /* Load MLLI tables to SRAM if necessary */
1998         cc_mlli_to_sram(req, desc, &seq_len);
1999 
2000         /*TODO: move seq len by reference */
2001         switch (ctx->auth_mode) {
2002         case DRV_HASH_SHA1:
2003         case DRV_HASH_SHA256:
2004                 cc_hmac_authenc(req, desc, &seq_len);
2005                 break;
2006         case DRV_HASH_XCBC_MAC:
2007                 cc_xcbc_authenc(req, desc, &seq_len);
2008                 break;
2009         case DRV_HASH_NULL:
2010                 if (ctx->cipher_mode == DRV_CIPHER_CCM)
2011                         cc_ccm(req, desc, &seq_len);
2012                 if (ctx->cipher_mode == DRV_CIPHER_GCTR)
2013                         cc_gcm(req, desc, &seq_len);
2014                 break;
2015         default:
2016                 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
2017                 cc_unmap_aead_request(dev, req);
2018                 rc = -ENOTSUPP;
2019                 goto exit;
2020         }
2021 
2022         /* STAT_PHASE_3: Lock HW and push sequence */
2023 
2024         rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);
2025 
2026         if (rc != -EINPROGRESS && rc != -EBUSY) {
2027                 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
2028                 cc_unmap_aead_request(dev, req);
2029         }
2030 
2031 exit:
2032         return rc;
2033 }
2034 
2035 static int cc_aead_encrypt(struct aead_request *req)
2036 {
2037         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2038         int rc;
2039 
2040         memset(areq_ctx, 0, sizeof(*areq_ctx));
2041 
2042         /* No generated IV required */
2043         areq_ctx->backup_iv = req->iv;
2044         areq_ctx->assoclen = req->assoclen;
2045         areq_ctx->is_gcm4543 = false;
2046 
2047         areq_ctx->plaintext_authenticate_only = false;
2048 
2049         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2050         if (rc != -EINPROGRESS && rc != -EBUSY)
2051                 req->iv = areq_ctx->backup_iv;
2052 
2053         return rc;
2054 }
2055 
2056 static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
2057 {
2058         /* Very similar to cc_aead_encrypt() above. */
2059 
2060         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2061         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2062         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2063         struct device *dev = drvdata_to_dev(ctx->drvdata);
2064         int rc = -EINVAL;
2065 
2066         if (!valid_assoclen(req)) {
2067                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2068                 goto out;
2069         }
2070 
2071         memset(areq_ctx, 0, sizeof(*areq_ctx));
2072 
2073         /* No generated IV required */
2074         areq_ctx->backup_iv = req->iv;
2075         areq_ctx->assoclen = req->assoclen;
2076         areq_ctx->is_gcm4543 = true;
2077 
2078         cc_proc_rfc4309_ccm(req);
2079 
2080         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2081         if (rc != -EINPROGRESS && rc != -EBUSY)
2082                 req->iv = areq_ctx->backup_iv;
2083 out:
2084         return rc;
2085 }
2086 
2087 static int cc_aead_decrypt(struct aead_request *req)
2088 {
2089         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2090         int rc;
2091 
2092         memset(areq_ctx, 0, sizeof(*areq_ctx));
2093 
2094         /* No generated IV required */
2095         areq_ctx->backup_iv = req->iv;
2096         areq_ctx->assoclen = req->assoclen;
2097         areq_ctx->is_gcm4543 = false;
2098 
2099         areq_ctx->plaintext_authenticate_only = false;
2100 
2101         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2102         if (rc != -EINPROGRESS && rc != -EBUSY)
2103                 req->iv = areq_ctx->backup_iv;
2104 
2105         return rc;
2106 }
2107 
2108 static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
2109 {
2110         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2111         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2112         struct device *dev = drvdata_to_dev(ctx->drvdata);
2113         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2114         int rc = -EINVAL;
2115 
2116         if (!valid_assoclen(req)) {
2117                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2118                 goto out;
2119         }
2120 
2121         memset(areq_ctx, 0, sizeof(*areq_ctx));
2122 
2123         /* No generated IV required */
2124         areq_ctx->backup_iv = req->iv;
2125         areq_ctx->assoclen = req->assoclen;
2126 
2127         areq_ctx->is_gcm4543 = true;
2128         cc_proc_rfc4309_ccm(req);
2129 
2130         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2131         if (rc != -EINPROGRESS && rc != -EBUSY)
2132                 req->iv = areq_ctx->backup_iv;
2133 
2134 out:
2135         return rc;
2136 }
2137 
2138 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2139                                  unsigned int keylen)
2140 {
2141         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2142         struct device *dev = drvdata_to_dev(ctx->drvdata);
2143 
2144         dev_dbg(dev, "%s()  keylen %d, key %p\n", __func__, keylen, key);
2145 
2146         if (keylen < 4)
2147                 return -EINVAL;
2148 
2149         keylen -= 4;
2150         memcpy(ctx->ctr_nonce, key + keylen, 4);
2151 
2152         return cc_aead_setkey(tfm, key, keylen);
2153 }
2154 
2155 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2156                                  unsigned int keylen)
2157 {
2158         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2159         struct device *dev = drvdata_to_dev(ctx->drvdata);
2160 
2161         dev_dbg(dev, "%s()  keylen %d, key %p\n", __func__, keylen, key);
2162 
2163         if (keylen < 4)
2164                 return -EINVAL;
2165 
2166         keylen -= 4;
2167         memcpy(ctx->ctr_nonce, key + keylen, 4);
2168 
2169         return cc_aead_setkey(tfm, key, keylen);
2170 }
2171 
2172 static int cc_gcm_setauthsize(struct crypto_aead *authenc,
2173                               unsigned int authsize)
2174 {
2175         switch (authsize) {
2176         case 4:
2177         case 8:
2178         case 12:
2179         case 13:
2180         case 14:
2181         case 15:
2182         case 16:
2183                 break;
2184         default:
2185                 return -EINVAL;
2186         }
2187 
2188         return cc_aead_setauthsize(authenc, authsize);
2189 }
2190 
2191 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
2192                                       unsigned int authsize)
2193 {
2194         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2195         struct device *dev = drvdata_to_dev(ctx->drvdata);
2196 
2197         dev_dbg(dev, "authsize %d\n", authsize);
2198 
2199         switch (authsize) {
2200         case 8:
2201         case 12:
2202         case 16:
2203                 break;
2204         default:
2205                 return -EINVAL;
2206         }
2207 
2208         return cc_aead_setauthsize(authenc, authsize);
2209 }
2210 
2211 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
2212                                       unsigned int authsize)
2213 {
2214         struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2215         struct device *dev = drvdata_to_dev(ctx->drvdata);
2216 
2217         dev_dbg(dev, "authsize %d\n", authsize);
2218 
2219         if (authsize != 16)
2220                 return -EINVAL;
2221 
2222         return cc_aead_setauthsize(authenc, authsize);
2223 }
2224 
2225 static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
2226 {
2227         /* Very similar to cc_aead_encrypt() above. */
2228 
2229         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2230         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2231         struct device *dev = drvdata_to_dev(ctx->drvdata);
2232         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2233         int rc = -EINVAL;
2234 
2235         if (!valid_assoclen(req)) {
2236                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2237                 goto out;
2238         }
2239 
2240         memset(areq_ctx, 0, sizeof(*areq_ctx));
2241 
2242         /* No generated IV required */
2243         areq_ctx->backup_iv = req->iv;
2244         areq_ctx->assoclen = req->assoclen;
2245         areq_ctx->plaintext_authenticate_only = false;
2246 
2247         cc_proc_rfc4_gcm(req);
2248         areq_ctx->is_gcm4543 = true;
2249 
2250         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2251         if (rc != -EINPROGRESS && rc != -EBUSY)
2252                 req->iv = areq_ctx->backup_iv;
2253 out:
2254         return rc;
2255 }
2256 
2257 static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
2258 {
2259         /* Very similar to cc_aead_encrypt() above. */
2260         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2261         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2262         struct device *dev = drvdata_to_dev(ctx->drvdata);
2263         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2264         int rc = -EINVAL;
2265 
2266         if (!valid_assoclen(req)) {
2267                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2268                 goto out;
2269         }
2270 
2271         memset(areq_ctx, 0, sizeof(*areq_ctx));
2272 
2273         //plaintext is not encryped with rfc4543
2274         areq_ctx->plaintext_authenticate_only = true;
2275 
2276         /* No generated IV required */
2277         areq_ctx->backup_iv = req->iv;
2278         areq_ctx->assoclen = req->assoclen;
2279 
2280         cc_proc_rfc4_gcm(req);
2281         areq_ctx->is_gcm4543 = true;
2282 
2283         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2284         if (rc != -EINPROGRESS && rc != -EBUSY)
2285                 req->iv = areq_ctx->backup_iv;
2286 out:
2287         return rc;
2288 }
2289 
2290 static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
2291 {
2292         /* Very similar to cc_aead_decrypt() above. */
2293 
2294         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2295         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2296         struct device *dev = drvdata_to_dev(ctx->drvdata);
2297         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2298         int rc = -EINVAL;
2299 
2300         if (!valid_assoclen(req)) {
2301                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2302                 goto out;
2303         }
2304 
2305         memset(areq_ctx, 0, sizeof(*areq_ctx));
2306 
2307         /* No generated IV required */
2308         areq_ctx->backup_iv = req->iv;
2309         areq_ctx->assoclen = req->assoclen;
2310         areq_ctx->plaintext_authenticate_only = false;
2311 
2312         cc_proc_rfc4_gcm(req);
2313         areq_ctx->is_gcm4543 = true;
2314 
2315         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2316         if (rc != -EINPROGRESS && rc != -EBUSY)
2317                 req->iv = areq_ctx->backup_iv;
2318 out:
2319         return rc;
2320 }
2321 
2322 static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
2323 {
2324         /* Very similar to cc_aead_decrypt() above. */
2325         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2326         struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2327         struct device *dev = drvdata_to_dev(ctx->drvdata);
2328         struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2329         int rc = -EINVAL;
2330 
2331         if (!valid_assoclen(req)) {
2332                 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2333                 goto out;
2334         }
2335 
2336         memset(areq_ctx, 0, sizeof(*areq_ctx));
2337 
2338         //plaintext is not decryped with rfc4543
2339         areq_ctx->plaintext_authenticate_only = true;
2340 
2341         /* No generated IV required */
2342         areq_ctx->backup_iv = req->iv;
2343         areq_ctx->assoclen = req->assoclen;
2344 
2345         cc_proc_rfc4_gcm(req);
2346         areq_ctx->is_gcm4543 = true;
2347 
2348         rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2349         if (rc != -EINPROGRESS && rc != -EBUSY)
2350                 req->iv = areq_ctx->backup_iv;
2351 out:
2352         return rc;
2353 }
2354 
2355 /* aead alg */
2356 static struct cc_alg_template aead_algs[] = {
2357         {
2358                 .name = "authenc(hmac(sha1),cbc(aes))",
2359                 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
2360                 .blocksize = AES_BLOCK_SIZE,
2361                 .template_aead = {
2362                         .setkey = cc_aead_setkey,
2363                         .setauthsize = cc_aead_setauthsize,
2364                         .encrypt = cc_aead_encrypt,
2365                         .decrypt = cc_aead_decrypt,
2366                         .init = cc_aead_init,
2367                         .exit = cc_aead_exit,
2368                         .ivsize = AES_BLOCK_SIZE,
2369                         .maxauthsize = SHA1_DIGEST_SIZE,
2370                 },
2371                 .cipher_mode = DRV_CIPHER_CBC,
2372                 .flow_mode = S_DIN_to_AES,
2373                 .auth_mode = DRV_HASH_SHA1,
2374                 .min_hw_rev = CC_HW_REV_630,
2375                 .std_body = CC_STD_NIST,
2376         },
2377         {
2378                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
2379                 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
2380                 .blocksize = DES3_EDE_BLOCK_SIZE,
2381                 .template_aead = {
2382                         .setkey = cc_des3_aead_setkey,
2383                         .setauthsize = cc_aead_setauthsize,
2384                         .encrypt = cc_aead_encrypt,
2385                         .decrypt = cc_aead_decrypt,
2386                         .init = cc_aead_init,
2387                         .exit = cc_aead_exit,
2388                         .ivsize = DES3_EDE_BLOCK_SIZE,
2389                         .maxauthsize = SHA1_DIGEST_SIZE,
2390                 },
2391                 .cipher_mode = DRV_CIPHER_CBC,
2392                 .flow_mode = S_DIN_to_DES,
2393                 .auth_mode = DRV_HASH_SHA1,
2394                 .min_hw_rev = CC_HW_REV_630,
2395                 .std_body = CC_STD_NIST,
2396         },
2397         {
2398                 .name = "authenc(hmac(sha256),cbc(aes))",
2399                 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
2400                 .blocksize = AES_BLOCK_SIZE,
2401                 .template_aead = {
2402                         .setkey = cc_aead_setkey,
2403                         .setauthsize = cc_aead_setauthsize,
2404                         .encrypt = cc_aead_encrypt,
2405                         .decrypt = cc_aead_decrypt,
2406                         .init = cc_aead_init,
2407                         .exit = cc_aead_exit,
2408                         .ivsize = AES_BLOCK_SIZE,
2409                         .maxauthsize = SHA256_DIGEST_SIZE,
2410                 },
2411                 .cipher_mode = DRV_CIPHER_CBC,
2412                 .flow_mode = S_DIN_to_AES,
2413                 .auth_mode = DRV_HASH_SHA256,
2414                 .min_hw_rev = CC_HW_REV_630,
2415                 .std_body = CC_STD_NIST,
2416         },
2417         {
2418                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
2419                 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
2420                 .blocksize = DES3_EDE_BLOCK_SIZE,
2421                 .template_aead = {
2422                         .setkey = cc_des3_aead_setkey,
2423                         .setauthsize = cc_aead_setauthsize,
2424                         .encrypt = cc_aead_encrypt,
2425                         .decrypt = cc_aead_decrypt,
2426                         .init = cc_aead_init,
2427                         .exit = cc_aead_exit,
2428                         .ivsize = DES3_EDE_BLOCK_SIZE,
2429                         .maxauthsize = SHA256_DIGEST_SIZE,
2430                 },
2431                 .cipher_mode = DRV_CIPHER_CBC,
2432                 .flow_mode = S_DIN_to_DES,
2433                 .auth_mode = DRV_HASH_SHA256,
2434                 .min_hw_rev = CC_HW_REV_630,
2435                 .std_body = CC_STD_NIST,
2436         },
2437         {
2438                 .name = "authenc(xcbc(aes),cbc(aes))",
2439                 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
2440                 .blocksize = AES_BLOCK_SIZE,
2441                 .template_aead = {
2442                         .setkey = cc_aead_setkey,
2443                         .setauthsize = cc_aead_setauthsize,
2444                         .encrypt = cc_aead_encrypt,
2445                         .decrypt = cc_aead_decrypt,
2446                         .init = cc_aead_init,
2447                         .exit = cc_aead_exit,
2448                         .ivsize = AES_BLOCK_SIZE,
2449                         .maxauthsize = AES_BLOCK_SIZE,
2450                 },
2451                 .cipher_mode = DRV_CIPHER_CBC,
2452                 .flow_mode = S_DIN_to_AES,
2453                 .auth_mode = DRV_HASH_XCBC_MAC,
2454                 .min_hw_rev = CC_HW_REV_630,
2455                 .std_body = CC_STD_NIST,
2456         },
2457         {
2458                 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2459                 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
2460                 .blocksize = 1,
2461                 .template_aead = {
2462                         .setkey = cc_aead_setkey,
2463                         .setauthsize = cc_aead_setauthsize,
2464                         .encrypt = cc_aead_encrypt,
2465                         .decrypt = cc_aead_decrypt,
2466                         .init = cc_aead_init,
2467                         .exit = cc_aead_exit,
2468                         .ivsize = CTR_RFC3686_IV_SIZE,
2469                         .maxauthsize = SHA1_DIGEST_SIZE,
2470                 },
2471                 .cipher_mode = DRV_CIPHER_CTR,
2472                 .flow_mode = S_DIN_to_AES,
2473                 .auth_mode = DRV_HASH_SHA1,
2474                 .min_hw_rev = CC_HW_REV_630,
2475                 .std_body = CC_STD_NIST,
2476         },
2477         {
2478                 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2479                 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
2480                 .blocksize = 1,
2481                 .template_aead = {
2482                         .setkey = cc_aead_setkey,
2483                         .setauthsize = cc_aead_setauthsize,
2484                         .encrypt = cc_aead_encrypt,
2485                         .decrypt = cc_aead_decrypt,
2486                         .init = cc_aead_init,
2487                         .exit = cc_aead_exit,
2488                         .ivsize = CTR_RFC3686_IV_SIZE,
2489                         .maxauthsize = SHA256_DIGEST_SIZE,
2490                 },
2491                 .cipher_mode = DRV_CIPHER_CTR,
2492                 .flow_mode = S_DIN_to_AES,
2493                 .auth_mode = DRV_HASH_SHA256,
2494                 .min_hw_rev = CC_HW_REV_630,
2495                 .std_body = CC_STD_NIST,
2496         },
2497         {
2498                 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
2499                 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
2500                 .blocksize = 1,
2501                 .template_aead = {
2502                         .setkey = cc_aead_setkey,
2503                         .setauthsize = cc_aead_setauthsize,
2504                         .encrypt = cc_aead_encrypt,
2505                         .decrypt = cc_aead_decrypt,
2506                         .init = cc_aead_init,
2507                         .exit = cc_aead_exit,
2508                         .ivsize = CTR_RFC3686_IV_SIZE,
2509                         .maxauthsize = AES_BLOCK_SIZE,
2510                 },
2511                 .cipher_mode = DRV_CIPHER_CTR,
2512                 .flow_mode = S_DIN_to_AES,
2513                 .auth_mode = DRV_HASH_XCBC_MAC,
2514                 .min_hw_rev = CC_HW_REV_630,
2515                 .std_body = CC_STD_NIST,
2516         },
2517         {
2518                 .name = "ccm(aes)",
2519                 .driver_name = "ccm-aes-ccree",
2520                 .blocksize = 1,
2521                 .template_aead = {
2522                         .setkey = cc_aead_setkey,
2523                         .setauthsize = cc_ccm_setauthsize,
2524                         .encrypt = cc_aead_encrypt,
2525                         .decrypt = cc_aead_decrypt,
2526                         .init = cc_aead_init,
2527                         .exit = cc_aead_exit,
2528                         .ivsize = AES_BLOCK_SIZE,
2529                         .maxauthsize = AES_BLOCK_SIZE,
2530                 },
2531                 .cipher_mode = DRV_CIPHER_CCM,
2532                 .flow_mode = S_DIN_to_AES,
2533                 .auth_mode = DRV_HASH_NULL,
2534                 .min_hw_rev = CC_HW_REV_630,
2535                 .std_body = CC_STD_NIST,
2536         },
2537         {
2538                 .name = "rfc4309(ccm(aes))",
2539                 .driver_name = "rfc4309-ccm-aes-ccree",
2540                 .blocksize = 1,
2541                 .template_aead = {
2542                         .setkey = cc_rfc4309_ccm_setkey,
2543                         .setauthsize = cc_rfc4309_ccm_setauthsize,
2544                         .encrypt = cc_rfc4309_ccm_encrypt,
2545                         .decrypt = cc_rfc4309_ccm_decrypt,
2546                         .init = cc_aead_init,
2547                         .exit = cc_aead_exit,
2548                         .ivsize = CCM_BLOCK_IV_SIZE,
2549                         .maxauthsize = AES_BLOCK_SIZE,
2550                 },
2551                 .cipher_mode = DRV_CIPHER_CCM,
2552                 .flow_mode = S_DIN_to_AES,
2553                 .auth_mode = DRV_HASH_NULL,
2554                 .min_hw_rev = CC_HW_REV_630,
2555                 .std_body = CC_STD_NIST,
2556         },
2557         {
2558                 .name = "gcm(aes)",
2559                 .driver_name = "gcm-aes-ccree",
2560                 .blocksize = 1,
2561                 .template_aead = {
2562                         .setkey = cc_aead_setkey,
2563                         .setauthsize = cc_gcm_setauthsize,
2564                         .encrypt = cc_aead_encrypt,
2565                         .decrypt = cc_aead_decrypt,
2566                         .init = cc_aead_init,
2567                         .exit = cc_aead_exit,
2568                         .ivsize = 12,
2569                         .maxauthsize = AES_BLOCK_SIZE,
2570                 },
2571                 .cipher_mode = DRV_CIPHER_GCTR,
2572                 .flow_mode = S_DIN_to_AES,
2573                 .auth_mode = DRV_HASH_NULL,
2574                 .min_hw_rev = CC_HW_REV_630,
2575                 .std_body = CC_STD_NIST,
2576         },
2577         {
2578                 .name = "rfc4106(gcm(aes))",
2579                 .driver_name = "rfc4106-gcm-aes-ccree",
2580                 .blocksize = 1,
2581                 .template_aead = {
2582                         .setkey = cc_rfc4106_gcm_setkey,
2583                         .setauthsize = cc_rfc4106_gcm_setauthsize,
2584                         .encrypt = cc_rfc4106_gcm_encrypt,
2585                         .decrypt = cc_rfc4106_gcm_decrypt,
2586                         .init = cc_aead_init,
2587                         .exit = cc_aead_exit,
2588                         .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2589                         .maxauthsize = AES_BLOCK_SIZE,
2590                 },
2591                 .cipher_mode = DRV_CIPHER_GCTR,
2592                 .flow_mode = S_DIN_to_AES,
2593                 .auth_mode = DRV_HASH_NULL,
2594                 .min_hw_rev = CC_HW_REV_630,
2595                 .std_body = CC_STD_NIST,
2596         },
2597         {
2598                 .name = "rfc4543(gcm(aes))",
2599                 .driver_name = "rfc4543-gcm-aes-ccree",
2600                 .blocksize = 1,
2601                 .template_aead = {
2602                         .setkey = cc_rfc4543_gcm_setkey,
2603                         .setauthsize = cc_rfc4543_gcm_setauthsize,
2604                         .encrypt = cc_rfc4543_gcm_encrypt,
2605                         .decrypt = cc_rfc4543_gcm_decrypt,
2606                         .init = cc_aead_init,
2607                         .exit = cc_aead_exit,
2608                         .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2609                         .maxauthsize = AES_BLOCK_SIZE,
2610                 },
2611                 .cipher_mode = DRV_CIPHER_GCTR,
2612                 .flow_mode = S_DIN_to_AES,
2613                 .auth_mode = DRV_HASH_NULL,
2614                 .min_hw_rev = CC_HW_REV_630,
2615                 .std_body = CC_STD_NIST,
2616         },
2617 };
2618 
2619 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
2620                                                 struct device *dev)
2621 {
2622         struct cc_crypto_alg *t_alg;
2623         struct aead_alg *alg;
2624 
2625         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
2626         if (!t_alg)
2627                 return ERR_PTR(-ENOMEM);
2628 
2629         alg = &tmpl->template_aead;
2630 
2631         snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
2632         snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2633                  tmpl->driver_name);
2634         alg->base.cra_module = THIS_MODULE;
2635         alg->base.cra_priority = CC_CRA_PRIO;
2636 
2637         alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
2638         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2639         alg->init = cc_aead_init;
2640         alg->exit = cc_aead_exit;
2641 
2642         t_alg->aead_alg = *alg;
2643 
2644         t_alg->cipher_mode = tmpl->cipher_mode;
2645         t_alg->flow_mode = tmpl->flow_mode;
2646         t_alg->auth_mode = tmpl->auth_mode;
2647 
2648         return t_alg;
2649 }
2650 
2651 int cc_aead_free(struct cc_drvdata *drvdata)
2652 {
2653         struct cc_crypto_alg *t_alg, *n;
2654         struct cc_aead_handle *aead_handle =
2655                 (struct cc_aead_handle *)drvdata->aead_handle;
2656 
2657         if (aead_handle) {
2658                 /* Remove registered algs */
2659                 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list,
2660                                          entry) {
2661                         crypto_unregister_aead(&t_alg->aead_alg);
2662                         list_del(&t_alg->entry);
2663                         kfree(t_alg);
2664                 }
2665                 kfree(aead_handle);
2666                 drvdata->aead_handle = NULL;
2667         }
2668 
2669         return 0;
2670 }
2671 
2672 int cc_aead_alloc(struct cc_drvdata *drvdata)
2673 {
2674         struct cc_aead_handle *aead_handle;
2675         struct cc_crypto_alg *t_alg;
2676         int rc = -ENOMEM;
2677         int alg;
2678         struct device *dev = drvdata_to_dev(drvdata);
2679 
2680         aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);
2681         if (!aead_handle) {
2682                 rc = -ENOMEM;
2683                 goto fail0;
2684         }
2685 
2686         INIT_LIST_HEAD(&aead_handle->aead_list);
2687         drvdata->aead_handle = aead_handle;
2688 
2689         aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,
2690                                                          MAX_HMAC_DIGEST_SIZE);
2691 
2692         if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {
2693                 dev_err(dev, "SRAM pool exhausted\n");
2694                 rc = -ENOMEM;
2695                 goto fail1;
2696         }
2697 
2698         /* Linux crypto */
2699         for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {
2700                 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) ||
2701                     !(drvdata->std_bodies & aead_algs[alg].std_body))
2702                         continue;
2703 
2704                 t_alg = cc_create_aead_alg(&aead_algs[alg], dev);
2705                 if (IS_ERR(t_alg)) {
2706                         rc = PTR_ERR(t_alg);
2707                         dev_err(dev, "%s alg allocation failed\n",
2708                                 aead_algs[alg].driver_name);
2709                         goto fail1;
2710                 }
2711                 t_alg->drvdata = drvdata;
2712                 rc = crypto_register_aead(&t_alg->aead_alg);
2713                 if (rc) {
2714                         dev_err(dev, "%s alg registration failed\n",
2715                                 t_alg->aead_alg.base.cra_driver_name);
2716                         goto fail2;
2717                 } else {
2718                         list_add_tail(&t_alg->entry, &aead_handle->aead_list);
2719                         dev_dbg(dev, "Registered %s\n",
2720                                 t_alg->aead_alg.base.cra_driver_name);
2721                 }
2722         }
2723 
2724         return 0;
2725 
2726 fail2:
2727         kfree(t_alg);
2728 fail1:
2729         cc_aead_free(drvdata);
2730 fail0:
2731         return rc;
2732 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/crypto/ccree/cc_aead.c

DEFINITIONS
