1/* 2 * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC 3 * 4 * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com> 5 * 6 * This file add support for AES cipher with 128,192,256 bits 7 * keysize in CBC and ECB mode. 8 * Add support also for DES and 3DES in CBC and ECB mode. 9 * 10 * You could find the datasheet in Documentation/arm/sunxi/README 11 * 12 * This program is free software; you can redistribute it and/or modify 13 * it under the terms of the GNU General Public License as published by 14 * the Free Software Foundation; either version 2 of the License, or 15 * (at your option) any later version. 16 */ 17#include "sun4i-ss.h" 18 19static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) 20{ 21 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 22 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 23 struct sun4i_ss_ctx *ss = op->ss; 24 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 25 struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); 26 u32 mode = ctx->mode; 27 /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ 28 u32 rx_cnt = SS_RX_DEFAULT; 29 u32 tx_cnt = 0; 30 u32 spaces; 31 u32 v; 32 int i, err = 0; 33 unsigned int ileft = areq->nbytes; 34 unsigned int oleft = areq->nbytes; 35 unsigned int todo; 36 struct sg_mapping_iter mi, mo; 37 unsigned int oi, oo; /* offset for in and out */ 38 unsigned long flags; 39 40 if (areq->nbytes == 0) 41 return 0; 42 43 if (!areq->info) { 44 dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); 45 return -EINVAL; 46 } 47 48 if (!areq->src || !areq->dst) { 49 dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); 50 return -EINVAL; 51 } 52 53 spin_lock_irqsave(&ss->slock, flags); 54 55 for (i = 0; i < op->keylen; i += 4) 56 writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); 57 58 if (areq->info) { 59 for (i = 0; i < 4 && i < ivsize / 4; i++) { 60 v = *(u32 *)(areq->info + i * 4); 61 writel(v, ss->base + SS_IV0 + i * 4); 62 } 63 } 64 writel(mode, ss->base + SS_CTL); 65 66 sg_miter_start(&mi, areq->src, sg_nents(areq->src), 67 SG_MITER_FROM_SG | SG_MITER_ATOMIC); 68 sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), 69 SG_MITER_TO_SG | SG_MITER_ATOMIC); 70 sg_miter_next(&mi); 71 sg_miter_next(&mo); 72 if (!mi.addr || !mo.addr) { 73 dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); 74 err = -EINVAL; 75 goto release_ss; 76 } 77 78 ileft = areq->nbytes / 4; 79 oleft = areq->nbytes / 4; 80 oi = 0; 81 oo = 0; 82 do { 83 todo = min3(rx_cnt, ileft, (mi.length - oi) / 4); 84 if (todo > 0) { 85 ileft -= todo; 86 writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); 87 oi += todo * 4; 88 } 89 if (oi == mi.length) { 90 sg_miter_next(&mi); 91 oi = 0; 92 } 93 94 spaces = readl(ss->base + SS_FCSR); 95 rx_cnt = SS_RXFIFO_SPACES(spaces); 96 tx_cnt = SS_TXFIFO_SPACES(spaces); 97 98 todo = min3(tx_cnt, oleft, (mo.length - oo) / 4); 99 if (todo > 0) { 100 oleft -= todo; 101 readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); 102 oo += todo * 4; 103 } 104 if (oo == mo.length) { 105 sg_miter_next(&mo); 106 oo = 0; 107 } 108 } while (oleft > 0); 109 110 if (areq->info) { 111 for (i = 0; i < 4 && i < ivsize / 4; i++) { 112 v = readl(ss->base + SS_IV0 + i * 4); 113 *(u32 *)(areq->info + i * 4) = v; 114 } 115 } 116 117release_ss: 118 sg_miter_stop(&mi); 119 sg_miter_stop(&mo); 120 writel(0, ss->base + SS_CTL); 121 spin_unlock_irqrestore(&ss->slock, flags); 122 return err; 123} 124 125/* Generic function that support SG with size not multiple of 4 */ 126static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) 127{ 128 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 129 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 130 struct sun4i_ss_ctx *ss = op->ss; 131 int no_chunk = 1; 132 struct scatterlist *in_sg = areq->src; 133 struct scatterlist *out_sg = areq->dst; 134 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 135 struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); 136 u32 mode = ctx->mode; 137 /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ 138 u32 rx_cnt = SS_RX_DEFAULT; 139 u32 tx_cnt = 0; 140 u32 v; 141 u32 spaces; 142 int i, err = 0; 143 unsigned int ileft = areq->nbytes; 144 unsigned int oleft = areq->nbytes; 145 unsigned int todo; 146 struct sg_mapping_iter mi, mo; 147 unsigned int oi, oo; /* offset for in and out */ 148 char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ 149 char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ 150 unsigned int ob = 0; /* offset in buf */ 151 unsigned int obo = 0; /* offset in bufo*/ 152 unsigned int obl = 0; /* length of data in bufo */ 153 unsigned long flags; 154 155 if (areq->nbytes == 0) 156 return 0; 157 158 if (!areq->info) { 159 dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); 160 return -EINVAL; 161 } 162 163 if (!areq->src || !areq->dst) { 164 dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); 165 return -EINVAL; 166 } 167 168 /* 169 * if we have only SGs with size multiple of 4, 170 * we can use the SS optimized function 171 */ 172 while (in_sg && no_chunk == 1) { 173 if ((in_sg->length % 4) != 0) 174 no_chunk = 0; 175 in_sg = sg_next(in_sg); 176 } 177 while (out_sg && no_chunk == 1) { 178 if ((out_sg->length % 4) != 0) 179 no_chunk = 0; 180 out_sg = sg_next(out_sg); 181 } 182 183 if (no_chunk == 1) 184 return sun4i_ss_opti_poll(areq); 185 186 spin_lock_irqsave(&ss->slock, flags); 187 188 for (i = 0; i < op->keylen; i += 4) 189 writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); 190 191 if (areq->info) { 192 for (i = 0; i < 4 && i < ivsize / 4; i++) { 193 v = *(u32 *)(areq->info + i * 4); 194 writel(v, ss->base + SS_IV0 + i * 4); 195 } 196 } 197 writel(mode, ss->base + SS_CTL); 198 199 sg_miter_start(&mi, areq->src, sg_nents(areq->src), 200 SG_MITER_FROM_SG | SG_MITER_ATOMIC); 201 sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), 202 SG_MITER_TO_SG | SG_MITER_ATOMIC); 203 sg_miter_next(&mi); 204 sg_miter_next(&mo); 205 if (!mi.addr || !mo.addr) { 206 dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); 207 err = -EINVAL; 208 goto release_ss; 209 } 210 ileft = areq->nbytes; 211 oleft = areq->nbytes; 212 oi = 0; 213 oo = 0; 214 215 while (oleft > 0) { 216 if (ileft > 0) { 217 /* 218 * todo is the number of consecutive 4byte word that we 219 * can read from current SG 220 */ 221 todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4); 222 if (todo > 0 && ob == 0) { 223 writesl(ss->base + SS_RXFIFO, mi.addr + oi, 224 todo); 225 ileft -= todo * 4; 226 oi += todo * 4; 227 } else { 228 /* 229 * not enough consecutive bytes, so we need to 230 * linearize in buf. todo is in bytes 231 * After that copy, if we have a multiple of 4 232 * we need to be able to write all buf in one 233 * pass, so it is why we min() with rx_cnt 234 */ 235 todo = min3(rx_cnt * 4 - ob, ileft, 236 mi.length - oi); 237 memcpy(buf + ob, mi.addr + oi, todo); 238 ileft -= todo; 239 oi += todo; 240 ob += todo; 241 if (ob % 4 == 0) { 242 writesl(ss->base + SS_RXFIFO, buf, 243 ob / 4); 244 ob = 0; 245 } 246 } 247 if (oi == mi.length) { 248 sg_miter_next(&mi); 249 oi = 0; 250 } 251 } 252 253 spaces = readl(ss->base + SS_FCSR); 254 rx_cnt = SS_RXFIFO_SPACES(spaces); 255 tx_cnt = SS_TXFIFO_SPACES(spaces); 256 dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u %u\n", 257 mode, 258 oi, mi.length, ileft, areq->nbytes, rx_cnt, 259 oo, mo.length, oleft, areq->nbytes, tx_cnt, 260 todo, ob); 261 262 if (tx_cnt == 0) 263 continue; 264 /* todo in 4bytes word */ 265 todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4); 266 if (todo > 0) { 267 readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); 268 oleft -= todo * 4; 269 oo += todo * 4; 270 if (oo == mo.length) { 271 sg_miter_next(&mo); 272 oo = 0; 273 } 274 } else { 275 /* 276 * read obl bytes in bufo, we read at maximum for 277 * emptying the device 278 */ 279 readsl(ss->base + SS_TXFIFO, bufo, tx_cnt); 280 obl = tx_cnt * 4; 281 obo = 0; 282 do { 283 /* 284 * how many bytes we can copy ? 285 * no more than remaining SG size 286 * no more than remaining buffer 287 * no need to test against oleft 288 */ 289 todo = min(mo.length - oo, obl - obo); 290 memcpy(mo.addr + oo, bufo + obo, todo); 291 oleft -= todo; 292 obo += todo; 293 oo += todo; 294 if (oo == mo.length) { 295 sg_miter_next(&mo); 296 oo = 0; 297 } 298 } while (obo < obl); 299 /* bufo must be fully used here */ 300 } 301 } 302 if (areq->info) { 303 for (i = 0; i < 4 && i < ivsize / 4; i++) { 304 v = readl(ss->base + SS_IV0 + i * 4); 305 *(u32 *)(areq->info + i * 4) = v; 306 } 307 } 308 309release_ss: 310 sg_miter_stop(&mi); 311 sg_miter_stop(&mo); 312 writel(0, ss->base + SS_CTL); 313 spin_unlock_irqrestore(&ss->slock, flags); 314 315 return err; 316} 317 318/* CBC AES */ 319int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq) 320{ 321 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 322 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 323 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 324 325 rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | 326 op->keymode; 327 return sun4i_ss_cipher_poll(areq); 328} 329 330int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq) 331{ 332 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 333 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 334 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 335 336 rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION | 337 op->keymode; 338 return sun4i_ss_cipher_poll(areq); 339} 340 341/* ECB AES */ 342int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq) 343{ 344 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 345 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 346 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 347 348 rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | 349 op->keymode; 350 return sun4i_ss_cipher_poll(areq); 351} 352 353int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq) 354{ 355 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 356 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 357 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 358 359 rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION | 360 op->keymode; 361 return sun4i_ss_cipher_poll(areq); 362} 363 364/* CBC DES */ 365int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq) 366{ 367 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 368 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 369 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 370 371 rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | 372 op->keymode; 373 return sun4i_ss_cipher_poll(areq); 374} 375 376int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq) 377{ 378 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 379 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 380 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 381 382 rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | 383 op->keymode; 384 return sun4i_ss_cipher_poll(areq); 385} 386 387/* ECB DES */ 388int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq) 389{ 390 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 391 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 392 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 393 394 rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | 395 op->keymode; 396 return sun4i_ss_cipher_poll(areq); 397} 398 399int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq) 400{ 401 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 402 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 403 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 404 405 rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | 406 op->keymode; 407 return sun4i_ss_cipher_poll(areq); 408} 409 410/* CBC 3DES */ 411int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq) 412{ 413 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 414 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 415 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 416 417 rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | 418 op->keymode; 419 return sun4i_ss_cipher_poll(areq); 420} 421 422int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq) 423{ 424 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 425 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 426 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 427 428 rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | 429 op->keymode; 430 return sun4i_ss_cipher_poll(areq); 431} 432 433/* ECB 3DES */ 434int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq) 435{ 436 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 437 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 438 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 439 440 rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | 441 op->keymode; 442 return sun4i_ss_cipher_poll(areq); 443} 444 445int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq) 446{ 447 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); 448 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 449 struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); 450 451 rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | 452 op->keymode; 453 return sun4i_ss_cipher_poll(areq); 454} 455 456int sun4i_ss_cipher_init(struct crypto_tfm *tfm) 457{ 458 struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); 459 struct crypto_alg *alg = tfm->__crt_alg; 460 struct sun4i_ss_alg_template *algt; 461 462 memset(op, 0, sizeof(struct sun4i_tfm_ctx)); 463 464 algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); 465 op->ss = algt->ss; 466 467 tfm->crt_ablkcipher.reqsize = sizeof(struct sun4i_cipher_req_ctx); 468 469 return 0; 470} 471 472/* check and set the AES key, prepare the mode to be used */ 473int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 474 unsigned int keylen) 475{ 476 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 477 struct sun4i_ss_ctx *ss = op->ss; 478 479 switch (keylen) { 480 case 128 / 8: 481 op->keymode = SS_AES_128BITS; 482 break; 483 case 192 / 8: 484 op->keymode = SS_AES_192BITS; 485 break; 486 case 256 / 8: 487 op->keymode = SS_AES_256BITS; 488 break; 489 default: 490 dev_err(ss->dev, "ERROR: Invalid keylen %u\n", keylen); 491 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 492 return -EINVAL; 493 } 494 op->keylen = keylen; 495 memcpy(op->key, key, keylen); 496 return 0; 497} 498 499/* check and set the DES key, prepare the mode to be used */ 500int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 501 unsigned int keylen) 502{ 503 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 504 struct sun4i_ss_ctx *ss = op->ss; 505 u32 flags; 506 u32 tmp[DES_EXPKEY_WORDS]; 507 int ret; 508 509 if (unlikely(keylen != DES_KEY_SIZE)) { 510 dev_err(ss->dev, "Invalid keylen %u\n", keylen); 511 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 512 return -EINVAL; 513 } 514 515 flags = crypto_ablkcipher_get_flags(tfm); 516 517 ret = des_ekey(tmp, key); 518 if (unlikely(ret == 0) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 519 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); 520 dev_dbg(ss->dev, "Weak key %u\n", keylen); 521 return -EINVAL; 522 } 523 524 op->keylen = keylen; 525 memcpy(op->key, key, keylen); 526 return 0; 527} 528 529/* check and set the 3DES key, prepare the mode to be used */ 530int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 531 unsigned int keylen) 532{ 533 struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); 534 struct sun4i_ss_ctx *ss = op->ss; 535 536 if (unlikely(keylen != 3 * DES_KEY_SIZE)) { 537 dev_err(ss->dev, "Invalid keylen %u\n", keylen); 538 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 539 return -EINVAL; 540 } 541 op->keylen = keylen; 542 memcpy(op->key, key, keylen); 543 return 0; 544} 545