1/* 2 * Shared glue code for 128bit block ciphers 3 * 4 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 5 * 6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 8 * CTR part based on code (crypto/ctr.c) by: 9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 24 * USA 25 * 26 */ 27 28#include <linux/module.h> 29#include <crypto/b128ops.h> 30#include <crypto/lrw.h> 31#include <crypto/xts.h> 32#include <asm/crypto/glue_helper.h> 33#include <crypto/scatterwalk.h> 34 35static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, 36 struct blkcipher_desc *desc, 37 struct blkcipher_walk *walk) 38{ 39 void *ctx = crypto_blkcipher_ctx(desc->tfm); 40 const unsigned int bsize = 128 / 8; 41 unsigned int nbytes, i, func_bytes; 42 bool fpu_enabled = false; 43 int err; 44 45 err = blkcipher_walk_virt(desc, walk); 46 47 while ((nbytes = walk->nbytes)) { 48 u8 *wsrc = walk->src.virt.addr; 49 u8 *wdst = walk->dst.virt.addr; 50 51 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, 52 desc, fpu_enabled, nbytes); 53 54 for (i = 0; i < gctx->num_funcs; i++) { 55 func_bytes = bsize * gctx->funcs[i].num_blocks; 56 57 /* Process multi-block batch */ 58 if (nbytes >= func_bytes) { 59 do { 60 gctx->funcs[i].fn_u.ecb(ctx, wdst, 61 wsrc); 62 63 wsrc += func_bytes; 64 wdst += func_bytes; 65 nbytes -= func_bytes; 66 } while (nbytes >= func_bytes); 67 68 if (nbytes < bsize) 69 goto done; 70 } 71 } 72 73done: 74 err = blkcipher_walk_done(desc, walk, nbytes); 75 } 76 77 glue_fpu_end(fpu_enabled); 78 return err; 79} 80 81int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, 82 struct blkcipher_desc *desc, struct scatterlist *dst, 83 struct scatterlist *src, unsigned int nbytes) 84{ 85 struct blkcipher_walk walk; 86 87 blkcipher_walk_init(&walk, dst, src, nbytes); 88 return __glue_ecb_crypt_128bit(gctx, desc, &walk); 89} 90EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit); 91 92static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn, 93 struct blkcipher_desc *desc, 94 struct blkcipher_walk *walk) 95{ 96 void *ctx = crypto_blkcipher_ctx(desc->tfm); 97 const unsigned int bsize = 128 / 8; 98 unsigned int nbytes = walk->nbytes; 99 u128 *src = (u128 *)walk->src.virt.addr; 100 u128 *dst = (u128 *)walk->dst.virt.addr; 101 u128 *iv = (u128 *)walk->iv; 102 103 do { 104 u128_xor(dst, src, iv); 105 fn(ctx, (u8 *)dst, (u8 *)dst); 106 iv = dst; 107 108 src += 1; 109 dst += 1; 110 nbytes -= bsize; 111 } while (nbytes >= bsize); 112 113 *(u128 *)walk->iv = *iv; 114 return nbytes; 115} 116 117int glue_cbc_encrypt_128bit(const common_glue_func_t fn, 118 struct blkcipher_desc *desc, 119 struct scatterlist *dst, 120 struct scatterlist *src, unsigned int nbytes) 121{ 122 struct blkcipher_walk walk; 123 int err; 124 125 blkcipher_walk_init(&walk, dst, src, nbytes); 126 err = blkcipher_walk_virt(desc, &walk); 127 128 while ((nbytes = walk.nbytes)) { 129 nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk); 130 err = blkcipher_walk_done(desc, &walk, nbytes); 131 } 132 133 return err; 134} 135EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit); 136 137static unsigned int 138__glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx, 139 struct blkcipher_desc *desc, 140 struct blkcipher_walk *walk) 141{ 142 void *ctx = crypto_blkcipher_ctx(desc->tfm); 143 const unsigned int bsize = 128 / 8; 144 unsigned int nbytes = walk->nbytes; 145 u128 *src = (u128 *)walk->src.virt.addr; 146 u128 *dst = (u128 *)walk->dst.virt.addr; 147 u128 last_iv; 148 unsigned int num_blocks, func_bytes; 149 unsigned int i; 150 151 /* Start of the last block. */ 152 src += nbytes / bsize - 1; 153 dst += nbytes / bsize - 1; 154 155 last_iv = *src; 156 157 for (i = 0; i < gctx->num_funcs; i++) { 158 num_blocks = gctx->funcs[i].num_blocks; 159 func_bytes = bsize * num_blocks; 160 161 /* Process multi-block batch */ 162 if (nbytes >= func_bytes) { 163 do { 164 nbytes -= func_bytes - bsize; 165 src -= num_blocks - 1; 166 dst -= num_blocks - 1; 167 168 gctx->funcs[i].fn_u.cbc(ctx, dst, src); 169 170 nbytes -= bsize; 171 if (nbytes < bsize) 172 goto done; 173 174 u128_xor(dst, dst, src - 1); 175 src -= 1; 176 dst -= 1; 177 } while (nbytes >= func_bytes); 178 179 if (nbytes < bsize) 180 goto done; 181 } 182 } 183 184done: 185 u128_xor(dst, dst, (u128 *)walk->iv); 186 *(u128 *)walk->iv = last_iv; 187 188 return nbytes; 189} 190 191int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx, 192 struct blkcipher_desc *desc, 193 struct scatterlist *dst, 194 struct scatterlist *src, unsigned int nbytes) 195{ 196 const unsigned int bsize = 128 / 8; 197 bool fpu_enabled = false; 198 struct blkcipher_walk walk; 199 int err; 200 201 blkcipher_walk_init(&walk, dst, src, nbytes); 202 err = blkcipher_walk_virt(desc, &walk); 203 204 while ((nbytes = walk.nbytes)) { 205 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, 206 desc, fpu_enabled, nbytes); 207 nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk); 208 err = blkcipher_walk_done(desc, &walk, nbytes); 209 } 210 211 glue_fpu_end(fpu_enabled); 212 return err; 213} 214EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit); 215 216static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr, 217 struct blkcipher_desc *desc, 218 struct blkcipher_walk *walk) 219{ 220 void *ctx = crypto_blkcipher_ctx(desc->tfm); 221 u8 *src = (u8 *)walk->src.virt.addr; 222 u8 *dst = (u8 *)walk->dst.virt.addr; 223 unsigned int nbytes = walk->nbytes; 224 le128 ctrblk; 225 u128 tmp; 226 227 be128_to_le128(&ctrblk, (be128 *)walk->iv); 228 229 memcpy(&tmp, src, nbytes); 230 fn_ctr(ctx, &tmp, &tmp, &ctrblk); 231 memcpy(dst, &tmp, nbytes); 232 233 le128_to_be128((be128 *)walk->iv, &ctrblk); 234} 235 236static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, 237 struct blkcipher_desc *desc, 238 struct blkcipher_walk *walk) 239{ 240 const unsigned int bsize = 128 / 8; 241 void *ctx = crypto_blkcipher_ctx(desc->tfm); 242 unsigned int nbytes = walk->nbytes; 243 u128 *src = (u128 *)walk->src.virt.addr; 244 u128 *dst = (u128 *)walk->dst.virt.addr; 245 le128 ctrblk; 246 unsigned int num_blocks, func_bytes; 247 unsigned int i; 248 249 be128_to_le128(&ctrblk, (be128 *)walk->iv); 250 251 /* Process multi-block batch */ 252 for (i = 0; i < gctx->num_funcs; i++) { 253 num_blocks = gctx->funcs[i].num_blocks; 254 func_bytes = bsize * num_blocks; 255 256 if (nbytes >= func_bytes) { 257 do { 258 gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk); 259 260 src += num_blocks; 261 dst += num_blocks; 262 nbytes -= func_bytes; 263 } while (nbytes >= func_bytes); 264 265 if (nbytes < bsize) 266 goto done; 267 } 268 } 269 270done: 271 le128_to_be128((be128 *)walk->iv, &ctrblk); 272 return nbytes; 273} 274 275int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, 276 struct blkcipher_desc *desc, struct scatterlist *dst, 277 struct scatterlist *src, unsigned int nbytes) 278{ 279 const unsigned int bsize = 128 / 8; 280 bool fpu_enabled = false; 281 struct blkcipher_walk walk; 282 int err; 283 284 blkcipher_walk_init(&walk, dst, src, nbytes); 285 err = blkcipher_walk_virt_block(desc, &walk, bsize); 286 287 while ((nbytes = walk.nbytes) >= bsize) { 288 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, 289 desc, fpu_enabled, nbytes); 290 nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk); 291 err = blkcipher_walk_done(desc, &walk, nbytes); 292 } 293 294 glue_fpu_end(fpu_enabled); 295 296 if (walk.nbytes) { 297 glue_ctr_crypt_final_128bit( 298 gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk); 299 err = blkcipher_walk_done(desc, &walk, 0); 300 } 301 302 return err; 303} 304EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit); 305 306static unsigned int __glue_xts_crypt_128bit(const struct common_glue_ctx *gctx, 307 void *ctx, 308 struct blkcipher_desc *desc, 309 struct blkcipher_walk *walk) 310{ 311 const unsigned int bsize = 128 / 8; 312 unsigned int nbytes = walk->nbytes; 313 u128 *src = (u128 *)walk->src.virt.addr; 314 u128 *dst = (u128 *)walk->dst.virt.addr; 315 unsigned int num_blocks, func_bytes; 316 unsigned int i; 317 318 /* Process multi-block batch */ 319 for (i = 0; i < gctx->num_funcs; i++) { 320 num_blocks = gctx->funcs[i].num_blocks; 321 func_bytes = bsize * num_blocks; 322 323 if (nbytes >= func_bytes) { 324 do { 325 gctx->funcs[i].fn_u.xts(ctx, dst, src, 326 (le128 *)walk->iv); 327 328 src += num_blocks; 329 dst += num_blocks; 330 nbytes -= func_bytes; 331 } while (nbytes >= func_bytes); 332 333 if (nbytes < bsize) 334 goto done; 335 } 336 } 337 338done: 339 return nbytes; 340} 341 342/* for implementations implementing faster XTS IV generator */ 343int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx, 344 struct blkcipher_desc *desc, struct scatterlist *dst, 345 struct scatterlist *src, unsigned int nbytes, 346 void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src), 347 void *tweak_ctx, void *crypt_ctx) 348{ 349 const unsigned int bsize = 128 / 8; 350 bool fpu_enabled = false; 351 struct blkcipher_walk walk; 352 int err; 353 354 blkcipher_walk_init(&walk, dst, src, nbytes); 355 356 err = blkcipher_walk_virt(desc, &walk); 357 nbytes = walk.nbytes; 358 if (!nbytes) 359 return err; 360 361 /* set minimum length to bsize, for tweak_fn */ 362 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, 363 desc, fpu_enabled, 364 nbytes < bsize ? bsize : nbytes); 365 366 /* calculate first value of T */ 367 tweak_fn(tweak_ctx, walk.iv, walk.iv); 368 369 while (nbytes) { 370 nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk); 371 372 err = blkcipher_walk_done(desc, &walk, nbytes); 373 nbytes = walk.nbytes; 374 } 375 376 glue_fpu_end(fpu_enabled); 377 378 return err; 379} 380EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit); 381 382void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv, 383 common_glue_func_t fn) 384{ 385 le128 ivblk = *iv; 386 387 /* generate next IV */ 388 le128_gf128mul_x_ble(iv, &ivblk); 389 390 /* CC <- T xor C */ 391 u128_xor(dst, src, (u128 *)&ivblk); 392 393 /* PP <- D(Key2,CC) */ 394 fn(ctx, (u8 *)dst, (u8 *)dst); 395 396 /* P <- T xor PP */ 397 u128_xor(dst, dst, (u128 *)&ivblk); 398} 399EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one); 400 401MODULE_LICENSE("GPL"); 402