1/* 2 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. 3 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) 4 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at: 5 * http://www.intel.com/products/processor/manuals/ 6 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual 7 * Volume 2A: Instruction Set Reference, A-M 8 * 9 * Copyright (C) 2008 Intel Corporation 10 * Authors: Austin Zhang <austin_zhang@linux.intel.com> 11 * Kent Liu <kent.liu@intel.com> 12 * 13 * This program is free software; you can redistribute it and/or modify it 14 * under the terms and conditions of the GNU General Public License, 15 * version 2, as published by the Free Software Foundation. 16 * 17 * This program is distributed in the hope it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 20 * more details. 21 * 22 * You should have received a copy of the GNU General Public License along with 23 * this program; if not, write to the Free Software Foundation, Inc., 24 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 25 * 26 */ 27#include <linux/init.h> 28#include <linux/module.h> 29#include <linux/string.h> 30#include <linux/kernel.h> 31#include <crypto/internal/hash.h> 32 33#include <asm/cpufeature.h> 34#include <asm/cpu_device_id.h> 35#include <asm/fpu/internal.h> 36 37#define CHKSUM_BLOCK_SIZE 1 38#define CHKSUM_DIGEST_SIZE 4 39 40#define SCALE_F sizeof(unsigned long) 41 42#ifdef CONFIG_X86_64 43#define REX_PRE "0x48, " 44#else 45#define REX_PRE 46#endif 47 48#ifdef CONFIG_X86_64 49/* 50 * use carryless multiply version of crc32c when buffer 51 * size is >= 512 (when eager fpu is enabled) or 52 * >= 1024 (when eager fpu is disabled) to account 53 * for fpu state save/restore overhead. 54 */ 55#define CRC32C_PCL_BREAKEVEN_EAGERFPU 512 56#define CRC32C_PCL_BREAKEVEN_NOEAGERFPU 1024 57 58asmlinkage unsigned int crc_pcl(const u8 *buffer, int len, 59 unsigned int crc_init); 60static int crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_EAGERFPU; 61#if defined(X86_FEATURE_EAGER_FPU) 62#define set_pcl_breakeven_point() \ 63do { \ 64 if (!use_eager_fpu()) \ 65 crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_NOEAGERFPU; \ 66} while (0) 67#else 68#define set_pcl_breakeven_point() \ 69 (crc32c_pcl_breakeven = CRC32C_PCL_BREAKEVEN_NOEAGERFPU) 70#endif 71#endif /* CONFIG_X86_64 */ 72 73static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length) 74{ 75 while (length--) { 76 __asm__ __volatile__( 77 ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" 78 :"=S"(crc) 79 :"0"(crc), "c"(*data) 80 ); 81 data++; 82 } 83 84 return crc; 85} 86 87static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len) 88{ 89 unsigned int iquotient = len / SCALE_F; 90 unsigned int iremainder = len % SCALE_F; 91 unsigned long *ptmp = (unsigned long *)p; 92 93 while (iquotient--) { 94 __asm__ __volatile__( 95 ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" 96 :"=S"(crc) 97 :"0"(crc), "c"(*ptmp) 98 ); 99 ptmp++; 100 } 101 102 if (iremainder) 103 crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp, 104 iremainder); 105 106 return crc; 107} 108 109/* 110 * Setting the seed allows arbitrary accumulators and flexible XOR policy 111 * If your algorithm starts with ~0, then XOR with ~0 before you set 112 * the seed. 113 */ 114static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key, 115 unsigned int keylen) 116{ 117 u32 *mctx = crypto_shash_ctx(hash); 118 119 if (keylen != sizeof(u32)) { 120 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 121 return -EINVAL; 122 } 123 *mctx = le32_to_cpup((__le32 *)key); 124 return 0; 125} 126 127static int crc32c_intel_init(struct shash_desc *desc) 128{ 129 u32 *mctx = crypto_shash_ctx(desc->tfm); 130 u32 *crcp = shash_desc_ctx(desc); 131 132 *crcp = *mctx; 133 134 return 0; 135} 136 137static int crc32c_intel_update(struct shash_desc *desc, const u8 *data, 138 unsigned int len) 139{ 140 u32 *crcp = shash_desc_ctx(desc); 141 142 *crcp = crc32c_intel_le_hw(*crcp, data, len); 143 return 0; 144} 145 146static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len, 147 u8 *out) 148{ 149 *(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); 150 return 0; 151} 152 153static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data, 154 unsigned int len, u8 *out) 155{ 156 return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out); 157} 158 159static int crc32c_intel_final(struct shash_desc *desc, u8 *out) 160{ 161 u32 *crcp = shash_desc_ctx(desc); 162 163 *(__le32 *)out = ~cpu_to_le32p(crcp); 164 return 0; 165} 166 167static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data, 168 unsigned int len, u8 *out) 169{ 170 return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len, 171 out); 172} 173 174static int crc32c_intel_cra_init(struct crypto_tfm *tfm) 175{ 176 u32 *key = crypto_tfm_ctx(tfm); 177 178 *key = ~0; 179 180 return 0; 181} 182 183#ifdef CONFIG_X86_64 184static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data, 185 unsigned int len) 186{ 187 u32 *crcp = shash_desc_ctx(desc); 188 189 /* 190 * use faster PCL version if datasize is large enough to 191 * overcome kernel fpu state save/restore overhead 192 */ 193 if (len >= crc32c_pcl_breakeven && irq_fpu_usable()) { 194 kernel_fpu_begin(); 195 *crcp = crc_pcl(data, len, *crcp); 196 kernel_fpu_end(); 197 } else 198 *crcp = crc32c_intel_le_hw(*crcp, data, len); 199 return 0; 200} 201 202static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len, 203 u8 *out) 204{ 205 if (len >= crc32c_pcl_breakeven && irq_fpu_usable()) { 206 kernel_fpu_begin(); 207 *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp)); 208 kernel_fpu_end(); 209 } else 210 *(__le32 *)out = 211 ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); 212 return 0; 213} 214 215static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data, 216 unsigned int len, u8 *out) 217{ 218 return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out); 219} 220 221static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data, 222 unsigned int len, u8 *out) 223{ 224 return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len, 225 out); 226} 227#endif /* CONFIG_X86_64 */ 228 229static struct shash_alg alg = { 230 .setkey = crc32c_intel_setkey, 231 .init = crc32c_intel_init, 232 .update = crc32c_intel_update, 233 .final = crc32c_intel_final, 234 .finup = crc32c_intel_finup, 235 .digest = crc32c_intel_digest, 236 .descsize = sizeof(u32), 237 .digestsize = CHKSUM_DIGEST_SIZE, 238 .base = { 239 .cra_name = "crc32c", 240 .cra_driver_name = "crc32c-intel", 241 .cra_priority = 200, 242 .cra_blocksize = CHKSUM_BLOCK_SIZE, 243 .cra_ctxsize = sizeof(u32), 244 .cra_module = THIS_MODULE, 245 .cra_init = crc32c_intel_cra_init, 246 } 247}; 248 249static const struct x86_cpu_id crc32c_cpu_id[] = { 250 X86_FEATURE_MATCH(X86_FEATURE_XMM4_2), 251 {} 252}; 253MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id); 254 255static int __init crc32c_intel_mod_init(void) 256{ 257 if (!x86_match_cpu(crc32c_cpu_id)) 258 return -ENODEV; 259#ifdef CONFIG_X86_64 260 if (cpu_has_pclmulqdq) { 261 alg.update = crc32c_pcl_intel_update; 262 alg.finup = crc32c_pcl_intel_finup; 263 alg.digest = crc32c_pcl_intel_digest; 264 set_pcl_breakeven_point(); 265 } 266#endif 267 return crypto_register_shash(&alg); 268} 269 270static void __exit crc32c_intel_mod_fini(void) 271{ 272 crypto_unregister_shash(&alg); 273} 274 275module_init(crc32c_intel_mod_init); 276module_exit(crc32c_intel_mod_fini); 277 278MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>"); 279MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware."); 280MODULE_LICENSE("GPL"); 281 282MODULE_ALIAS_CRYPTO("crc32c"); 283MODULE_ALIAS_CRYPTO("crc32c-intel"); 284