1/* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA256 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha256_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation. 10 * 11 * Author: 12 * Tim Chen <tim.c.chen@linux.intel.com> 13 * 14 * This program is free software; you can redistribute it and/or modify it 15 * under the terms of the GNU General Public License as published by the Free 16 * Software Foundation; either version 2 of the License, or (at your option) 17 * any later version. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 * SOFTWARE. 27 */ 28 29 30#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 31 32#include <crypto/internal/hash.h> 33#include <linux/init.h> 34#include <linux/module.h> 35#include <linux/mm.h> 36#include <linux/cryptohash.h> 37#include <linux/types.h> 38#include <crypto/sha.h> 39#include <crypto/sha256_base.h> 40#include <asm/i387.h> 41#include <asm/xcr.h> 42#include <asm/xsave.h> 43#include <linux/string.h> 44 45asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data, 46 u64 rounds); 47#ifdef CONFIG_AS_AVX 48asmlinkage void sha256_transform_avx(u32 *digest, const char *data, 49 u64 rounds); 50#endif 51#ifdef CONFIG_AS_AVX2 52asmlinkage void sha256_transform_rorx(u32 *digest, const char *data, 53 u64 rounds); 54#endif 55 56static void (*sha256_transform_asm)(u32 *, const char *, u64); 57 58static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, 59 unsigned int len) 60{ 61 struct sha256_state *sctx = shash_desc_ctx(desc); 62 63 if (!irq_fpu_usable() || 64 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) 65 return crypto_sha256_update(desc, data, len); 66 67 /* make sure casting to sha256_block_fn() is safe */ 68 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); 69 70 kernel_fpu_begin(); 71 sha256_base_do_update(desc, data, len, 72 (sha256_block_fn *)sha256_transform_asm); 73 kernel_fpu_end(); 74 75 return 0; 76} 77 78static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, 79 unsigned int len, u8 *out) 80{ 81 if (!irq_fpu_usable()) 82 return crypto_sha256_finup(desc, data, len, out); 83 84 kernel_fpu_begin(); 85 if (len) 86 sha256_base_do_update(desc, data, len, 87 (sha256_block_fn *)sha256_transform_asm); 88 sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_transform_asm); 89 kernel_fpu_end(); 90 91 return sha256_base_finish(desc, out); 92} 93 94/* Add padding and return the message digest. */ 95static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) 96{ 97 return sha256_ssse3_finup(desc, NULL, 0, out); 98} 99 100static struct shash_alg algs[] = { { 101 .digestsize = SHA256_DIGEST_SIZE, 102 .init = sha256_base_init, 103 .update = sha256_ssse3_update, 104 .final = sha256_ssse3_final, 105 .finup = sha256_ssse3_finup, 106 .descsize = sizeof(struct sha256_state), 107 .base = { 108 .cra_name = "sha256", 109 .cra_driver_name = "sha256-ssse3", 110 .cra_priority = 150, 111 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 112 .cra_blocksize = SHA256_BLOCK_SIZE, 113 .cra_module = THIS_MODULE, 114 } 115}, { 116 .digestsize = SHA224_DIGEST_SIZE, 117 .init = sha224_base_init, 118 .update = sha256_ssse3_update, 119 .final = sha256_ssse3_final, 120 .finup = sha256_ssse3_finup, 121 .descsize = sizeof(struct sha256_state), 122 .base = { 123 .cra_name = "sha224", 124 .cra_driver_name = "sha224-ssse3", 125 .cra_priority = 150, 126 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 127 .cra_blocksize = SHA224_BLOCK_SIZE, 128 .cra_module = THIS_MODULE, 129 } 130} }; 131 132#ifdef CONFIG_AS_AVX 133static bool __init avx_usable(void) 134{ 135 u64 xcr0; 136 137 if (!cpu_has_avx || !cpu_has_osxsave) 138 return false; 139 140 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 141 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { 142 pr_info("AVX detected but unusable.\n"); 143 144 return false; 145 } 146 147 return true; 148} 149#endif 150 151static int __init sha256_ssse3_mod_init(void) 152{ 153 /* test for SSSE3 first */ 154 if (cpu_has_ssse3) 155 sha256_transform_asm = sha256_transform_ssse3; 156 157#ifdef CONFIG_AS_AVX 158 /* allow AVX to override SSSE3, it's a little faster */ 159 if (avx_usable()) { 160#ifdef CONFIG_AS_AVX2 161 if (boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2)) 162 sha256_transform_asm = sha256_transform_rorx; 163 else 164#endif 165 sha256_transform_asm = sha256_transform_avx; 166 } 167#endif 168 169 if (sha256_transform_asm) { 170#ifdef CONFIG_AS_AVX 171 if (sha256_transform_asm == sha256_transform_avx) 172 pr_info("Using AVX optimized SHA-256 implementation\n"); 173#ifdef CONFIG_AS_AVX2 174 else if (sha256_transform_asm == sha256_transform_rorx) 175 pr_info("Using AVX2 optimized SHA-256 implementation\n"); 176#endif 177 else 178#endif 179 pr_info("Using SSSE3 optimized SHA-256 implementation\n"); 180 return crypto_register_shashes(algs, ARRAY_SIZE(algs)); 181 } 182 pr_info("Neither AVX nor SSSE3 is available/usable.\n"); 183 184 return -ENODEV; 185} 186 187static void __exit sha256_ssse3_mod_fini(void) 188{ 189 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 190} 191 192module_init(sha256_ssse3_mod_init); 193module_exit(sha256_ssse3_mod_fini); 194 195MODULE_LICENSE("GPL"); 196MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 197 198MODULE_ALIAS_CRYPTO("sha256"); 199MODULE_ALIAS_CRYPTO("sha224"); 200