1/* 2 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. 3 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) 4 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at: 5 * http://www.intel.com/products/processor/manuals/ 6 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual 7 * Volume 2A: Instruction Set Reference, A-M 8 * 9 * Copyright (C) 2008 Intel Corporation 10 * Authors: Austin Zhang <austin_zhang@linux.intel.com> 11 * Kent Liu <kent.liu@intel.com> 12 * 13 * This program is free software; you can redistribute it and/or modify it 14 * under the terms and conditions of the GNU General Public License, 15 * version 2, as published by the Free Software Foundation. 16 * 17 * This program is distributed in the hope it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 20 * more details. 21 * 22 * You should have received a copy of the GNU General Public License along with 23 * this program; if not, write to the Free Software Foundation, Inc., 24 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 25 * 26 */ 27#include <linux/init.h> 28#include <linux/module.h> 29#include <linux/string.h> 30#include <linux/kernel.h> 31#include <crypto/internal/hash.h> 32 33#include <asm/cpufeature.h> 34 35#define CHKSUM_BLOCK_SIZE 1 36#define CHKSUM_DIGEST_SIZE 4 37 38#define SCALE_F sizeof(unsigned long) 39 40#ifdef CONFIG_X86_64 41#define REX_PRE "0x48, " 42#else 43#define REX_PRE 44#endif 45 46static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length) 47{ 48 while (length--) { 49 __asm__ __volatile__( 50 ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" 51 :"=S"(crc) 52 :"0"(crc), "c"(*data) 53 ); 54 data++; 55 } 56 57 return crc; 58} 59 60static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len) 61{ 62 unsigned int iquotient = len / SCALE_F; 63 unsigned int iremainder = len % SCALE_F; 64 unsigned long *ptmp = (unsigned long *)p; 65 66 while (iquotient--) { 67 __asm__ __volatile__( 68 ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" 69 :"=S"(crc) 70 :"0"(crc), "c"(*ptmp) 71 ); 72 ptmp++; 73 } 74 75 if (iremainder) 76 crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp, 77 iremainder); 78 79 return crc; 80} 81 82/* 83 * Setting the seed allows arbitrary accumulators and flexible XOR policy 84 * If your algorithm starts with ~0, then XOR with ~0 before you set 85 * the seed. 86 */ 87static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key, 88 unsigned int keylen) 89{ 90 u32 *mctx = crypto_shash_ctx(hash); 91 92 if (keylen != sizeof(u32)) { 93 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 94 return -EINVAL; 95 } 96 *mctx = le32_to_cpup((__le32 *)key); 97 return 0; 98} 99 100static int crc32c_intel_init(struct shash_desc *desc) 101{ 102 u32 *mctx = crypto_shash_ctx(desc->tfm); 103 u32 *crcp = shash_desc_ctx(desc); 104 105 *crcp = *mctx; 106 107 return 0; 108} 109 110static int crc32c_intel_update(struct shash_desc *desc, const u8 *data, 111 unsigned int len) 112{ 113 u32 *crcp = shash_desc_ctx(desc); 114 115 *crcp = crc32c_intel_le_hw(*crcp, data, len); 116 return 0; 117} 118 119static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len, 120 u8 *out) 121{ 122 *(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); 123 return 0; 124} 125 126static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data, 127 unsigned int len, u8 *out) 128{ 129 return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out); 130} 131 132static int crc32c_intel_final(struct shash_desc *desc, u8 *out) 133{ 134 u32 *crcp = shash_desc_ctx(desc); 135 136 *(__le32 *)out = ~cpu_to_le32p(crcp); 137 return 0; 138} 139 140static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data, 141 unsigned int len, u8 *out) 142{ 143 return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len, 144 out); 145} 146 147static int crc32c_intel_cra_init(struct crypto_tfm *tfm) 148{ 149 u32 *key = crypto_tfm_ctx(tfm); 150 151 *key = ~0; 152 153 return 0; 154} 155 156static struct shash_alg alg = { 157 .setkey = crc32c_intel_setkey, 158 .init = crc32c_intel_init, 159 .update = crc32c_intel_update, 160 .final = crc32c_intel_final, 161 .finup = crc32c_intel_finup, 162 .digest = crc32c_intel_digest, 163 .descsize = sizeof(u32), 164 .digestsize = CHKSUM_DIGEST_SIZE, 165 .base = { 166 .cra_name = "crc32c", 167 .cra_driver_name = "crc32c-intel", 168 .cra_priority = 200, 169 .cra_blocksize = CHKSUM_BLOCK_SIZE, 170 .cra_ctxsize = sizeof(u32), 171 .cra_module = THIS_MODULE, 172 .cra_init = crc32c_intel_cra_init, 173 } 174}; 175 176 177static int __init crc32c_intel_mod_init(void) 178{ 179 if (cpu_has_xmm4_2) 180 return crypto_register_shash(&alg); 181 else 182 return -ENODEV; 183} 184 185static void __exit crc32c_intel_mod_fini(void) 186{ 187 crypto_unregister_shash(&alg); 188} 189 190module_init(crc32c_intel_mod_init); 191module_exit(crc32c_intel_mod_fini); 192 193MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>"); 194MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware."); 195MODULE_LICENSE("GPL"); 196 197MODULE_ALIAS("crc32c"); 198MODULE_ALIAS("crc32c-intel"); 199