152284Sobrien/* SPDX-License-Identifier: GPL-2.0-or-later */ 250397Sobrien/* 352284Sobrien * Glue Code for the AVX512/GFNI assembler implementation of the ARIA Cipher 450397Sobrien * 550397Sobrien * Copyright (c) 2022 Taehee Yoo <ap420073@gmail.com> 650397Sobrien */ 750397Sobrien 850397Sobrien#include <crypto/algapi.h> 950397Sobrien#include <crypto/internal/simd.h> 1050397Sobrien#include <crypto/aria.h> 1150397Sobrien#include <linux/crypto.h> 1250397Sobrien#include <linux/err.h> 1350397Sobrien#include <linux/module.h> 1450397Sobrien#include <linux/types.h> 1550397Sobrien 1650397Sobrien#include "ecb_cbc_helpers.h" 1750397Sobrien#include "aria-avx.h" 1850397Sobrien 1950397Sobrienasmlinkage void aria_gfni_avx512_encrypt_64way(const void *ctx, u8 *dst, 2050397Sobrien const u8 *src); 2150397Sobrienasmlinkage void aria_gfni_avx512_decrypt_64way(const void *ctx, u8 *dst, 2250397Sobrien const u8 *src); 2350397Sobrienasmlinkage void aria_gfni_avx512_ctr_crypt_64way(const void *ctx, u8 *dst, 2450397Sobrien const u8 *src, 2550397Sobrien u8 *keystream, u8 *iv); 2650397Sobrien 2752284Sobrienstatic struct aria_avx_ops aria_ops; 2852284Sobrien 2952284Sobrienstruct aria_avx512_request_ctx { 3052284Sobrien u8 keystream[ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE]; 3150397Sobrien}; 3250397Sobrien 3350397Sobrienstatic int ecb_do_encrypt(struct skcipher_request *req, const u32 *rkey) 3450397Sobrien{ 3550397Sobrien ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); 3650397Sobrien ECB_BLOCK(ARIA_GFNI_AVX512_PARALLEL_BLOCKS, aria_ops.aria_encrypt_64way); 3750397Sobrien ECB_BLOCK(ARIA_AESNI_AVX2_PARALLEL_BLOCKS, aria_ops.aria_encrypt_32way); 3850397Sobrien ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_encrypt_16way); 3950397Sobrien ECB_BLOCK(1, aria_encrypt); 4050397Sobrien ECB_WALK_END(); 4150397Sobrien} 4250397Sobrien 4350397Sobrienstatic int ecb_do_decrypt(struct skcipher_request *req, const u32 *rkey) 4450397Sobrien{ 4550397Sobrien ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); 4650397Sobrien ECB_BLOCK(ARIA_GFNI_AVX512_PARALLEL_BLOCKS, aria_ops.aria_decrypt_64way); 4750397Sobrien ECB_BLOCK(ARIA_AESNI_AVX2_PARALLEL_BLOCKS, aria_ops.aria_decrypt_32way); 4850397Sobrien ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_decrypt_16way); 4950397Sobrien ECB_BLOCK(1, aria_decrypt); 5050397Sobrien ECB_WALK_END(); 5150397Sobrien} 5250397Sobrien 5350397Sobrienstatic int aria_avx512_ecb_encrypt(struct skcipher_request *req) 5450397Sobrien{ 5550397Sobrien struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 5650397Sobrien struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); 5750397Sobrien 5850397Sobrien return ecb_do_encrypt(req, ctx->enc_key[0]); 5950397Sobrien} 6050397Sobrien 6150397Sobrienstatic int aria_avx512_ecb_decrypt(struct skcipher_request *req) 6250397Sobrien{ 6350397Sobrien struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 6450397Sobrien struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); 6550397Sobrien 6650397Sobrien return ecb_do_decrypt(req, ctx->dec_key[0]); 6750397Sobrien} 6850397Sobrien 6950397Sobrienstatic int aria_avx512_set_key(struct crypto_skcipher *tfm, const u8 *key, 7050397Sobrien unsigned int keylen) 7150397Sobrien{ 7250397Sobrien return aria_set_key(&tfm->base, key, keylen); 7350397Sobrien} 7450397Sobrien 7550397Sobrienstatic int aria_avx512_ctr_encrypt(struct skcipher_request *req) 7650397Sobrien{ 7750397Sobrien struct aria_avx512_request_ctx *req_ctx = skcipher_request_ctx(req); 7850397Sobrien struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 7950397Sobrien struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); 8050397Sobrien struct skcipher_walk walk; 8150397Sobrien unsigned int nbytes; 8250397Sobrien int err; 8350397Sobrien 8450397Sobrien err = skcipher_walk_virt(&walk, req, false); 8550397Sobrien 8650397Sobrien while ((nbytes = walk.nbytes) > 0) { 8750397Sobrien const u8 *src = walk.src.virt.addr; 8850397Sobrien u8 *dst = walk.dst.virt.addr; 8950397Sobrien 9050397Sobrien while (nbytes >= ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE) { 9150397Sobrien kernel_fpu_begin(); 9250397Sobrien aria_ops.aria_ctr_crypt_64way(ctx, dst, src, 9350397Sobrien &req_ctx->keystream[0], 9450397Sobrien walk.iv); 9550397Sobrien kernel_fpu_end(); 9650397Sobrien dst += ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE; 9750397Sobrien src += ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE; 9850397Sobrien nbytes -= ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE; 9950397Sobrien } 10050397Sobrien 10150397Sobrien while (nbytes >= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE) { 10250397Sobrien kernel_fpu_begin(); 10350397Sobrien aria_ops.aria_ctr_crypt_32way(ctx, dst, src, 10450397Sobrien &req_ctx->keystream[0], 10550397Sobrien walk.iv); 10650397Sobrien kernel_fpu_end(); 10750397Sobrien dst += ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE; 10850397Sobrien src += ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE; 10950397Sobrien nbytes -= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE; 11050397Sobrien } 11150397Sobrien 11250397Sobrien while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) { 11350397Sobrien kernel_fpu_begin(); 11450397Sobrien aria_ops.aria_ctr_crypt_16way(ctx, dst, src, 11550397Sobrien &req_ctx->keystream[0], 11650397Sobrien walk.iv); 11750397Sobrien kernel_fpu_end(); 11850397Sobrien dst += ARIA_AESNI_PARALLEL_BLOCK_SIZE; 11950397Sobrien src += ARIA_AESNI_PARALLEL_BLOCK_SIZE; 12050397Sobrien nbytes -= ARIA_AESNI_PARALLEL_BLOCK_SIZE; 12150397Sobrien } 12250397Sobrien 12350397Sobrien while (nbytes >= ARIA_BLOCK_SIZE) { 12450397Sobrien memcpy(&req_ctx->keystream[0], walk.iv, 12552284Sobrien ARIA_BLOCK_SIZE); 12652284Sobrien crypto_inc(walk.iv, ARIA_BLOCK_SIZE); 12752284Sobrien 12852284Sobrien aria_encrypt(ctx, &req_ctx->keystream[0], 12952284Sobrien &req_ctx->keystream[0]); 13052284Sobrien 13152284Sobrien crypto_xor_cpy(dst, src, &req_ctx->keystream[0], 13252284Sobrien ARIA_BLOCK_SIZE); 13352284Sobrien dst += ARIA_BLOCK_SIZE; 13452284Sobrien src += ARIA_BLOCK_SIZE; 13552284Sobrien nbytes -= ARIA_BLOCK_SIZE; 13652284Sobrien } 13752284Sobrien 13850397Sobrien if (walk.nbytes == walk.total && nbytes > 0) { 13950397Sobrien memcpy(&req_ctx->keystream[0], walk.iv, 14050397Sobrien ARIA_BLOCK_SIZE); 14150397Sobrien crypto_inc(walk.iv, ARIA_BLOCK_SIZE); 14250397Sobrien 14350397Sobrien aria_encrypt(ctx, &req_ctx->keystream[0], 14452284Sobrien &req_ctx->keystream[0]); 14550397Sobrien 14650397Sobrien crypto_xor_cpy(dst, src, &req_ctx->keystream[0], 14750397Sobrien nbytes); 14850397Sobrien dst += nbytes; 14950397Sobrien src += nbytes; 15050397Sobrien nbytes = 0; 15150397Sobrien } 15250397Sobrien err = skcipher_walk_done(&walk, nbytes); 15350397Sobrien } 15450397Sobrien 15552284Sobrien return err; 15650397Sobrien} 15750397Sobrien 15850397Sobrienstatic int aria_avx512_init_tfm(struct crypto_skcipher *tfm) 15950397Sobrien{ 16050397Sobrien crypto_skcipher_set_reqsize(tfm, 16150397Sobrien sizeof(struct aria_avx512_request_ctx)); 16250397Sobrien 16350397Sobrien return 0; 16450397Sobrien} 16550397Sobrien 16650397Sobrienstatic struct skcipher_alg aria_algs[] = { 16750397Sobrien { 16850397Sobrien .base.cra_name = "__ecb(aria)", 16950397Sobrien .base.cra_driver_name = "__ecb-aria-avx512", 17050397Sobrien .base.cra_priority = 600, 17150397Sobrien .base.cra_flags = CRYPTO_ALG_INTERNAL, 17250397Sobrien .base.cra_blocksize = ARIA_BLOCK_SIZE, 17350397Sobrien .base.cra_ctxsize = sizeof(struct aria_ctx), 17450397Sobrien .base.cra_module = THIS_MODULE, 17550397Sobrien .min_keysize = ARIA_MIN_KEY_SIZE, 17650397Sobrien .max_keysize = ARIA_MAX_KEY_SIZE, 17752284Sobrien .setkey = aria_avx512_set_key, 17852284Sobrien .encrypt = aria_avx512_ecb_encrypt, 17950397Sobrien .decrypt = aria_avx512_ecb_decrypt, 18052284Sobrien }, { 18150397Sobrien .base.cra_name = "__ctr(aria)", 18250397Sobrien .base.cra_driver_name = "__ctr-aria-avx512", 18350397Sobrien .base.cra_priority = 600, 18450397Sobrien .base.cra_flags = CRYPTO_ALG_INTERNAL | 18550397Sobrien CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, 18650397Sobrien .base.cra_blocksize = 1, 18750397Sobrien .base.cra_ctxsize = sizeof(struct aria_ctx), 18850397Sobrien .base.cra_module = THIS_MODULE, 18950397Sobrien .min_keysize = ARIA_MIN_KEY_SIZE, 19052284Sobrien .max_keysize = ARIA_MAX_KEY_SIZE, 19150397Sobrien .ivsize = ARIA_BLOCK_SIZE, 19250397Sobrien .chunksize = ARIA_BLOCK_SIZE, 19350397Sobrien .setkey = aria_avx512_set_key, 19450397Sobrien .encrypt = aria_avx512_ctr_encrypt, 19550397Sobrien .decrypt = aria_avx512_ctr_encrypt, 19650397Sobrien .init = aria_avx512_init_tfm, 19750397Sobrien } 19850397Sobrien}; 19950397Sobrien 20050397Sobrienstatic struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; 20150397Sobrien 20250397Sobrienstatic int __init aria_avx512_init(void) 20350397Sobrien{ 20452284Sobrien const char *feature_name; 20550397Sobrien 20650397Sobrien if (!boot_cpu_has(X86_FEATURE_AVX) || 20750397Sobrien !boot_cpu_has(X86_FEATURE_AVX2) || 20850397Sobrien !boot_cpu_has(X86_FEATURE_AVX512F) || 20950397Sobrien !boot_cpu_has(X86_FEATURE_AVX512VL) || 21050397Sobrien !boot_cpu_has(X86_FEATURE_GFNI) || 21150397Sobrien !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 21250397Sobrien pr_info("AVX512/GFNI instructions are not detected.\n"); 21350397Sobrien return -ENODEV; 21450397Sobrien } 21550397Sobrien 21650397Sobrien if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | 21750397Sobrien XFEATURE_MASK_AVX512, &feature_name)) { 21850397Sobrien pr_info("CPU feature '%s' is not supported.\n", feature_name); 21950397Sobrien return -ENODEV; 22050397Sobrien } 22150397Sobrien 22250397Sobrien aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; 22350397Sobrien aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; 22450397Sobrien aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; 22550397Sobrien aria_ops.aria_encrypt_32way = aria_aesni_avx2_gfni_encrypt_32way; 22650397Sobrien aria_ops.aria_decrypt_32way = aria_aesni_avx2_gfni_decrypt_32way; 22750397Sobrien aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_gfni_ctr_crypt_32way; 22850397Sobrien aria_ops.aria_encrypt_64way = aria_gfni_avx512_encrypt_64way; 22950397Sobrien aria_ops.aria_decrypt_64way = aria_gfni_avx512_decrypt_64way; 23050397Sobrien aria_ops.aria_ctr_crypt_64way = aria_gfni_avx512_ctr_crypt_64way; 23150397Sobrien 23250397Sobrien return simd_register_skciphers_compat(aria_algs, 23350397Sobrien ARRAY_SIZE(aria_algs), 23450397Sobrien aria_simd_algs); 23550397Sobrien} 23650397Sobrien 23750397Sobrienstatic void __exit aria_avx512_exit(void) 23850397Sobrien{ 23950397Sobrien simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), 24050397Sobrien aria_simd_algs); 24150397Sobrien} 24250397Sobrien 24350397Sobrienmodule_init(aria_avx512_init); 24450397Sobrienmodule_exit(aria_avx512_exit); 24550397Sobrien 24650397SobrienMODULE_LICENSE("GPL"); 24750397SobrienMODULE_AUTHOR("Taehee Yoo <ap420073@gmail.com>"); 24850397SobrienMODULE_DESCRIPTION("ARIA Cipher Algorithm, AVX512/GFNI optimized"); 24950397SobrienMODULE_ALIAS_CRYPTO("aria"); 25050397SobrienMODULE_ALIAS_CRYPTO("aria-gfni-avx512"); 25150397Sobrien