1// SPDX-License-Identifier: GPL-2.0 2#include <linux/crypto.h> 3#include <linux/kernel.h> 4#include <linux/module.h> 5#include <linux/printk.h> 6 7#include <crypto/aes.h> 8#include <crypto/skcipher.h> 9#include <crypto/scatterwalk.h> 10#include <crypto/ctr.h> 11#include <crypto/internal/des.h> 12#include <crypto/xts.h> 13 14#include "nitrox_dev.h" 15#include "nitrox_common.h" 16#include "nitrox_req.h" 17 18struct nitrox_cipher { 19 const char *name; 20 enum flexi_cipher value; 21}; 22 23/* 24 * supported cipher list 25 */ 26static const struct nitrox_cipher flexi_cipher_table[] = { 27 { "null", CIPHER_NULL }, 28 { "cbc(des3_ede)", CIPHER_3DES_CBC }, 29 { "ecb(des3_ede)", CIPHER_3DES_ECB }, 30 { "cbc(aes)", CIPHER_AES_CBC }, 31 { "ecb(aes)", CIPHER_AES_ECB }, 32 { "cfb(aes)", CIPHER_AES_CFB }, 33 { "rfc3686(ctr(aes))", CIPHER_AES_CTR }, 34 { "xts(aes)", CIPHER_AES_XTS }, 35 { "cts(cbc(aes))", CIPHER_AES_CBC_CTS }, 36 { NULL, CIPHER_INVALID } 37}; 38 39static enum flexi_cipher flexi_cipher_type(const char *name) 40{ 41 const struct nitrox_cipher *cipher = flexi_cipher_table; 42 43 while (cipher->name) { 44 if (!strcmp(cipher->name, name)) 45 break; 46 cipher++; 47 } 48 return cipher->value; 49} 50 51static void free_src_sglist(struct skcipher_request *skreq) 52{ 53 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 54 55 kfree(nkreq->src); 56} 57 58static void free_dst_sglist(struct skcipher_request *skreq) 59{ 60 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 61 62 kfree(nkreq->dst); 63} 64 65static void nitrox_skcipher_callback(void *arg, int err) 66{ 67 struct skcipher_request *skreq = arg; 68 69 free_src_sglist(skreq); 70 free_dst_sglist(skreq); 71 if (err) { 72 pr_err_ratelimited("request failed status 0x%0x\n", err); 73 err = -EINVAL; 74 } 75 76 skcipher_request_complete(skreq, err); 77} 78 79static void nitrox_cbc_cipher_callback(void *arg, int err) 80{ 81 struct skcipher_request *skreq = arg; 82 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 83 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); 84 int ivsize = crypto_skcipher_ivsize(cipher); 85 unsigned int start = skreq->cryptlen - ivsize; 86 87 if (err) { 88 nitrox_skcipher_callback(arg, err); 89 return; 90 } 91 92 if (nkreq->creq.ctrl.s.arg == ENCRYPT) { 93 scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize, 94 0); 95 } else { 96 if (skreq->src != skreq->dst) { 97 scatterwalk_map_and_copy(skreq->iv, skreq->src, start, 98 ivsize, 0); 99 } else { 100 memcpy(skreq->iv, nkreq->iv_out, ivsize); 101 kfree(nkreq->iv_out); 102 } 103 } 104 105 nitrox_skcipher_callback(arg, err); 106} 107 108static int nitrox_skcipher_init(struct crypto_skcipher *tfm) 109{ 110 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 111 struct crypto_ctx_hdr *chdr; 112 113 /* get the first device */ 114 nctx->ndev = nitrox_get_first_device(); 115 if (!nctx->ndev) 116 return -ENODEV; 117 118 /* allocate nitrox crypto context */ 119 chdr = crypto_alloc_context(nctx->ndev); 120 if (!chdr) { 121 nitrox_put_device(nctx->ndev); 122 return -ENOMEM; 123 } 124 125 nctx->callback = nitrox_skcipher_callback; 126 nctx->chdr = chdr; 127 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + 128 sizeof(struct ctx_hdr)); 129 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) + 130 sizeof(struct nitrox_kcrypt_request)); 131 return 0; 132} 133 134static int nitrox_cbc_init(struct crypto_skcipher *tfm) 135{ 136 int err; 137 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 138 139 err = nitrox_skcipher_init(tfm); 140 if (err) 141 return err; 142 143 nctx->callback = nitrox_cbc_cipher_callback; 144 return 0; 145} 146 147static void nitrox_skcipher_exit(struct crypto_skcipher *tfm) 148{ 149 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 150 151 /* free the nitrox crypto context */ 152 if (nctx->u.ctx_handle) { 153 struct flexi_crypto_context *fctx = nctx->u.fctx; 154 155 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); 156 memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); 157 crypto_free_context((void *)nctx->chdr); 158 } 159 nitrox_put_device(nctx->ndev); 160 161 nctx->u.ctx_handle = 0; 162 nctx->ndev = NULL; 163} 164 165static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher, 166 int aes_keylen, const u8 *key, 167 unsigned int keylen) 168{ 169 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 170 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); 171 struct flexi_crypto_context *fctx; 172 union fc_ctx_flags *flags; 173 enum flexi_cipher cipher_type; 174 const char *name; 175 176 name = crypto_tfm_alg_name(tfm); 177 cipher_type = flexi_cipher_type(name); 178 if (unlikely(cipher_type == CIPHER_INVALID)) { 179 pr_err("unsupported cipher: %s\n", name); 180 return -EINVAL; 181 } 182 183 /* fill crypto context */ 184 fctx = nctx->u.fctx; 185 flags = &fctx->flags; 186 flags->f = 0; 187 flags->w0.cipher_type = cipher_type; 188 flags->w0.aes_keylen = aes_keylen; 189 flags->w0.iv_source = IV_FROM_DPTR; 190 flags->f = cpu_to_be64(*(u64 *)&flags->w0); 191 /* copy the key to context */ 192 memcpy(fctx->crypto.u.key, key, keylen); 193 194 return 0; 195} 196 197static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 198 unsigned int keylen) 199{ 200 int aes_keylen; 201 202 aes_keylen = flexi_aes_keylen(keylen); 203 if (aes_keylen < 0) 204 return -EINVAL; 205 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 206} 207 208static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize) 209{ 210 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 211 int nents = sg_nents(skreq->src) + 1; 212 int ret; 213 214 /* Allocate buffer to hold IV and input scatterlist array */ 215 ret = alloc_src_req_buf(nkreq, nents, ivsize); 216 if (ret) 217 return ret; 218 219 nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize); 220 nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src, 221 skreq->cryptlen); 222 223 return 0; 224} 225 226static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize) 227{ 228 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 229 int nents = sg_nents(skreq->dst) + 3; 230 int ret; 231 232 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist 233 * array 234 */ 235 ret = alloc_dst_req_buf(nkreq, nents); 236 if (ret) 237 return ret; 238 239 nitrox_creq_set_orh(nkreq); 240 nitrox_creq_set_comp(nkreq); 241 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst, 242 skreq->cryptlen); 243 244 return 0; 245} 246 247static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc) 248{ 249 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); 250 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); 251 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 252 int ivsize = crypto_skcipher_ivsize(cipher); 253 struct se_crypto_request *creq; 254 int ret; 255 256 creq = &nkreq->creq; 257 creq->flags = skreq->base.flags; 258 creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 259 GFP_KERNEL : GFP_ATOMIC; 260 261 /* fill the request */ 262 creq->ctrl.value = 0; 263 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; 264 creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT); 265 /* param0: length of the data to be encrypted */ 266 creq->gph.param0 = cpu_to_be16(skreq->cryptlen); 267 creq->gph.param1 = 0; 268 /* param2: encryption data offset */ 269 creq->gph.param2 = cpu_to_be16(ivsize); 270 creq->gph.param3 = 0; 271 272 creq->ctx_handle = nctx->u.ctx_handle; 273 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); 274 275 ret = alloc_src_sglist(skreq, ivsize); 276 if (ret) 277 return ret; 278 279 ret = alloc_dst_sglist(skreq, ivsize); 280 if (ret) { 281 free_src_sglist(skreq); 282 return ret; 283 } 284 285 /* send the crypto request */ 286 return nitrox_process_se_request(nctx->ndev, creq, nctx->callback, 287 skreq); 288} 289 290static int nitrox_cbc_decrypt(struct skcipher_request *skreq) 291{ 292 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 293 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); 294 int ivsize = crypto_skcipher_ivsize(cipher); 295 gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 296 GFP_KERNEL : GFP_ATOMIC; 297 unsigned int start = skreq->cryptlen - ivsize; 298 299 if (skreq->src != skreq->dst) 300 return nitrox_skcipher_crypt(skreq, false); 301 302 nkreq->iv_out = kmalloc(ivsize, flags); 303 if (!nkreq->iv_out) 304 return -ENOMEM; 305 306 scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0); 307 return nitrox_skcipher_crypt(skreq, false); 308} 309 310static int nitrox_aes_encrypt(struct skcipher_request *skreq) 311{ 312 return nitrox_skcipher_crypt(skreq, true); 313} 314 315static int nitrox_aes_decrypt(struct skcipher_request *skreq) 316{ 317 return nitrox_skcipher_crypt(skreq, false); 318} 319 320static int nitrox_3des_setkey(struct crypto_skcipher *cipher, 321 const u8 *key, unsigned int keylen) 322{ 323 return verify_skcipher_des3_key(cipher, key) ?: 324 nitrox_skcipher_setkey(cipher, 0, key, keylen); 325} 326 327static int nitrox_3des_encrypt(struct skcipher_request *skreq) 328{ 329 return nitrox_skcipher_crypt(skreq, true); 330} 331 332static int nitrox_3des_decrypt(struct skcipher_request *skreq) 333{ 334 return nitrox_skcipher_crypt(skreq, false); 335} 336 337static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher, 338 const u8 *key, unsigned int keylen) 339{ 340 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); 341 struct flexi_crypto_context *fctx; 342 int aes_keylen, ret; 343 344 ret = xts_verify_key(cipher, key, keylen); 345 if (ret) 346 return ret; 347 348 keylen /= 2; 349 350 aes_keylen = flexi_aes_keylen(keylen); 351 if (aes_keylen < 0) 352 return -EINVAL; 353 354 fctx = nctx->u.fctx; 355 /* copy KEY2 */ 356 memcpy(fctx->auth.u.key2, (key + keylen), keylen); 357 358 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 359} 360 361static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher, 362 const u8 *key, unsigned int keylen) 363{ 364 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); 365 struct flexi_crypto_context *fctx; 366 int aes_keylen; 367 368 if (keylen < CTR_RFC3686_NONCE_SIZE) 369 return -EINVAL; 370 371 fctx = nctx->u.fctx; 372 373 memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE), 374 CTR_RFC3686_NONCE_SIZE); 375 376 keylen -= CTR_RFC3686_NONCE_SIZE; 377 378 aes_keylen = flexi_aes_keylen(keylen); 379 if (aes_keylen < 0) 380 return -EINVAL; 381 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 382} 383 384static struct skcipher_alg nitrox_skciphers[] = { { 385 .base = { 386 .cra_name = "cbc(aes)", 387 .cra_driver_name = "n5_cbc(aes)", 388 .cra_priority = PRIO, 389 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 390 .cra_blocksize = AES_BLOCK_SIZE, 391 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 392 .cra_alignmask = 0, 393 .cra_module = THIS_MODULE, 394 }, 395 .min_keysize = AES_MIN_KEY_SIZE, 396 .max_keysize = AES_MAX_KEY_SIZE, 397 .ivsize = AES_BLOCK_SIZE, 398 .setkey = nitrox_aes_setkey, 399 .encrypt = nitrox_aes_encrypt, 400 .decrypt = nitrox_cbc_decrypt, 401 .init = nitrox_cbc_init, 402 .exit = nitrox_skcipher_exit, 403}, { 404 .base = { 405 .cra_name = "ecb(aes)", 406 .cra_driver_name = "n5_ecb(aes)", 407 .cra_priority = PRIO, 408 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 409 .cra_blocksize = AES_BLOCK_SIZE, 410 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 411 .cra_alignmask = 0, 412 .cra_module = THIS_MODULE, 413 }, 414 .min_keysize = AES_MIN_KEY_SIZE, 415 .max_keysize = AES_MAX_KEY_SIZE, 416 .ivsize = AES_BLOCK_SIZE, 417 .setkey = nitrox_aes_setkey, 418 .encrypt = nitrox_aes_encrypt, 419 .decrypt = nitrox_aes_decrypt, 420 .init = nitrox_skcipher_init, 421 .exit = nitrox_skcipher_exit, 422}, { 423 .base = { 424 .cra_name = "xts(aes)", 425 .cra_driver_name = "n5_xts(aes)", 426 .cra_priority = PRIO, 427 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 428 .cra_blocksize = AES_BLOCK_SIZE, 429 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 430 .cra_alignmask = 0, 431 .cra_module = THIS_MODULE, 432 }, 433 .min_keysize = 2 * AES_MIN_KEY_SIZE, 434 .max_keysize = 2 * AES_MAX_KEY_SIZE, 435 .ivsize = AES_BLOCK_SIZE, 436 .setkey = nitrox_aes_xts_setkey, 437 .encrypt = nitrox_aes_encrypt, 438 .decrypt = nitrox_aes_decrypt, 439 .init = nitrox_skcipher_init, 440 .exit = nitrox_skcipher_exit, 441}, { 442 .base = { 443 .cra_name = "rfc3686(ctr(aes))", 444 .cra_driver_name = "n5_rfc3686(ctr(aes))", 445 .cra_priority = PRIO, 446 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 447 .cra_blocksize = 1, 448 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 449 .cra_alignmask = 0, 450 .cra_module = THIS_MODULE, 451 }, 452 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 453 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 454 .ivsize = CTR_RFC3686_IV_SIZE, 455 .init = nitrox_skcipher_init, 456 .exit = nitrox_skcipher_exit, 457 .setkey = nitrox_aes_ctr_rfc3686_setkey, 458 .encrypt = nitrox_aes_encrypt, 459 .decrypt = nitrox_aes_decrypt, 460}, { 461 .base = { 462 .cra_name = "cts(cbc(aes))", 463 .cra_driver_name = "n5_cts(cbc(aes))", 464 .cra_priority = PRIO, 465 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 466 .cra_blocksize = AES_BLOCK_SIZE, 467 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 468 .cra_alignmask = 0, 469 .cra_module = THIS_MODULE, 470 }, 471 .min_keysize = AES_MIN_KEY_SIZE, 472 .max_keysize = AES_MAX_KEY_SIZE, 473 .ivsize = AES_BLOCK_SIZE, 474 .setkey = nitrox_aes_setkey, 475 .encrypt = nitrox_aes_encrypt, 476 .decrypt = nitrox_aes_decrypt, 477 .init = nitrox_skcipher_init, 478 .exit = nitrox_skcipher_exit, 479}, { 480 .base = { 481 .cra_name = "cbc(des3_ede)", 482 .cra_driver_name = "n5_cbc(des3_ede)", 483 .cra_priority = PRIO, 484 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 485 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 486 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 487 .cra_alignmask = 0, 488 .cra_module = THIS_MODULE, 489 }, 490 .min_keysize = DES3_EDE_KEY_SIZE, 491 .max_keysize = DES3_EDE_KEY_SIZE, 492 .ivsize = DES3_EDE_BLOCK_SIZE, 493 .setkey = nitrox_3des_setkey, 494 .encrypt = nitrox_3des_encrypt, 495 .decrypt = nitrox_cbc_decrypt, 496 .init = nitrox_cbc_init, 497 .exit = nitrox_skcipher_exit, 498}, { 499 .base = { 500 .cra_name = "ecb(des3_ede)", 501 .cra_driver_name = "n5_ecb(des3_ede)", 502 .cra_priority = PRIO, 503 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 504 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 505 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 506 .cra_alignmask = 0, 507 .cra_module = THIS_MODULE, 508 }, 509 .min_keysize = DES3_EDE_KEY_SIZE, 510 .max_keysize = DES3_EDE_KEY_SIZE, 511 .ivsize = DES3_EDE_BLOCK_SIZE, 512 .setkey = nitrox_3des_setkey, 513 .encrypt = nitrox_3des_encrypt, 514 .decrypt = nitrox_3des_decrypt, 515 .init = nitrox_skcipher_init, 516 .exit = nitrox_skcipher_exit, 517} 518 519}; 520 521int nitrox_register_skciphers(void) 522{ 523 return crypto_register_skciphers(nitrox_skciphers, 524 ARRAY_SIZE(nitrox_skciphers)); 525} 526 527void nitrox_unregister_skciphers(void) 528{ 529 crypto_unregister_skciphers(nitrox_skciphers, 530 ARRAY_SIZE(nitrox_skciphers)); 531} 532