1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Intel Keem Bay OCS ECC Crypto Driver. 4 * 5 * Copyright (C) 2019-2021 Intel Corporation 6 */ 7 8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10#include <crypto/ecc_curve.h> 11#include <crypto/ecdh.h> 12#include <crypto/engine.h> 13#include <crypto/internal/ecc.h> 14#include <crypto/internal/kpp.h> 15#include <crypto/kpp.h> 16#include <crypto/rng.h> 17#include <linux/clk.h> 18#include <linux/completion.h> 19#include <linux/err.h> 20#include <linux/fips.h> 21#include <linux/interrupt.h> 22#include <linux/io.h> 23#include <linux/iopoll.h> 24#include <linux/irq.h> 25#include <linux/kernel.h> 26#include <linux/module.h> 27#include <linux/of.h> 28#include <linux/platform_device.h> 29#include <linux/scatterlist.h> 30#include <linux/string.h> 31 32#define DRV_NAME "keembay-ocs-ecc" 33 34#define KMB_OCS_ECC_PRIORITY 350 35 36#define HW_OFFS_OCS_ECC_COMMAND 0x00000000 37#define HW_OFFS_OCS_ECC_STATUS 0x00000004 38#define HW_OFFS_OCS_ECC_DATA_IN 0x00000080 39#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x00000100 40#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x00000180 41#define HW_OFFS_OCS_ECC_ISR 0x00000400 42#define HW_OFFS_OCS_ECC_IER 0x00000404 43 44#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0) 45#define HW_OCS_ECC_COMMAND_INS_BP BIT(0) 46 47#define HW_OCS_ECC_COMMAND_START_VAL BIT(0) 48 49#define OCS_ECC_OP_SIZE_384 BIT(8) 50#define OCS_ECC_OP_SIZE_256 0 51 52/* ECC Instruction : for ECC_COMMAND */ 53#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP) 54#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP) 55#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP) 56#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP) 57#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP) 58#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP) 59#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP) 60#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP) 61#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP) 62#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP) 63 64#define ECC_ENABLE_INTR 1 65 66#define POLL_USEC 100 67#define TIMEOUT_USEC 10000 68 69#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS 70#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \ 71 << ECC_DIGITS_TO_BYTES_SHIFT) 72 73#define POW_CUBE 3 74 75/** 76 * struct ocs_ecc_dev - ECC device context 77 * @list: List of device contexts 78 * @dev: OCS ECC device 79 * @base_reg: IO base address of OCS ECC 80 * @engine: Crypto engine for the device 81 * @irq_done: IRQ done completion. 82 * @irq: IRQ number 83 */ 84struct ocs_ecc_dev { 85 struct list_head list; 86 struct device *dev; 87 void __iomem *base_reg; 88 struct crypto_engine *engine; 89 struct completion irq_done; 90 int irq; 91}; 92 93/** 94 * struct ocs_ecc_ctx - Transformation context. 95 * @ecc_dev: The ECC driver associated with this context. 96 * @curve: The elliptic curve used by this transformation. 97 * @private_key: The private key. 98 */ 99struct ocs_ecc_ctx { 100 struct ocs_ecc_dev *ecc_dev; 101 const struct ecc_curve *curve; 102 u64 private_key[KMB_ECC_VLI_MAX_DIGITS]; 103}; 104 105/* Driver data. */ 106struct ocs_ecc_drv { 107 struct list_head dev_list; 108 spinlock_t lock; /* Protects dev_list. */ 109}; 110 111/* Global variable holding the list of OCS ECC devices (only one expected). */ 112static struct ocs_ecc_drv ocs_ecc = { 113 .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list), 114 .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock), 115}; 116 117/* Get OCS ECC tfm context from kpp_request. */ 118static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req) 119{ 120 return kpp_tfm_ctx(crypto_kpp_reqtfm(req)); 121} 122 123/* Converts number of digits to number of bytes. */ 124static inline unsigned int digits_to_bytes(unsigned int n) 125{ 126 return n << ECC_DIGITS_TO_BYTES_SHIFT; 127} 128 129/* 130 * Wait for ECC idle i.e when an operation (other than write operations) 131 * is done. 132 */ 133static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev) 134{ 135 u32 value; 136 137 return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS), 138 value, 139 !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE), 140 POLL_USEC, TIMEOUT_USEC); 141} 142 143static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size) 144{ 145 iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL, 146 ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); 147} 148 149/* Direct write of u32 buffer to ECC engine with associated instruction. */ 150static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev, 151 u32 op_size, 152 u32 inst, 153 const void *data_in, 154 size_t data_size) 155{ 156 iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); 157 158 /* MMIO Write src uint32 to dst. */ 159 memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in, 160 data_size); 161} 162 163/* Start OCS ECC operation and wait for its completion. */ 164static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size, 165 u32 inst) 166{ 167 reinit_completion(&ecc_dev->irq_done); 168 169 iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER); 170 iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); 171 172 return wait_for_completion_interruptible(&ecc_dev->irq_done); 173} 174 175/** 176 * ocs_ecc_read_cx_out() - Read the CX data output buffer. 177 * @dev: The OCS ECC device to read from. 178 * @cx_out: The buffer where to store the CX value. Must be at least 179 * @byte_count byte long. 180 * @byte_count: The amount of data to read. 181 */ 182static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out, 183 size_t byte_count) 184{ 185 memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT, 186 byte_count); 187} 188 189/** 190 * ocs_ecc_read_cy_out() - Read the CX data output buffer. 191 * @dev: The OCS ECC device to read from. 192 * @cy_out: The buffer where to store the CY value. Must be at least 193 * @byte_count byte long. 194 * @byte_count: The amount of data to read. 195 */ 196static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out, 197 size_t byte_count) 198{ 199 memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT, 200 byte_count); 201} 202 203static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx) 204{ 205 if (tctx->ecc_dev) 206 return tctx->ecc_dev; 207 208 spin_lock(&ocs_ecc.lock); 209 210 /* Only a single OCS device available. */ 211 tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev, 212 list); 213 214 spin_unlock(&ocs_ecc.lock); 215 216 return tctx->ecc_dev; 217} 218 219/* Do point multiplication using OCS ECC HW. */ 220static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev, 221 struct ecc_point *result, 222 const struct ecc_point *point, 223 u64 *scalar, 224 const struct ecc_curve *curve) 225{ 226 u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */ 227 u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ? 228 OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; 229 size_t nbytes = digits_to_bytes(curve->g.ndigits); 230 int rc = 0; 231 232 /* Generate random nbytes for Simple and Differential SCA protection. */ 233 rc = crypto_get_default_rng(); 234 if (rc) 235 return rc; 236 237 rc = crypto_rng_get_bytes(crypto_default_rng, sca, nbytes); 238 crypto_put_default_rng(); 239 if (rc) 240 return rc; 241 242 /* Wait engine to be idle before starting new operation. */ 243 rc = ocs_ecc_wait_idle(ecc_dev); 244 if (rc) 245 return rc; 246 247 /* Send ecc_start pulse as well as indicating operation size. */ 248 ocs_ecc_cmd_start(ecc_dev, op_size); 249 250 /* Write ax param; Base point (Gx). */ 251 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, 252 point->x, nbytes); 253 254 /* Write ay param; Base point (Gy). */ 255 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, 256 point->y, nbytes); 257 258 /* 259 * Write the private key into DATA_IN reg. 260 * 261 * Since DATA_IN register is used to write different values during the 262 * computation private Key value is overwritten with 263 * side-channel-resistance value. 264 */ 265 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D, 266 scalar, nbytes); 267 268 /* Write operand by/l. */ 269 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L, 270 sca, nbytes); 271 memzero_explicit(sca, sizeof(sca)); 272 273 /* Write p = curve prime(GF modulus). */ 274 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, 275 curve->p, nbytes); 276 277 /* Write a = curve coefficient. */ 278 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A, 279 curve->a, nbytes); 280 281 /* Make hardware perform the multiplication. */ 282 rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A); 283 if (rc) 284 return rc; 285 286 /* Read result. */ 287 ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes); 288 ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes); 289 290 return 0; 291} 292 293/** 294 * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW. 295 * @ecc_dev: The OCS ECC device to use. 296 * @scalar_out: Where to store the output scalar. 297 * @scalar_a: Input scalar operand 'a'. 298 * @scalar_b: Input scalar operand 'b' 299 * @curve: The curve on which the operation is performed. 300 * @ndigits: The size of the operands (in digits). 301 * @inst: The operation to perform (as an OCS ECC instruction). 302 * 303 * Return: 0 on success, negative error code otherwise. 304 */ 305static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out, 306 const u64 *scalar_a, const u64 *scalar_b, 307 const struct ecc_curve *curve, 308 unsigned int ndigits, const u32 inst) 309{ 310 u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ? 311 OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; 312 size_t nbytes = digits_to_bytes(ndigits); 313 int rc; 314 315 /* Wait engine to be idle before starting new operation. */ 316 rc = ocs_ecc_wait_idle(ecc_dev); 317 if (rc) 318 return rc; 319 320 /* Send ecc_start pulse as well as indicating operation size. */ 321 ocs_ecc_cmd_start(ecc_dev, op_size); 322 323 /* Write ax param (Base point (Gx).*/ 324 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, 325 scalar_a, nbytes); 326 327 /* Write ay param Base point (Gy).*/ 328 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, 329 scalar_b, nbytes); 330 331 /* Write p = curve prime(GF modulus).*/ 332 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, 333 curve->p, nbytes); 334 335 /* Give instruction A.B or A+B to ECC engine. */ 336 rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst); 337 if (rc) 338 return rc; 339 340 ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes); 341 342 if (vli_is_zero(scalar_out, ndigits)) 343 return -EINVAL; 344 345 return 0; 346} 347 348/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */ 349static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev, 350 const struct ecc_curve *curve, 351 struct ecc_point *pk) 352{ 353 u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; 354 u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; 355 u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; 356 int rc; 357 358 if (WARN_ON(pk->ndigits != curve->g.ndigits)) 359 return -EINVAL; 360 361 /* Check 1: Verify key is not the zero point. */ 362 if (ecc_point_is_zero(pk)) 363 return -EINVAL; 364 365 /* Check 2: Verify key is in the range [0, p-1]. */ 366 if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1) 367 return -EINVAL; 368 369 if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1) 370 return -EINVAL; 371 372 /* Check 3: Verify that y^2 == (x^3 + a��x + b) mod p */ 373 374 /* y^2 */ 375 /* Compute y^2 -> store in yy */ 376 rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits, 377 OCS_ECC_INST_CALC_A_MUL_B_MODP); 378 if (rc) 379 goto exit; 380 381 /* x^3 */ 382 /* Assigning w = 3, used for calculating x^3. */ 383 w[0] = POW_CUBE; 384 /* Load the next stage.*/ 385 rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits, 386 OCS_ECC_INST_CALC_A_POW_B_MODP); 387 if (rc) 388 goto exit; 389 390 /* Do a*x -> store in w. */ 391 rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve, 392 pk->ndigits, 393 OCS_ECC_INST_CALC_A_MUL_B_MODP); 394 if (rc) 395 goto exit; 396 397 /* Do ax + b == w + b; store in w. */ 398 rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve, 399 pk->ndigits, 400 OCS_ECC_INST_CALC_A_ADD_B_MODP); 401 if (rc) 402 goto exit; 403 404 /* x^3 + ax + b == x^3 + w -> store in w. */ 405 rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits, 406 OCS_ECC_INST_CALC_A_ADD_B_MODP); 407 if (rc) 408 goto exit; 409 410 /* Compare y^2 == x^3 + a��x + b. */ 411 rc = vli_cmp(yy, w, pk->ndigits); 412 if (rc) 413 rc = -EINVAL; 414 415exit: 416 memzero_explicit(xxx, sizeof(xxx)); 417 memzero_explicit(yy, sizeof(yy)); 418 memzero_explicit(w, sizeof(w)); 419 420 return rc; 421} 422 423/* SP800-56A section 5.6.2.3.3 full verification */ 424static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev, 425 const struct ecc_curve *curve, 426 struct ecc_point *pk) 427{ 428 struct ecc_point *nQ; 429 int rc; 430 431 /* Checks 1 through 3 */ 432 rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); 433 if (rc) 434 return rc; 435 436 /* Check 4: Verify that nQ is the zero point. */ 437 nQ = ecc_alloc_point(pk->ndigits); 438 if (!nQ) 439 return -ENOMEM; 440 441 rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve); 442 if (rc) 443 goto exit; 444 445 if (!ecc_point_is_zero(nQ)) 446 rc = -EINVAL; 447 448exit: 449 ecc_free_point(nQ); 450 451 return rc; 452} 453 454static int kmb_ecc_is_key_valid(const struct ecc_curve *curve, 455 const u64 *private_key, size_t private_key_len) 456{ 457 size_t ndigits = curve->g.ndigits; 458 u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1}; 459 u64 res[KMB_ECC_VLI_MAX_DIGITS]; 460 461 if (private_key_len != digits_to_bytes(ndigits)) 462 return -EINVAL; 463 464 if (!private_key) 465 return -EINVAL; 466 467 /* Make sure the private key is in the range [2, n-3]. */ 468 if (vli_cmp(one, private_key, ndigits) != -1) 469 return -EINVAL; 470 471 vli_sub(res, curve->n, one, ndigits); 472 vli_sub(res, res, one, ndigits); 473 if (vli_cmp(res, private_key, ndigits) != 1) 474 return -EINVAL; 475 476 return 0; 477} 478 479/* 480 * ECC private keys are generated using the method of extra random bits, 481 * equivalent to that described in FIPS 186-4, Appendix B.4.1. 482 * 483 * d = (c mod(n���1)) + 1 where c is a string of random bits, 64 bits longer 484 * than requested 485 * 0 <= c mod(n-1) <= n-2 and implies that 486 * 1 <= d <= n-1 487 * 488 * This method generates a private key uniformly distributed in the range 489 * [1, n-1]. 490 */ 491static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey) 492{ 493 size_t nbytes = digits_to_bytes(curve->g.ndigits); 494 u64 priv[KMB_ECC_VLI_MAX_DIGITS]; 495 size_t nbits; 496 int rc; 497 498 nbits = vli_num_bits(curve->n, curve->g.ndigits); 499 500 /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */ 501 if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv)) 502 return -EINVAL; 503 504 /* 505 * FIPS 186-4 recommends that the private key should be obtained from a 506 * RBG with a security strength equal to or greater than the security 507 * strength associated with N. 508 * 509 * The maximum security strength identified by NIST SP800-57pt1r4 for 510 * ECC is 256 (N >= 512). 511 * 512 * This condition is met by the default RNG because it selects a favored 513 * DRBG with a security strength of 256. 514 */ 515 if (crypto_get_default_rng()) 516 return -EFAULT; 517 518 rc = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes); 519 crypto_put_default_rng(); 520 if (rc) 521 goto cleanup; 522 523 rc = kmb_ecc_is_key_valid(curve, priv, nbytes); 524 if (rc) 525 goto cleanup; 526 527 ecc_swap_digits(priv, privkey, curve->g.ndigits); 528 529cleanup: 530 memzero_explicit(&priv, sizeof(priv)); 531 532 return rc; 533} 534 535static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, 536 unsigned int len) 537{ 538 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); 539 struct ecdh params; 540 int rc = 0; 541 542 rc = crypto_ecdh_decode_key(buf, len, ¶ms); 543 if (rc) 544 goto cleanup; 545 546 /* Ensure key size is not bigger then expected. */ 547 if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) { 548 rc = -EINVAL; 549 goto cleanup; 550 } 551 552 /* Auto-generate private key is not provided. */ 553 if (!params.key || !params.key_size) { 554 rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key); 555 goto cleanup; 556 } 557 558 rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key, 559 params.key_size); 560 if (rc) 561 goto cleanup; 562 563 ecc_swap_digits((const u64 *)params.key, tctx->private_key, 564 tctx->curve->g.ndigits); 565cleanup: 566 memzero_explicit(¶ms, sizeof(params)); 567 568 if (rc) 569 tctx->curve = NULL; 570 571 return rc; 572} 573 574/* Compute shared secret. */ 575static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx, 576 struct kpp_request *req) 577{ 578 struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; 579 const struct ecc_curve *curve = tctx->curve; 580 u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS]; 581 u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; 582 size_t copied, nbytes, pubk_len; 583 struct ecc_point *pk, *result; 584 int rc; 585 586 nbytes = digits_to_bytes(curve->g.ndigits); 587 588 /* Public key is a point, thus it has two coordinates */ 589 pubk_len = 2 * nbytes; 590 591 /* Copy public key from SG list to pubk_buf. */ 592 copied = sg_copy_to_buffer(req->src, 593 sg_nents_for_len(req->src, pubk_len), 594 pubk_buf, pubk_len); 595 if (copied != pubk_len) 596 return -EINVAL; 597 598 /* Allocate and initialize public key point. */ 599 pk = ecc_alloc_point(curve->g.ndigits); 600 if (!pk) 601 return -ENOMEM; 602 603 ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits); 604 ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits); 605 606 /* 607 * Check the public key for following 608 * Check 1: Verify key is not the zero point. 609 * Check 2: Verify key is in the range [1, p-1]. 610 * Check 3: Verify that y^2 == (x^3 + a��x + b) mod p 611 */ 612 rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); 613 if (rc) 614 goto exit_free_pk; 615 616 /* Allocate point for storing computed shared secret. */ 617 result = ecc_alloc_point(pk->ndigits); 618 if (!result) { 619 rc = -ENOMEM; 620 goto exit_free_pk; 621 } 622 623 /* Calculate the shared secret.*/ 624 rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve); 625 if (rc) 626 goto exit_free_result; 627 628 if (ecc_point_is_zero(result)) { 629 rc = -EFAULT; 630 goto exit_free_result; 631 } 632 633 /* Copy shared secret from point to buffer. */ 634 ecc_swap_digits(result->x, shared_secret, result->ndigits); 635 636 /* Request might ask for less bytes than what we have. */ 637 nbytes = min_t(size_t, nbytes, req->dst_len); 638 639 copied = sg_copy_from_buffer(req->dst, 640 sg_nents_for_len(req->dst, nbytes), 641 shared_secret, nbytes); 642 643 if (copied != nbytes) 644 rc = -EINVAL; 645 646 memzero_explicit(shared_secret, sizeof(shared_secret)); 647 648exit_free_result: 649 ecc_free_point(result); 650 651exit_free_pk: 652 ecc_free_point(pk); 653 654 return rc; 655} 656 657/* Compute public key. */ 658static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx, 659 struct kpp_request *req) 660{ 661 const struct ecc_curve *curve = tctx->curve; 662 u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; 663 struct ecc_point *pk; 664 size_t pubk_len; 665 size_t copied; 666 int rc; 667 668 /* Public key is a point, so it has double the digits. */ 669 pubk_len = 2 * digits_to_bytes(curve->g.ndigits); 670 671 pk = ecc_alloc_point(curve->g.ndigits); 672 if (!pk) 673 return -ENOMEM; 674 675 /* Public Key(pk) = priv * G. */ 676 rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key, 677 curve); 678 if (rc) 679 goto exit; 680 681 /* SP800-56A rev 3 5.6.2.1.3 key check */ 682 if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) { 683 rc = -EAGAIN; 684 goto exit; 685 } 686 687 /* Copy public key from point to buffer. */ 688 ecc_swap_digits(pk->x, pubk_buf, pk->ndigits); 689 ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits); 690 691 /* Copy public key to req->dst. */ 692 copied = sg_copy_from_buffer(req->dst, 693 sg_nents_for_len(req->dst, pubk_len), 694 pubk_buf, pubk_len); 695 696 if (copied != pubk_len) 697 rc = -EINVAL; 698 699exit: 700 ecc_free_point(pk); 701 702 return rc; 703} 704 705static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine, 706 void *areq) 707{ 708 struct kpp_request *req = container_of(areq, struct kpp_request, base); 709 struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); 710 struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; 711 int rc; 712 713 if (req->src) 714 rc = kmb_ecc_do_shared_secret(tctx, req); 715 else 716 rc = kmb_ecc_do_public_key(tctx, req); 717 718 crypto_finalize_kpp_request(ecc_dev->engine, req, rc); 719 720 return 0; 721} 722 723static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req) 724{ 725 struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); 726 const struct ecc_curve *curve = tctx->curve; 727 728 /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ 729 if (!tctx->curve) 730 return -EINVAL; 731 732 /* Ensure dst is present. */ 733 if (!req->dst) 734 return -EINVAL; 735 736 /* Check the request dst is big enough to hold the public key. */ 737 if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits))) 738 return -EINVAL; 739 740 /* 'src' is not supposed to be present when generate pubk is called. */ 741 if (req->src) 742 return -EINVAL; 743 744 return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, 745 req); 746} 747 748static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req) 749{ 750 struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); 751 const struct ecc_curve *curve = tctx->curve; 752 753 /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ 754 if (!tctx->curve) 755 return -EINVAL; 756 757 /* Ensure dst is present. */ 758 if (!req->dst) 759 return -EINVAL; 760 761 /* Ensure src is present. */ 762 if (!req->src) 763 return -EINVAL; 764 765 /* 766 * req->src is expected to the (other-side) public key, so its length 767 * must be 2 * coordinate size (in bytes). 768 */ 769 if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits)) 770 return -EINVAL; 771 772 return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, 773 req); 774} 775 776static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id) 777{ 778 memset(tctx, 0, sizeof(*tctx)); 779 780 tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx); 781 782 if (IS_ERR(tctx->ecc_dev)) { 783 pr_err("Failed to find the device : %ld\n", 784 PTR_ERR(tctx->ecc_dev)); 785 return PTR_ERR(tctx->ecc_dev); 786 } 787 788 tctx->curve = ecc_get_curve(curve_id); 789 if (!tctx->curve) 790 return -EOPNOTSUPP; 791 792 return 0; 793} 794 795static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) 796{ 797 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); 798 799 return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256); 800} 801 802static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) 803{ 804 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); 805 806 return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384); 807} 808 809static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm) 810{ 811 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); 812 813 memzero_explicit(tctx->private_key, sizeof(*tctx->private_key)); 814} 815 816static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm) 817{ 818 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); 819 820 /* Public key is made of two coordinates, so double the digits. */ 821 return digits_to_bytes(tctx->curve->g.ndigits) * 2; 822} 823 824static struct kpp_engine_alg ocs_ecdh_p256 = { 825 .base.set_secret = kmb_ocs_ecdh_set_secret, 826 .base.generate_public_key = kmb_ocs_ecdh_generate_public_key, 827 .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, 828 .base.init = kmb_ocs_ecdh_nist_p256_init_tfm, 829 .base.exit = kmb_ocs_ecdh_exit_tfm, 830 .base.max_size = kmb_ocs_ecdh_max_size, 831 .base.base = { 832 .cra_name = "ecdh-nist-p256", 833 .cra_driver_name = "ecdh-nist-p256-keembay-ocs", 834 .cra_priority = KMB_OCS_ECC_PRIORITY, 835 .cra_module = THIS_MODULE, 836 .cra_ctxsize = sizeof(struct ocs_ecc_ctx), 837 }, 838 .op.do_one_request = kmb_ocs_ecc_do_one_request, 839}; 840 841static struct kpp_engine_alg ocs_ecdh_p384 = { 842 .base.set_secret = kmb_ocs_ecdh_set_secret, 843 .base.generate_public_key = kmb_ocs_ecdh_generate_public_key, 844 .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, 845 .base.init = kmb_ocs_ecdh_nist_p384_init_tfm, 846 .base.exit = kmb_ocs_ecdh_exit_tfm, 847 .base.max_size = kmb_ocs_ecdh_max_size, 848 .base.base = { 849 .cra_name = "ecdh-nist-p384", 850 .cra_driver_name = "ecdh-nist-p384-keembay-ocs", 851 .cra_priority = KMB_OCS_ECC_PRIORITY, 852 .cra_module = THIS_MODULE, 853 .cra_ctxsize = sizeof(struct ocs_ecc_ctx), 854 }, 855 .op.do_one_request = kmb_ocs_ecc_do_one_request, 856}; 857 858static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id) 859{ 860 struct ocs_ecc_dev *ecc_dev = dev_id; 861 u32 status; 862 863 /* 864 * Read the status register and write it back to clear the 865 * DONE_INT_STATUS bit. 866 */ 867 status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); 868 iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); 869 870 if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE)) 871 return IRQ_NONE; 872 873 complete(&ecc_dev->irq_done); 874 875 return IRQ_HANDLED; 876} 877 878static int kmb_ocs_ecc_probe(struct platform_device *pdev) 879{ 880 struct device *dev = &pdev->dev; 881 struct ocs_ecc_dev *ecc_dev; 882 int rc; 883 884 ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL); 885 if (!ecc_dev) 886 return -ENOMEM; 887 888 ecc_dev->dev = dev; 889 890 platform_set_drvdata(pdev, ecc_dev); 891 892 INIT_LIST_HEAD(&ecc_dev->list); 893 init_completion(&ecc_dev->irq_done); 894 895 /* Get base register address. */ 896 ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); 897 if (IS_ERR(ecc_dev->base_reg)) { 898 dev_err(dev, "Failed to get base address\n"); 899 rc = PTR_ERR(ecc_dev->base_reg); 900 goto list_del; 901 } 902 903 /* Get and request IRQ */ 904 ecc_dev->irq = platform_get_irq(pdev, 0); 905 if (ecc_dev->irq < 0) { 906 rc = ecc_dev->irq; 907 goto list_del; 908 } 909 910 rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler, 911 NULL, 0, "keembay-ocs-ecc", ecc_dev); 912 if (rc < 0) { 913 dev_err(dev, "Could not request IRQ\n"); 914 goto list_del; 915 } 916 917 /* Add device to the list of OCS ECC devices. */ 918 spin_lock(&ocs_ecc.lock); 919 list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list); 920 spin_unlock(&ocs_ecc.lock); 921 922 /* Initialize crypto engine. */ 923 ecc_dev->engine = crypto_engine_alloc_init(dev, 1); 924 if (!ecc_dev->engine) { 925 dev_err(dev, "Could not allocate crypto engine\n"); 926 rc = -ENOMEM; 927 goto list_del; 928 } 929 930 rc = crypto_engine_start(ecc_dev->engine); 931 if (rc) { 932 dev_err(dev, "Could not start crypto engine\n"); 933 goto cleanup; 934 } 935 936 /* Register the KPP algo. */ 937 rc = crypto_engine_register_kpp(&ocs_ecdh_p256); 938 if (rc) { 939 dev_err(dev, 940 "Could not register OCS algorithms with Crypto API\n"); 941 goto cleanup; 942 } 943 944 rc = crypto_engine_register_kpp(&ocs_ecdh_p384); 945 if (rc) { 946 dev_err(dev, 947 "Could not register OCS algorithms with Crypto API\n"); 948 goto ocs_ecdh_p384_error; 949 } 950 951 return 0; 952 953ocs_ecdh_p384_error: 954 crypto_engine_unregister_kpp(&ocs_ecdh_p256); 955 956cleanup: 957 crypto_engine_exit(ecc_dev->engine); 958 959list_del: 960 spin_lock(&ocs_ecc.lock); 961 list_del(&ecc_dev->list); 962 spin_unlock(&ocs_ecc.lock); 963 964 return rc; 965} 966 967static void kmb_ocs_ecc_remove(struct platform_device *pdev) 968{ 969 struct ocs_ecc_dev *ecc_dev; 970 971 ecc_dev = platform_get_drvdata(pdev); 972 973 crypto_engine_unregister_kpp(&ocs_ecdh_p384); 974 crypto_engine_unregister_kpp(&ocs_ecdh_p256); 975 976 spin_lock(&ocs_ecc.lock); 977 list_del(&ecc_dev->list); 978 spin_unlock(&ocs_ecc.lock); 979 980 crypto_engine_exit(ecc_dev->engine); 981} 982 983/* Device tree driver match. */ 984static const struct of_device_id kmb_ocs_ecc_of_match[] = { 985 { 986 .compatible = "intel,keembay-ocs-ecc", 987 }, 988 {} 989}; 990 991/* The OCS driver is a platform device. */ 992static struct platform_driver kmb_ocs_ecc_driver = { 993 .probe = kmb_ocs_ecc_probe, 994 .remove_new = kmb_ocs_ecc_remove, 995 .driver = { 996 .name = DRV_NAME, 997 .of_match_table = kmb_ocs_ecc_of_match, 998 }, 999}; 1000module_platform_driver(kmb_ocs_ecc_driver); 1001 1002MODULE_LICENSE("GPL"); 1003MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver"); 1004MODULE_ALIAS_CRYPTO("ecdh-nist-p256"); 1005MODULE_ALIAS_CRYPTO("ecdh-nist-p384"); 1006MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs"); 1007MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs"); 1008