1/*- 2 * Copyright (c) 2003-2012 Broadcom Corporation 3 * All Rights Reserved 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: releng/10.3/sys/mips/nlm/dev/sec/nlmrsa.c 256046 2013-10-04 11:11:51Z jchandra $"); 31 32#include <sys/cdefs.h> 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/proc.h> 36#include <sys/errno.h> 37#include <sys/endian.h> 38#include <sys/malloc.h> 39#include <sys/kernel.h> 40#include <sys/module.h> 41#include <sys/mbuf.h> 42#include <sys/lock.h> 43#include <sys/mutex.h> 44#include <sys/sysctl.h> 45#include <sys/bus.h> 46#include <sys/random.h> 47#include <sys/rman.h> 48#include <sys/uio.h> 49#include <sys/kobj.h> 50 51#include <dev/pci/pcivar.h> 52 53#include <opencrypto/cryptodev.h> 54 55#include "cryptodev_if.h" 56 57#include <vm/vm.h> 58#include <vm/pmap.h> 59 60#include <mips/nlm/hal/haldefs.h> 61#include <mips/nlm/hal/iomap.h> 62#include <mips/nlm/xlp.h> 63#include <mips/nlm/hal/sys.h> 64#include <mips/nlm/hal/fmn.h> 65#include <mips/nlm/hal/nlmsaelib.h> 66#include <mips/nlm/dev/sec/rsa_ucode.h> 67#include <mips/nlm/hal/cop2.h> 68#include <mips/nlm/hal/mips-extns.h> 69#include <mips/nlm/msgring.h> 70#include <mips/nlm/dev/sec/nlmrsalib.h> 71 72#ifdef NLM_RSA_DEBUG 73static void print_krp_params(struct cryptkop *krp); 74#endif 75 76static int xlp_rsa_init(struct xlp_rsa_softc *sc, int node); 77static int xlp_rsa_newsession(device_t , uint32_t *, struct cryptoini *); 78static int xlp_rsa_freesession(device_t , uint64_t); 79static int xlp_rsa_kprocess(device_t , struct cryptkop *, int); 80static int xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits); 81static void xlp_free_cmd_params(struct xlp_rsa_command *cmd); 82static int xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst, 83 uint32_t paramsize, uint8_t result); 84 85static int xlp_rsa_probe(device_t); 86static int xlp_rsa_attach(device_t); 87static int xlp_rsa_detach(device_t); 88 89static device_method_t xlp_rsa_methods[] = { 90 /* device interface */ 91 DEVMETHOD(device_probe, xlp_rsa_probe), 92 DEVMETHOD(device_attach, xlp_rsa_attach), 93 DEVMETHOD(device_detach, xlp_rsa_detach), 94 95 /* bus interface */ 96 DEVMETHOD(bus_print_child, bus_generic_print_child), 97 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 98 99 /* crypto device methods */ 100 DEVMETHOD(cryptodev_newsession, xlp_rsa_newsession), 101 DEVMETHOD(cryptodev_freesession, xlp_rsa_freesession), 102 DEVMETHOD(cryptodev_kprocess, xlp_rsa_kprocess), 103 104 DEVMETHOD_END 105}; 106 107static driver_t xlp_rsa_driver = { 108 "nlmrsa", 109 xlp_rsa_methods, 110 sizeof(struct xlp_rsa_softc) 111}; 112static devclass_t xlp_rsa_devclass; 113 114DRIVER_MODULE(nlmrsa, pci, xlp_rsa_driver, xlp_rsa_devclass, 0, 0); 115MODULE_DEPEND(nlmrsa, crypto, 1, 1, 1); 116 117#ifdef NLM_RSA_DEBUG 118static void 119print_krp_params(struct cryptkop *krp) 120{ 121 int i; 122 123 printf("krp->krp_op :%d\n", krp->krp_op); 124 printf("krp->krp_status :%d\n", krp->krp_status); 125 printf("krp->krp_iparams:%d\n", krp->krp_iparams); 126 printf("krp->krp_oparams:%d\n", krp->krp_oparams); 127 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 128 printf("krp->krp_param[%d].crp_p :0x%llx\n", i, 129 (unsigned long long)krp->krp_param[i].crp_p); 130 printf("krp->krp_param[%d].crp_nbits :%d\n", i, 131 krp->krp_param[i].crp_nbits); 132 printf("krp->krp_param[%d].crp_nbytes :%d\n", i, 133 howmany(krp->krp_param[i].crp_nbits, 8)); 134 } 135} 136#endif 137 138static int 139xlp_rsa_init(struct xlp_rsa_softc *sc, int node) 140{ 141 struct xlp_rsa_command *cmd = NULL; 142 uint32_t fbvc, dstvc, endsel, regval; 143 struct nlm_fmn_msg m; 144 int err, ret, i; 145 uint64_t base; 146 147 /* Register interrupt handler for the RSA/ECC CMS messages */ 148 if (register_msgring_handler(sc->rsaecc_vc_start, 149 sc->rsaecc_vc_end, nlm_xlprsaecc_msgring_handler, sc) != 0) { 150 err = -1; 151 printf("Couldn't register rsa/ecc msgring handler\n"); 152 goto errout; 153 } 154 fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC; 155 /* Do the CMS credit initialization */ 156 /* Currently it is configured by default to 50 when kernel comes up */ 157 158#if BYTE_ORDER == LITTLE_ENDIAN 159 for (i = 0; i < nitems(nlm_rsa_ucode_data); i++) 160 nlm_rsa_ucode_data[i] = htobe64(nlm_rsa_ucode_data[i]); 161#endif 162 for (dstvc = sc->rsaecc_vc_start; dstvc <= sc->rsaecc_vc_end; dstvc++) { 163 cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF, 164 M_NOWAIT | M_ZERO); 165 KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__)); 166 cmd->rsasrc = contigmalloc(sizeof(nlm_rsa_ucode_data), 167 M_DEVBUF, 168 (M_WAITOK | M_ZERO), 169 0UL /* low address */, -1UL /* high address */, 170 XLP_L2L3_CACHELINE_SIZE /* alignment */, 171 0UL /* boundary */); 172 KASSERT(cmd->rsasrc != NULL, 173 ("%s:cmd->rsasrc is NULL\n", __func__)); 174 memcpy(cmd->rsasrc, nlm_rsa_ucode_data, 175 sizeof(nlm_rsa_ucode_data)); 176 m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, 0x70, 0, 177 vtophys(cmd->rsasrc)); 178 m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc, 179 vtophys(cmd->rsasrc)); 180 /* Software scratch pad */ 181 m.msg[2] = (uintptr_t)cmd; 182 m.msg[3] = 0; 183 184 ret = nlm_fmn_msgsend(dstvc, 3, FMN_SWCODE_RSA, &m); 185 if (ret != 0) { 186 err = -1; 187 printf("%s: msgsnd failed (%x)\n", __func__, ret); 188 goto errout; 189 } 190 } 191 /* Configure so that all VCs send request to all RSA pipes */ 192 base = nlm_get_rsa_regbase(node); 193 if (nlm_is_xlp3xx()) { 194 endsel = 1; 195 regval = 0xFFFF; 196 } else { 197 endsel = 3; 198 regval = 0x07FFFFFF; 199 } 200 for (i = 0; i < endsel; i++) 201 nlm_write_rsa_reg(base, RSA_ENG_SEL_0 + i, regval); 202 return (0); 203errout: 204 xlp_free_cmd_params(cmd); 205 return (err); 206} 207 208/* This function is called from an interrupt handler */ 209void 210nlm_xlprsaecc_msgring_handler(int vc, int size, int code, int src_id, 211 struct nlm_fmn_msg *msg, void *data) 212{ 213 struct xlp_rsa_command *cmd; 214 struct xlp_rsa_softc *sc; 215 struct crparam *outparam; 216 int ostart; 217 218 KASSERT(code == FMN_SWCODE_RSA, 219 ("%s: bad code = %d, expected code = %d\n", __func__, code, 220 FMN_SWCODE_RSA)); 221 222 sc = data; 223 KASSERT(src_id >= sc->rsaecc_vc_start && src_id <= sc->rsaecc_vc_end, 224 ("%s: bad src_id = %d, expect %d - %d\n", __func__, 225 src_id, sc->rsaecc_vc_start, sc->rsaecc_vc_end)); 226 227 cmd = (struct xlp_rsa_command *)(uintptr_t)msg->msg[1]; 228 KASSERT(cmd != NULL, ("%s:cmd not received properly\n", __func__)); 229 230 if (RSA_ERROR(msg->msg[0]) != 0) { 231 printf("%s: Message rcv msg0 %llx msg1 %llx err %x \n", 232 __func__, (unsigned long long)msg->msg[0], 233 (unsigned long long)msg->msg[1], 234 (int)RSA_ERROR(msg->msg[0])); 235 cmd->krp->krp_status = EBADMSG; 236 } 237 238 if (cmd->krp != NULL) { 239 ostart = cmd->krp->krp_iparams; 240 outparam = &cmd->krp->krp_param[ostart]; 241 xlp_rsa_inp2hwformat(cmd->rsasrc + cmd->rsaopsize * ostart, 242 outparam->crp_p, 243 howmany(outparam->crp_nbits, 8), 244 1); 245 crypto_kdone(cmd->krp); 246 } 247 248 xlp_free_cmd_params(cmd); 249} 250 251static int 252xlp_rsa_probe(device_t dev) 253{ 254 struct xlp_rsa_softc *sc; 255 256 if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC && 257 pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) { 258 sc = device_get_softc(dev); 259 return (BUS_PROBE_DEFAULT); 260 } 261 return (ENXIO); 262} 263 264/* 265 * Attach an interface that successfully probed. 266 */ 267static int 268xlp_rsa_attach(device_t dev) 269{ 270 struct xlp_rsa_softc *sc = device_get_softc(dev); 271 uint64_t base; 272 int qstart, qnum; 273 int freq, node; 274 275 sc->sc_dev = dev; 276 277 node = nlm_get_device_node(pci_get_slot(dev)); 278 freq = nlm_set_device_frequency(node, DFS_DEVICE_RSA, 250); 279 if (bootverbose) 280 device_printf(dev, "RSA Freq: %dMHz\n", freq); 281 if (pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) { 282 device_set_desc(dev, "XLP RSA/ECC Accelerator"); 283 if ((sc->sc_cid = crypto_get_driverid(dev, 284 CRYPTOCAP_F_HARDWARE)) < 0) { 285 printf("xlp_rsaecc-err:couldn't get the driver id\n"); 286 goto error_exit; 287 } 288 if (crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0) != 0) 289 goto error_exit; 290 291 base = nlm_get_rsa_pcibase(node); 292 qstart = nlm_qidstart(base); 293 qnum = nlm_qnum(base); 294 sc->rsaecc_vc_start = qstart; 295 sc->rsaecc_vc_end = qstart + qnum - 1; 296 } 297 if (xlp_rsa_init(sc, node) != 0) 298 goto error_exit; 299 device_printf(dev, "RSA Initialization complete!\n"); 300 return (0); 301 302error_exit: 303 return (ENXIO); 304} 305 306/* 307 * Detach an interface that successfully probed. 308 */ 309static int 310xlp_rsa_detach(device_t dev) 311{ 312 return (0); 313} 314 315/* 316 * Allocate a new 'session' and return an encoded session id. 'sidp' 317 * contains our registration id, and should contain an encoded session 318 * id on successful allocation. 319 */ 320static int 321xlp_rsa_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 322{ 323 struct xlp_rsa_softc *sc = device_get_softc(dev); 324 struct xlp_rsa_session *ses = NULL; 325 int sesn; 326 327 if (sidp == NULL || cri == NULL || sc == NULL) 328 return (EINVAL); 329 330 if (sc->sc_sessions == NULL) { 331 ses = sc->sc_sessions = malloc(sizeof(struct xlp_rsa_session), 332 M_DEVBUF, M_NOWAIT); 333 if (ses == NULL) 334 return (ENOMEM); 335 sesn = 0; 336 sc->sc_nsessions = 1; 337 } else { 338 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 339 if (!sc->sc_sessions[sesn].hs_used) { 340 ses = &sc->sc_sessions[sesn]; 341 break; 342 } 343 } 344 345 if (ses == NULL) { 346 sesn = sc->sc_nsessions; 347 ses = malloc((sesn + 1) * sizeof(*ses), 348 M_DEVBUF, M_NOWAIT); 349 if (ses == NULL) 350 return (ENOMEM); 351 bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses)); 352 bzero(sc->sc_sessions, sesn * sizeof(*ses)); 353 free(sc->sc_sessions, M_DEVBUF); 354 sc->sc_sessions = ses; 355 ses = &sc->sc_sessions[sesn]; 356 sc->sc_nsessions++; 357 } 358 } 359 bzero(ses, sizeof(*ses)); 360 ses->sessionid = sesn; 361 ses->hs_used = 1; 362 363 *sidp = XLP_RSA_SID(device_get_unit(sc->sc_dev), sesn); 364 return (0); 365} 366 367/* 368 * Deallocate a session. 369 * XXX this routine should run a zero'd mac/encrypt key into context ram. 370 * XXX to blow away any keys already stored there. 371 */ 372static int 373xlp_rsa_freesession(device_t dev, u_int64_t tid) 374{ 375 struct xlp_rsa_softc *sc = device_get_softc(dev); 376 int session; 377 u_int32_t sid = CRYPTO_SESID2LID(tid); 378 379 if (sc == NULL) 380 return (EINVAL); 381 382 session = XLP_RSA_SESSION(sid); 383 if (session >= sc->sc_nsessions) 384 return (EINVAL); 385 386 sc->sc_sessions[session].hs_used = 0; 387 return (0); 388} 389 390static void 391xlp_free_cmd_params(struct xlp_rsa_command *cmd) 392{ 393 394 if (cmd == NULL) 395 return; 396 if (cmd->rsasrc != NULL) { 397 if (cmd->krp == NULL) /* Micro code load */ 398 contigfree(cmd->rsasrc, sizeof(nlm_rsa_ucode_data), 399 M_DEVBUF); 400 else 401 free(cmd->rsasrc, M_DEVBUF); 402 } 403 free(cmd, M_DEVBUF); 404} 405 406static int 407xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits) 408{ 409 410 if (bits == 0 || bits > 8192) 411 return (-1); 412 /* XLP hardware expects always a fixed size with unused bytes 413 * zeroed out in the input data */ 414 if (bits <= 512) { 415 cmd->rsatype = 0x40; 416 cmd->rsaopsize = 64; 417 } else if (bits <= 1024) { 418 cmd->rsatype = 0x41; 419 cmd->rsaopsize = 128; 420 } else if (bits <= 2048) { 421 cmd->rsatype = 0x42; 422 cmd->rsaopsize = 256; 423 } else if (bits <= 4096) { 424 cmd->rsatype = 0x43; 425 cmd->rsaopsize = 512; 426 } else if (bits <= 8192) { 427 cmd->rsatype = 0x44; 428 cmd->rsaopsize = 1024; 429 } 430 return (0); 431} 432 433static int 434xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst, uint32_t paramsize, 435 uint8_t result) 436{ 437 uint32_t pdwords, pbytes; 438 int i, j, k; 439 440 pdwords = paramsize / 8; 441 pbytes = paramsize % 8; 442 443 for (i = 0, k = 0; i < pdwords; i++) { 444 /* copy dwords of inp/hw to hw/out format */ 445 for (j = 7; j >= 0; j--, k++) 446 dst[i * 8 + j] = src[k]; 447 } 448 if (pbytes) { 449 if (result == 0) { 450 /* copy rem bytes of input data to hw format */ 451 for (j = 7; k < paramsize; j--, k++) 452 dst[i * 8 + j] = src[k]; 453 } else { 454 /* copy rem bytes of hw data to exp output format */ 455 for (j = 7; k < paramsize; j--, k++) 456 dst[k] = src[i * 8 + j]; 457 } 458 } 459 460 return (0); 461} 462 463static int 464nlm_crypto_complete_rsa_request(struct xlp_rsa_softc *sc, 465 struct xlp_rsa_command *cmd) 466{ 467 unsigned int fbvc; 468 struct nlm_fmn_msg m; 469 int ret; 470 471 fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC; 472 473 m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, cmd->rsatype, 474 cmd->rsafn, vtophys(cmd->rsasrc)); 475 m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc, 476 vtophys(cmd->rsasrc + cmd->rsaopsize * cmd->krp->krp_iparams)); 477 /* Software scratch pad */ 478 m.msg[2] = (uintptr_t)cmd; 479 m.msg[3] = 0; 480 481 /* Send the message to rsa engine vc */ 482 ret = nlm_fmn_msgsend(sc->rsaecc_vc_start, 3, FMN_SWCODE_RSA, &m); 483 if (ret != 0) { 484#ifdef NLM_SEC_DEBUG 485 printf("%s: msgsnd failed (%x)\n", __func__, ret); 486#endif 487 return (ERESTART); 488 } 489 return (0); 490} 491 492static int 493xlp_rsa_kprocess(device_t dev, struct cryptkop *krp, int hint) 494{ 495 struct xlp_rsa_softc *sc = device_get_softc(dev); 496 struct xlp_rsa_command *cmd; 497 struct crparam *kp; 498 int err, i; 499 500 if (krp == NULL || krp->krp_callback == NULL) 501 return (EINVAL); 502 503 cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF, 504 M_NOWAIT | M_ZERO); 505 KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__)); 506 cmd->krp = krp; 507 508#ifdef NLM_RSA_DEBUG 509 print_krp_params(krp); 510#endif 511 err = EOPNOTSUPP; 512 switch (krp->krp_op) { 513 case CRK_MOD_EXP: 514 if (krp->krp_iparams == 3 && krp->krp_oparams == 1) 515 break; 516 goto errout; 517 default: 518 device_printf(dev, "Op:%d not yet supported\n", krp->krp_op); 519 goto errout; 520 } 521 522 err = xlp_get_rsa_opsize(cmd, 523 krp->krp_param[krp->krp_iparams - 1].crp_nbits); 524 if (err != 0) { 525 err = EINVAL; 526 goto errout; 527 } 528 cmd->rsafn = 0; /* Mod Exp */ 529 cmd->rsasrc = malloc( 530 cmd->rsaopsize * (krp->krp_iparams + krp->krp_oparams), 531 M_DEVBUF, 532 M_NOWAIT | M_ZERO); 533 if (cmd->rsasrc == NULL) { 534 err = ENOMEM; 535 goto errout; 536 } 537 538 for (i = 0, kp = krp->krp_param; i < krp->krp_iparams; i++, kp++) { 539 KASSERT(kp->crp_nbits != 0, 540 ("%s: parameter[%d]'s length is zero\n", __func__, i)); 541 xlp_rsa_inp2hwformat(kp->crp_p, 542 cmd->rsasrc + i * cmd->rsaopsize, 543 howmany(kp->crp_nbits, 8), 0); 544 } 545 err = nlm_crypto_complete_rsa_request(sc, cmd); 546 if (err != 0) 547 goto errout; 548 549 return (0); 550errout: 551 xlp_free_cmd_params(cmd); 552 krp->krp_status = err; 553 crypto_kdone(krp); 554 return (err); 555} 556