1/* $NetBSD: mvxpsec.c,v 1.20 2024/02/09 22:08:35 andvar Exp $ */ 2/* 3 * Copyright (c) 2015 Internet Initiative Japan Inc. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 19 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 23 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#ifdef _KERNEL_OPT 29#include "opt_ipsec.h" 30#endif 31 32/* 33 * Cryptographic Engine and Security Accelerator(MVXPSEC) 34 */ 35#include <sys/cdefs.h> 36#include <sys/param.h> 37#include <sys/types.h> 38#include <sys/kernel.h> 39#include <sys/queue.h> 40#include <sys/conf.h> 41#include <sys/proc.h> 42#include <sys/bus.h> 43#include <sys/evcnt.h> 44#include <sys/device.h> 45#include <sys/endian.h> 46#include <sys/errno.h> 47#include <sys/kmem.h> 48#include <sys/mbuf.h> 49#include <sys/callout.h> 50#include <sys/pool.h> 51#include <sys/cprng.h> 52#include <sys/syslog.h> 53#include <sys/mutex.h> 54#include <sys/kthread.h> 55#include <sys/atomic.h> 56#include <sys/sha1.h> 57#include <sys/md5.h> 58 59#include <uvm/uvm_extern.h> 60 61#include <opencrypto/cryptodev.h> 62#include <opencrypto/xform.h> 63 64#include <net/net_stats.h> 65 66#include <netinet/in_systm.h> 67#include <netinet/in.h> 68#include <netinet/ip.h> 69#include <netinet/ip6.h> 70 71#if NIPSEC > 0 72#include <netipsec/esp_var.h> 73#endif 74 75#include <arm/cpufunc.h> 76#include <arm/marvell/mvsocvar.h> 77#include <arm/marvell/armadaxpreg.h> 78#include <dev/marvell/marvellreg.h> 79#include <dev/marvell/marvellvar.h> 80#include <dev/marvell/mvxpsecreg.h> 81#include <dev/marvell/mvxpsecvar.h> 82 83#ifdef DEBUG 84#define STATIC __attribute__ ((noinline)) extern 85#define _STATIC __attribute__ ((noinline)) extern 86#define INLINE __attribute__ ((noinline)) extern 87#define _INLINE __attribute__ ((noinline)) extern 88#else 89#define STATIC static 90#define _STATIC __attribute__ ((unused)) static 91#define INLINE static inline 92#define _INLINE __attribute__ ((unused)) static inline 93#endif 94 95/* 96 * IRQ and SRAM spaces for each of unit 97 * XXX: move to attach_args 98 */ 99struct { 100 int err_int; 101} mvxpsec_config[] = { 102 { .err_int = ARMADAXP_IRQ_CESA0_ERR, }, /* unit 0 */ 103 { .err_int = ARMADAXP_IRQ_CESA1_ERR, }, /* unit 1 */ 104}; 105#define MVXPSEC_ERR_INT(sc) \ 106 mvxpsec_config[device_unit((sc)->sc_dev)].err_int 107 108/* 109 * AES 110 */ 111#define MAXBC (128/32) 112#define MAXKC (256/32) 113#define MAXROUNDS 14 114STATIC int mv_aes_ksched(uint8_t[4][MAXKC], int, 115 uint8_t[MAXROUNDS+1][4][MAXBC]); 116STATIC int mv_aes_deckey(uint8_t *, uint8_t *, int); 117 118/* 119 * device driver autoconf interface 120 */ 121STATIC int mvxpsec_match(device_t, cfdata_t, void *); 122STATIC void mvxpsec_attach(device_t, device_t, void *); 123STATIC void mvxpsec_evcnt_attach(struct mvxpsec_softc *); 124 125/* 126 * register setup 127 */ 128STATIC int mvxpsec_wininit(struct mvxpsec_softc *, enum marvell_tags *); 129 130/* 131 * timer(callout) interface 132 * 133 * XXX: callout is not MP safe... 134 */ 135STATIC void mvxpsec_timer(void *); 136 137/* 138 * interrupt interface 139 */ 140STATIC int mvxpsec_intr(void *); 141INLINE void mvxpsec_intr_cleanup(struct mvxpsec_softc *); 142STATIC int mvxpsec_eintr(void *); 143STATIC uint32_t mvxpsec_intr_ack(struct mvxpsec_softc *); 144STATIC uint32_t mvxpsec_eintr_ack(struct mvxpsec_softc *); 145INLINE void mvxpsec_intr_cnt(struct mvxpsec_softc *, int); 146 147/* 148 * memory allocators and VM management 149 */ 150STATIC struct mvxpsec_devmem *mvxpsec_alloc_devmem(struct mvxpsec_softc *, 151 paddr_t, int); 152STATIC int mvxpsec_init_sram(struct mvxpsec_softc *); 153 154/* 155 * Low-level DMA interface 156 */ 157STATIC int mvxpsec_init_dma(struct mvxpsec_softc *, 158 struct marvell_attach_args *); 159INLINE int mvxpsec_dma_wait(struct mvxpsec_softc *); 160INLINE int mvxpsec_acc_wait(struct mvxpsec_softc *); 161INLINE struct mvxpsec_descriptor_handle *mvxpsec_dma_getdesc(struct mvxpsec_softc *); 162_INLINE void mvxpsec_dma_putdesc(struct mvxpsec_softc *, struct mvxpsec_descriptor_handle *); 163INLINE void mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *, 164 uint32_t, uint32_t, uint32_t); 165INLINE void mvxpsec_dma_cat(struct mvxpsec_softc *, 166 struct mvxpsec_descriptor_handle *, struct mvxpsec_descriptor_handle *); 167 168/* 169 * High-level DMA interface 170 */ 171INLINE int mvxpsec_dma_copy0(struct mvxpsec_softc *, 172 mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t); 173INLINE int mvxpsec_dma_copy(struct mvxpsec_softc *, 174 mvxpsec_dma_ring *, uint32_t, uint32_t, uint32_t); 175INLINE int mvxpsec_dma_acc_activate(struct mvxpsec_softc *, 176 mvxpsec_dma_ring *); 177INLINE void mvxpsec_dma_finalize(struct mvxpsec_softc *, 178 mvxpsec_dma_ring *); 179INLINE void mvxpsec_dma_free(struct mvxpsec_softc *, 180 mvxpsec_dma_ring *); 181INLINE int mvxpsec_dma_copy_packet(struct mvxpsec_softc *, struct mvxpsec_packet *); 182INLINE int mvxpsec_dma_sync_packet(struct mvxpsec_softc *, struct mvxpsec_packet *); 183 184/* 185 * Session management interface (OpenCrypto) 186 */ 187#define MVXPSEC_SESSION(sid) ((sid) & 0x0fffffff) 188#define MVXPSEC_SID(crd, sesn) (((crd) << 28) | ((sesn) & 0x0fffffff)) 189/* pool management */ 190STATIC int mvxpsec_session_ctor(void *, void *, int); 191STATIC void mvxpsec_session_dtor(void *, void *); 192STATIC int mvxpsec_packet_ctor(void *, void *, int); 193STATIC void mvxpsec_packet_dtor(void *, void *); 194 195/* session management */ 196STATIC struct mvxpsec_session *mvxpsec_session_alloc(struct mvxpsec_softc *); 197STATIC void mvxpsec_session_dealloc(struct mvxpsec_session *); 198INLINE struct mvxpsec_session *mvxpsec_session_lookup(struct mvxpsec_softc *, int); 199INLINE int mvxpsec_session_ref(struct mvxpsec_session *); 200INLINE void mvxpsec_session_unref(struct mvxpsec_session *); 201 202/* packet management */ 203STATIC struct mvxpsec_packet *mvxpsec_packet_alloc(struct mvxpsec_session *); 204INLINE void mvxpsec_packet_enqueue(struct mvxpsec_packet *); 205STATIC void mvxpsec_packet_dealloc(struct mvxpsec_packet *); 206STATIC int mvxpsec_done_packet(struct mvxpsec_packet *); 207 208/* session header manegement */ 209STATIC int mvxpsec_header_finalize(struct mvxpsec_packet *); 210 211/* packet queue management */ 212INLINE void mvxpsec_drop(struct mvxpsec_softc *, struct cryptop *, struct mvxpsec_packet *, int); 213STATIC int mvxpsec_dispatch_queue(struct mvxpsec_softc *); 214 215/* opencrypto operation */ 216INLINE int mvxpsec_parse_crd(struct mvxpsec_packet *, struct cryptodesc *); 217INLINE int mvxpsec_parse_crp(struct mvxpsec_packet *); 218 219/* payload data management */ 220INLINE int mvxpsec_packet_setcrp(struct mvxpsec_packet *, struct cryptop *); 221STATIC int mvxpsec_packet_setdata(struct mvxpsec_packet *, void *, uint32_t); 222STATIC int mvxpsec_packet_setmbuf(struct mvxpsec_packet *, struct mbuf *); 223STATIC int mvxpsec_packet_setuio(struct mvxpsec_packet *, struct uio *); 224STATIC int mvxpsec_packet_rdata(struct mvxpsec_packet *, int, int, void *); 225_STATIC int mvxpsec_packet_wdata(struct mvxpsec_packet *, int, int, void *); 226STATIC int mvxpsec_packet_write_iv(struct mvxpsec_packet *, void *, int); 227STATIC int mvxpsec_packet_copy_iv(struct mvxpsec_packet *, int, int); 228 229/* key pre-computation */ 230STATIC int mvxpsec_key_precomp(int, void *, int, void *, void *); 231STATIC int mvxpsec_hmac_precomp(int, void *, int, void *, void *); 232 233/* crypto operation management */ 234INLINE void mvxpsec_packet_reset_op(struct mvxpsec_packet *); 235INLINE void mvxpsec_packet_update_op_order(struct mvxpsec_packet *, int); 236 237/* 238 * parameter converters 239 */ 240INLINE uint32_t mvxpsec_alg2acc(uint32_t alg); 241INLINE uint32_t mvxpsec_aesklen(int klen); 242 243/* 244 * string formatters 245 */ 246_STATIC const char *s_ctrlreg(uint32_t); 247_STATIC const char *s_winreg(uint32_t); 248_STATIC const char *s_errreg(uint32_t); 249_STATIC const char *s_xpsecintr(uint32_t); 250_STATIC const char *s_ctlalg(uint32_t); 251_STATIC const char *s_xpsec_op(uint32_t); 252_STATIC const char *s_xpsec_enc(uint32_t); 253_STATIC const char *s_xpsec_mac(uint32_t); 254_STATIC const char *s_xpsec_frag(uint32_t); 255 256/* 257 * debugging supports 258 */ 259#ifdef MVXPSEC_DEBUG 260_STATIC void mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *); 261_STATIC void mvxpsec_dump_reg(struct mvxpsec_softc *); 262_STATIC void mvxpsec_dump_sram(const char *, struct mvxpsec_softc *, size_t); 263_STATIC void mvxpsec_dump_data(const char *, void *, size_t); 264 265_STATIC void mvxpsec_dump_packet(const char *, struct mvxpsec_packet *); 266_STATIC void mvxpsec_dump_packet_data(const char *, struct mvxpsec_packet *); 267_STATIC void mvxpsec_dump_packet_desc(const char *, struct mvxpsec_packet *); 268 269_STATIC void mvxpsec_dump_acc_config(const char *, uint32_t); 270_STATIC void mvxpsec_dump_acc_encdata(const char *, uint32_t, uint32_t); 271_STATIC void mvxpsec_dump_acc_enclen(const char *, uint32_t); 272_STATIC void mvxpsec_dump_acc_enckey(const char *, uint32_t); 273_STATIC void mvxpsec_dump_acc_enciv(const char *, uint32_t); 274_STATIC void mvxpsec_dump_acc_macsrc(const char *, uint32_t); 275_STATIC void mvxpsec_dump_acc_macdst(const char *, uint32_t); 276_STATIC void mvxpsec_dump_acc_maciv(const char *, uint32_t); 277#endif 278 279/* 280 * global configurations, params, work spaces, ... 281 * 282 * XXX: use sysctl for global configurations 283 */ 284/* waiting for device */ 285static int mvxpsec_wait_interval = 10; /* usec */ 286static int mvxpsec_wait_retry = 100; /* times = wait for 1 [msec] */ 287#ifdef MVXPSEC_DEBUG 288static uint32_t mvxpsec_debug = MVXPSEC_DEBUG; /* debug level */ 289#endif 290 291/* 292 * Register accessors 293 */ 294#define MVXPSEC_WRITE(sc, off, val) \ 295 bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (off), (val)) 296#define MVXPSEC_READ(sc, off) \ 297 bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (off)) 298 299/* 300 * device driver autoconf interface 301 */ 302CFATTACH_DECL2_NEW(mvxpsec_mbus, sizeof(struct mvxpsec_softc), 303 mvxpsec_match, mvxpsec_attach, NULL, NULL, NULL, NULL); 304 305STATIC int 306mvxpsec_match(device_t dev, cfdata_t match, void *aux) 307{ 308 struct marvell_attach_args *mva = aux; 309 uint32_t tag; 310 int window; 311 312 if (strcmp(mva->mva_name, match->cf_name) != 0) 313 return 0; 314 if (mva->mva_offset == MVA_OFFSET_DEFAULT) 315 return 0; 316 317 switch (mva->mva_unit) { 318 case 0: 319 tag = ARMADAXP_TAG_CRYPT0; 320 break; 321 case 1: 322 tag = ARMADAXP_TAG_CRYPT1; 323 break; 324 default: 325 aprint_error_dev(dev, 326 "unit %d is not supported\n", mva->mva_unit); 327 return 0; 328 } 329 330 window = mvsoc_target(tag, NULL, NULL, NULL, NULL); 331 if (window >= nwindow) { 332 aprint_error_dev(dev, 333 "Security Accelerator SRAM is not configured.\n"); 334 return 0; 335 } 336 337 return 1; 338} 339 340STATIC void 341mvxpsec_attach(device_t parent, device_t self, void *aux) 342{ 343 struct marvell_attach_args *mva = aux; 344 struct mvxpsec_softc *sc = device_private(self); 345 int v; 346 int i; 347 348 sc->sc_dev = self; 349 350 aprint_normal(": Marvell Crypto Engines and Security Accelerator\n"); 351 aprint_naive("\n"); 352#ifdef MVXPSEC_MULTI_PACKET 353 aprint_normal_dev(sc->sc_dev, "multi-packet chained mode enabled.\n"); 354#else 355 aprint_normal_dev(sc->sc_dev, "multi-packet chained mode disabled.\n"); 356#endif 357 aprint_normal_dev(sc->sc_dev, 358 "Max %d sessions.\n", MVXPSEC_MAX_SESSIONS); 359 360 /* mutex */ 361 mutex_init(&sc->sc_session_mtx, MUTEX_DEFAULT, IPL_NET); 362 mutex_init(&sc->sc_dma_mtx, MUTEX_DEFAULT, IPL_NET); 363 mutex_init(&sc->sc_queue_mtx, MUTEX_DEFAULT, IPL_NET); 364 365 /* Packet queue */ 366 SIMPLEQ_INIT(&sc->sc_wait_queue); 367 SIMPLEQ_INIT(&sc->sc_run_queue); 368 SLIST_INIT(&sc->sc_free_list); 369 sc->sc_wait_qlen = 0; 370#ifdef MVXPSEC_MULTI_PACKET 371 sc->sc_wait_qlimit = 16; 372#else 373 sc->sc_wait_qlimit = 0; 374#endif 375 sc->sc_free_qlen = 0; 376 377 /* Timer */ 378 callout_init(&sc->sc_timeout, 0); /* XXX: use CALLOUT_MPSAFE */ 379 callout_setfunc(&sc->sc_timeout, mvxpsec_timer, sc); 380 381 /* I/O */ 382 sc->sc_iot = mva->mva_iot; 383 if (bus_space_subregion(mva->mva_iot, mva->mva_ioh, 384 mva->mva_offset, mva->mva_size, &sc->sc_ioh)) { 385 aprint_error_dev(self, "Cannot map registers\n"); 386 return; 387 } 388 389 /* DMA */ 390 sc->sc_dmat = mva->mva_dmat; 391 if (mvxpsec_init_dma(sc, mva) < 0) 392 return; 393 394 /* SRAM */ 395 if (mvxpsec_init_sram(sc) < 0) 396 return; 397 398 /* Registers */ 399 mvxpsec_wininit(sc, mva->mva_tags); 400 401 /* INTR */ 402 MVXPSEC_WRITE(sc, MVXPSEC_INT_MASK, MVXPSEC_DEFAULT_INT); 403 MVXPSEC_WRITE(sc, MV_TDMA_ERR_MASK, MVXPSEC_DEFAULT_ERR); 404 sc->sc_done_ih = 405 marvell_intr_establish(mva->mva_irq, IPL_NET, mvxpsec_intr, sc); 406 /* XXX: should pass error IRQ using mva */ 407 sc->sc_error_ih = marvell_intr_establish(MVXPSEC_ERR_INT(sc), 408 IPL_NET, mvxpsec_eintr, sc); 409 aprint_normal_dev(self, 410 "Error Reporting IRQ %d\n", MVXPSEC_ERR_INT(sc)); 411 412 /* Initialize TDMA (It's enabled here, but waiting for SA) */ 413 if (mvxpsec_dma_wait(sc) < 0) 414 panic("%s: DMA DEVICE not responding\n", __func__); 415 MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0); 416 MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0); 417 MVXPSEC_WRITE(sc, MV_TDMA_DST, 0); 418 MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0); 419 MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0); 420 v = MVXPSEC_READ(sc, MV_TDMA_CONTROL); 421 v |= MV_TDMA_CONTROL_ENABLE; 422 MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, v); 423 424 /* Initialize SA */ 425 if (mvxpsec_acc_wait(sc) < 0) 426 panic("%s: MVXPSEC not responding\n", __func__); 427 v = MVXPSEC_READ(sc, MV_ACC_CONFIG); 428 v &= ~MV_ACC_CONFIG_STOP_ON_ERR; 429 v |= MV_ACC_CONFIG_MULT_PKT; 430 v |= MV_ACC_CONFIG_WAIT_TDMA; 431 v |= MV_ACC_CONFIG_ACT_TDMA; 432 MVXPSEC_WRITE(sc, MV_ACC_CONFIG, v); 433 MVXPSEC_WRITE(sc, MV_ACC_DESC, 0); 434 MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP); 435 436 /* Session */ 437 sc->sc_session_pool = 438 pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0, 439 "mvxpsecpl", NULL, IPL_NET, 440 mvxpsec_session_ctor, mvxpsec_session_dtor, sc); 441 pool_cache_sethiwat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS); 442 pool_cache_setlowat(sc->sc_session_pool, MVXPSEC_MAX_SESSIONS / 2); 443 sc->sc_last_session = NULL; 444 445 /* Packet */ 446 sc->sc_packet_pool = 447 pool_cache_init(sizeof(struct mvxpsec_session), 0, 0, 0, 448 "mvxpsec_pktpl", NULL, IPL_NET, 449 mvxpsec_packet_ctor, mvxpsec_packet_dtor, sc); 450 pool_cache_sethiwat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS); 451 pool_cache_setlowat(sc->sc_packet_pool, MVXPSEC_MAX_SESSIONS / 2); 452 453 /* Register to EVCNT framework */ 454 mvxpsec_evcnt_attach(sc); 455 456 /* Register to Opencrypto */ 457 for (i = 0; i < MVXPSEC_MAX_SESSIONS; i++) { 458 sc->sc_sessions[i] = NULL; 459 } 460 if (mvxpsec_register(sc)) 461 panic("cannot initialize OpenCrypto module.\n"); 462 463 return; 464} 465 466STATIC void 467mvxpsec_evcnt_attach(struct mvxpsec_softc *sc) 468{ 469 struct mvxpsec_evcnt *sc_ev = &sc->sc_ev; 470 471 evcnt_attach_dynamic(&sc_ev->intr_all, EVCNT_TYPE_INTR, 472 NULL, device_xname(sc->sc_dev), "Main Intr."); 473 evcnt_attach_dynamic(&sc_ev->intr_auth, EVCNT_TYPE_INTR, 474 NULL, device_xname(sc->sc_dev), "Auth Intr."); 475 evcnt_attach_dynamic(&sc_ev->intr_des, EVCNT_TYPE_INTR, 476 NULL, device_xname(sc->sc_dev), "DES Intr."); 477 evcnt_attach_dynamic(&sc_ev->intr_aes_enc, EVCNT_TYPE_INTR, 478 NULL, device_xname(sc->sc_dev), "AES-Encrypt Intr."); 479 evcnt_attach_dynamic(&sc_ev->intr_aes_dec, EVCNT_TYPE_INTR, 480 NULL, device_xname(sc->sc_dev), "AES-Decrypt Intr."); 481 evcnt_attach_dynamic(&sc_ev->intr_enc, EVCNT_TYPE_INTR, 482 NULL, device_xname(sc->sc_dev), "Crypto Intr."); 483 evcnt_attach_dynamic(&sc_ev->intr_sa, EVCNT_TYPE_INTR, 484 NULL, device_xname(sc->sc_dev), "SA Intr."); 485 evcnt_attach_dynamic(&sc_ev->intr_acctdma, EVCNT_TYPE_INTR, 486 NULL, device_xname(sc->sc_dev), "AccTDMA Intr."); 487 evcnt_attach_dynamic(&sc_ev->intr_comp, EVCNT_TYPE_INTR, 488 NULL, device_xname(sc->sc_dev), "TDMA-Complete Intr."); 489 evcnt_attach_dynamic(&sc_ev->intr_own, EVCNT_TYPE_INTR, 490 NULL, device_xname(sc->sc_dev), "TDMA-Ownership Intr."); 491 evcnt_attach_dynamic(&sc_ev->intr_acctdma_cont, EVCNT_TYPE_INTR, 492 NULL, device_xname(sc->sc_dev), "AccTDMA-Continue Intr."); 493 494 evcnt_attach_dynamic(&sc_ev->session_new, EVCNT_TYPE_MISC, 495 NULL, device_xname(sc->sc_dev), "New-Session"); 496 evcnt_attach_dynamic(&sc_ev->session_free, EVCNT_TYPE_MISC, 497 NULL, device_xname(sc->sc_dev), "Free-Session"); 498 499 evcnt_attach_dynamic(&sc_ev->packet_ok, EVCNT_TYPE_MISC, 500 NULL, device_xname(sc->sc_dev), "Packet-OK"); 501 evcnt_attach_dynamic(&sc_ev->packet_err, EVCNT_TYPE_MISC, 502 NULL, device_xname(sc->sc_dev), "Packet-ERR"); 503 504 evcnt_attach_dynamic(&sc_ev->dispatch_packets, EVCNT_TYPE_MISC, 505 NULL, device_xname(sc->sc_dev), "Packet-Dispatch"); 506 evcnt_attach_dynamic(&sc_ev->dispatch_queue, EVCNT_TYPE_MISC, 507 NULL, device_xname(sc->sc_dev), "Queue-Dispatch"); 508 evcnt_attach_dynamic(&sc_ev->queue_full, EVCNT_TYPE_MISC, 509 NULL, device_xname(sc->sc_dev), "Queue-Full"); 510 evcnt_attach_dynamic(&sc_ev->max_dispatch, EVCNT_TYPE_MISC, 511 NULL, device_xname(sc->sc_dev), "Max-Dispatch"); 512 evcnt_attach_dynamic(&sc_ev->max_done, EVCNT_TYPE_MISC, 513 NULL, device_xname(sc->sc_dev), "Max-Done"); 514} 515 516/* 517 * Register setup 518 */ 519STATIC int mvxpsec_wininit(struct mvxpsec_softc *sc, enum marvell_tags *tags) 520{ 521 device_t pdev = device_parent(sc->sc_dev); 522 uint64_t base; 523 uint32_t size, reg; 524 int window, target, attr, rv, i; 525 526 /* disable all window */ 527 for (window = 0; window < MV_TDMA_NWINDOW; window++) 528 { 529 MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), 0); 530 MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), 0); 531 } 532 533 for (window = 0, i = 0; 534 tags[i] != MARVELL_TAG_UNDEFINED && window < MV_TDMA_NWINDOW; i++) { 535 rv = marvell_winparams_by_tag(pdev, tags[i], 536 &target, &attr, &base, &size); 537 if (rv != 0 || size == 0) 538 continue; 539 540 if (base > 0xffffffffULL) { 541 aprint_error_dev(sc->sc_dev, 542 "can't remap window %d\n", window); 543 continue; 544 } 545 546 reg = MV_TDMA_BAR_BASE(base); 547 MVXPSEC_WRITE(sc, MV_TDMA_BAR(window), reg); 548 549 reg = MV_TDMA_ATTR_TARGET(target); 550 reg |= MV_TDMA_ATTR_ATTR(attr); 551 reg |= MV_TDMA_ATTR_SIZE(size); 552 reg |= MV_TDMA_ATTR_ENABLE; 553 MVXPSEC_WRITE(sc, MV_TDMA_ATTR(window), reg); 554 555 window++; 556 } 557 558 return 0; 559} 560 561/* 562 * Timer handling 563 */ 564STATIC void 565mvxpsec_timer(void *aux) 566{ 567 struct mvxpsec_softc *sc = aux; 568 struct mvxpsec_packet *mv_p; 569 uint32_t reg; 570 int ndone; 571 int refill; 572 int s; 573 574 /* IPL_SOFTCLOCK */ 575 576 log(LOG_ERR, "%s: device timeout.\n", __func__); 577#ifdef MVXPSEC_DEBUG 578 mvxpsec_dump_reg(sc); 579#endif 580 581 s = splnet(); 582 /* stop security accelerator */ 583 MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_STOP); 584 585 /* stop TDMA */ 586 MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, 0); 587 588 /* cleanup packet queue */ 589 mutex_enter(&sc->sc_queue_mtx); 590 ndone = 0; 591 while ( (mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue)) != NULL) { 592 SIMPLEQ_REMOVE_HEAD(&sc->sc_run_queue, queue); 593 594 mv_p->crp->crp_etype = EINVAL; 595 mvxpsec_done_packet(mv_p); 596 ndone++; 597 } 598 MVXPSEC_EVCNT_MAX(sc, max_done, ndone); 599 sc->sc_flags &= ~HW_RUNNING; 600 refill = (sc->sc_wait_qlen > 0) ? 1 : 0; 601 mutex_exit(&sc->sc_queue_mtx); 602 603 /* reenable TDMA */ 604 if (mvxpsec_dma_wait(sc) < 0) 605 panic("%s: failed to reset DMA DEVICE. give up.", __func__); 606 MVXPSEC_WRITE(sc, MV_TDMA_CNT, 0); 607 MVXPSEC_WRITE(sc, MV_TDMA_SRC, 0); 608 MVXPSEC_WRITE(sc, MV_TDMA_DST, 0); 609 MVXPSEC_WRITE(sc, MV_TDMA_CUR, 0); 610 MVXPSEC_WRITE(sc, MV_TDMA_NXT, 0); 611 reg = MV_TDMA_DEFAULT_CONTROL; 612 reg |= MV_TDMA_CONTROL_ENABLE; 613 MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, reg); 614 615 if (mvxpsec_acc_wait(sc) < 0) 616 panic("%s: failed to reset MVXPSEC. give up.", __func__); 617 reg = MV_ACC_CONFIG_MULT_PKT; 618 reg |= MV_ACC_CONFIG_WAIT_TDMA; 619 reg |= MV_ACC_CONFIG_ACT_TDMA; 620 MVXPSEC_WRITE(sc, MV_ACC_CONFIG, reg); 621 MVXPSEC_WRITE(sc, MV_ACC_DESC, 0); 622 623 if (refill) { 624 mutex_enter(&sc->sc_queue_mtx); 625 mvxpsec_dispatch_queue(sc); 626 mutex_exit(&sc->sc_queue_mtx); 627 } 628 629 crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); 630 splx(s); 631} 632 633/* 634 * DMA handling 635 */ 636 637/* 638 * Allocate kernel devmem and DMA safe memory with bus_dma API 639 * used for DMA descriptors. 640 * 641 * if phys != 0, assume phys is a DMA safe memory and bypass 642 * allocator. 643 */ 644STATIC struct mvxpsec_devmem * 645mvxpsec_alloc_devmem(struct mvxpsec_softc *sc, paddr_t phys, int size) 646{ 647 struct mvxpsec_devmem *devmem; 648 bus_dma_segment_t seg; 649 int rseg; 650 int err; 651 652 if (sc == NULL) 653 return NULL; 654 655 devmem = kmem_alloc(sizeof(*devmem), KM_SLEEP); 656 devmem->size = size; 657 658 if (phys) { 659 seg.ds_addr = phys; 660 seg.ds_len = devmem->size; 661 rseg = 1; 662 err = 0; 663 } 664 else { 665 err = bus_dmamem_alloc(sc->sc_dmat, 666 devmem->size, PAGE_SIZE, 0, 667 &seg, MVXPSEC_DMA_MAX_SEGS, &rseg, BUS_DMA_NOWAIT); 668 } 669 if (err) { 670 aprint_error_dev(sc->sc_dev, "can't alloc DMA buffer\n"); 671 goto fail_kmem_free; 672 } 673 674 err = bus_dmamem_map(sc->sc_dmat, &seg, rseg, 675 devmem->size, &devmem->kva, BUS_DMA_NOWAIT); 676 if (err) { 677 aprint_error_dev(sc->sc_dev, "can't map DMA buffer\n"); 678 goto fail_dmamem_free; 679 } 680 681 err = bus_dmamap_create(sc->sc_dmat, 682 size, 1, size, 0, BUS_DMA_NOWAIT, &devmem->map); 683 if (err) { 684 aprint_error_dev(sc->sc_dev, "can't create DMA map\n"); 685 goto fail_unmap; 686 } 687 688 err = bus_dmamap_load(sc->sc_dmat, 689 devmem->map, devmem->kva, devmem->size, NULL, 690 BUS_DMA_NOWAIT); 691 if (err) { 692 aprint_error_dev(sc->sc_dev, 693 "can't load DMA buffer VA:%p PA:0x%08x\n", 694 devmem->kva, (int)seg.ds_addr); 695 goto fail_destroy; 696 } 697 698 return devmem; 699 700fail_destroy: 701 bus_dmamap_destroy(sc->sc_dmat, devmem->map); 702fail_unmap: 703 bus_dmamem_unmap(sc->sc_dmat, devmem->kva, devmem->size); 704fail_dmamem_free: 705 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 706fail_kmem_free: 707 kmem_free(devmem, sizeof(*devmem)); 708 709 return NULL; 710} 711 712/* 713 * Get DMA Descriptor from (DMA safe) descriptor pool. 714 */ 715INLINE struct mvxpsec_descriptor_handle * 716mvxpsec_dma_getdesc(struct mvxpsec_softc *sc) 717{ 718 struct mvxpsec_descriptor_handle *entry; 719 720 /* must called with sc->sc_dma_mtx held */ 721 KASSERT(mutex_owned(&sc->sc_dma_mtx)); 722 723 if (sc->sc_desc_ring_prod == sc->sc_desc_ring_cons) 724 return NULL; 725 726 entry = &sc->sc_desc_ring[sc->sc_desc_ring_prod]; 727 sc->sc_desc_ring_prod++; 728 if (sc->sc_desc_ring_prod >= sc->sc_desc_ring_size) 729 sc->sc_desc_ring_prod -= sc->sc_desc_ring_size; 730 731 return entry; 732} 733 734/* 735 * Put DMA Descriptor to descriptor pool. 736 */ 737_INLINE void 738mvxpsec_dma_putdesc(struct mvxpsec_softc *sc, 739 struct mvxpsec_descriptor_handle *dh) 740{ 741 /* must called with sc->sc_dma_mtx held */ 742 KASSERT(mutex_owned(&sc->sc_dma_mtx)); 743 744 sc->sc_desc_ring_cons++; 745 if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size) 746 sc->sc_desc_ring_cons -= sc->sc_desc_ring_size; 747 748 return; 749} 750 751/* 752 * Setup DMA Descriptor 753 * copy from 'src' to 'dst' by 'size' bytes. 754 * 'src' or 'dst' must be SRAM address. 755 */ 756INLINE void 757mvxpsec_dma_setup(struct mvxpsec_descriptor_handle *dh, 758 uint32_t dst, uint32_t src, uint32_t size) 759{ 760 struct mvxpsec_descriptor *desc; 761 762 desc = (struct mvxpsec_descriptor *)dh->_desc; 763 764 desc->tdma_dst = dst; 765 desc->tdma_src = src; 766 desc->tdma_word0 = size; 767 if (size != 0) 768 desc->tdma_word0 |= MV_TDMA_CNT_OWN; 769 /* size == 0 is owned by ACC, not TDMA */ 770 771#ifdef MVXPSEC_DEBUG 772 mvxpsec_dump_dmaq(dh); 773#endif 774} 775 776/* 777 * Concat 2 DMA 778 */ 779INLINE void 780mvxpsec_dma_cat(struct mvxpsec_softc *sc, 781 struct mvxpsec_descriptor_handle *dh1, 782 struct mvxpsec_descriptor_handle *dh2) 783{ 784 ((struct mvxpsec_descriptor*)dh1->_desc)->tdma_nxt = dh2->phys_addr; 785 MVXPSEC_SYNC_DESC(sc, dh1, BUS_DMASYNC_PREWRITE); 786} 787 788/* 789 * Schedule DMA Copy 790 */ 791INLINE int 792mvxpsec_dma_copy0(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r, 793 uint32_t dst, uint32_t src, uint32_t size) 794{ 795 struct mvxpsec_descriptor_handle *dh; 796 797 dh = mvxpsec_dma_getdesc(sc); 798 if (dh == NULL) { 799 log(LOG_ERR, "%s: descriptor full\n", __func__); 800 return -1; 801 } 802 803 mvxpsec_dma_setup(dh, dst, src, size); 804 if (r->dma_head == NULL) { 805 r->dma_head = dh; 806 r->dma_last = dh; 807 r->dma_size = 1; 808 } 809 else { 810 mvxpsec_dma_cat(sc, r->dma_last, dh); 811 r->dma_last = dh; 812 r->dma_size++; 813 } 814 815 return 0; 816} 817 818INLINE int 819mvxpsec_dma_copy(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r, 820 uint32_t dst, uint32_t src, uint32_t size) 821{ 822 if (size == 0) /* 0 is very special descriptor */ 823 return 0; 824 825 return mvxpsec_dma_copy0(sc, r, dst, src, size); 826} 827 828/* 829 * Schedule ACC Activate 830 */ 831INLINE int 832mvxpsec_dma_acc_activate(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r) 833{ 834 return mvxpsec_dma_copy0(sc, r, 0, 0, 0); 835} 836 837/* 838 * Finalize DMA setup 839 */ 840INLINE void 841mvxpsec_dma_finalize(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r) 842{ 843 struct mvxpsec_descriptor_handle *dh; 844 845 dh = r->dma_last; 846 ((struct mvxpsec_descriptor*)dh->_desc)->tdma_nxt = 0; 847 MVXPSEC_SYNC_DESC(sc, dh, BUS_DMASYNC_PREWRITE); 848} 849 850/* 851 * Free entire DMA ring 852 */ 853INLINE void 854mvxpsec_dma_free(struct mvxpsec_softc *sc, mvxpsec_dma_ring *r) 855{ 856 sc->sc_desc_ring_cons += r->dma_size; 857 if (sc->sc_desc_ring_cons >= sc->sc_desc_ring_size) 858 sc->sc_desc_ring_cons -= sc->sc_desc_ring_size; 859 r->dma_head = NULL; 860 r->dma_last = NULL; 861 r->dma_size = 0; 862} 863 864/* 865 * create DMA descriptor chain for the packet 866 */ 867INLINE int 868mvxpsec_dma_copy_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p) 869{ 870 struct mvxpsec_session *mv_s = mv_p->mv_s; 871 uint32_t src, dst, len; 872 uint32_t pkt_off, pkt_off_r; 873 int err; 874 int i; 875 876 /* must called with sc->sc_dma_mtx held */ 877 KASSERT(mutex_owned(&sc->sc_dma_mtx)); 878 879 /* 880 * set offset for mem->device copy 881 * 882 * typical packet image: 883 * 884 * enc_ivoff 885 * mac_off 886 * | 887 * | enc_off 888 * | | 889 * v v 890 * +----+--------... 891 * |IV |DATA 892 * +----+--------... 893 */ 894 pkt_off = 0; 895 if (mv_p->mac_off > 0) 896 pkt_off = mv_p->mac_off; 897 if ((mv_p->flags & CRP_EXT_IV) == 0 && pkt_off > mv_p->enc_ivoff) 898 pkt_off = mv_p->enc_ivoff; 899 if (mv_p->enc_off > 0 && pkt_off > mv_p->enc_off) 900 pkt_off = mv_p->enc_off; 901 pkt_off_r = pkt_off; 902 903 /* make DMA descriptors to copy packet header: DRAM -> SRAM */ 904 dst = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc); 905 src = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr; 906 len = sizeof(mv_p->pkt_header); 907 err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); 908 if (__predict_false(err)) 909 return err; 910 911 /* 912 * make DMA descriptors to copy session header: DRAM -> SRAM 913 * we can reuse session header on SRAM if session is not changed. 914 */ 915 if (sc->sc_last_session != mv_s) { 916 dst = (uint32_t)MVXPSEC_SRAM_SESS_HDR_PA(sc); 917 src = (uint32_t)mv_s->session_header_map->dm_segs[0].ds_addr; 918 len = sizeof(mv_s->session_header); 919 err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); 920 if (__predict_false(err)) 921 return err; 922 sc->sc_last_session = mv_s; 923 } 924 925 /* make DMA descriptor to copy payload data: DRAM -> SRAM */ 926 dst = MVXPSEC_SRAM_PAYLOAD_PA(sc, 0); 927 for (i = 0; i < mv_p->data_map->dm_nsegs; i++) { 928 src = mv_p->data_map->dm_segs[i].ds_addr; 929 len = mv_p->data_map->dm_segs[i].ds_len; 930 if (pkt_off) { 931 if (len <= pkt_off) { 932 /* ignore the segment */ 933 dst += len; 934 pkt_off -= len; 935 continue; 936 } 937 /* copy from the middle of the segment */ 938 dst += pkt_off; 939 src += pkt_off; 940 len -= pkt_off; 941 pkt_off = 0; 942 } 943 err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); 944 if (__predict_false(err)) 945 return err; 946 dst += len; 947 } 948 949 /* make special descriptor to activate security accelerator */ 950 err = mvxpsec_dma_acc_activate(sc, &mv_p->dma_ring); 951 if (__predict_false(err)) 952 return err; 953 954 /* make DMA descriptors to copy payload: SRAM -> DRAM */ 955 src = (uint32_t)MVXPSEC_SRAM_PAYLOAD_PA(sc, 0); 956 for (i = 0; i < mv_p->data_map->dm_nsegs; i++) { 957 dst = (uint32_t)mv_p->data_map->dm_segs[i].ds_addr; 958 len = (uint32_t)mv_p->data_map->dm_segs[i].ds_len; 959 if (pkt_off_r) { 960 if (len <= pkt_off_r) { 961 /* ignore the segment */ 962 src += len; 963 pkt_off_r -= len; 964 continue; 965 } 966 /* copy from the middle of the segment */ 967 src += pkt_off_r; 968 dst += pkt_off_r; 969 len -= pkt_off_r; 970 pkt_off_r = 0; 971 } 972 err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); 973 if (__predict_false(err)) 974 return err; 975 src += len; 976 } 977 KASSERT(pkt_off == 0); 978 KASSERT(pkt_off_r == 0); 979 980 /* 981 * make DMA descriptors to copy packet header: SRAM->DRAM 982 * if IV is present in the payload, no need to copy. 983 */ 984 if (mv_p->flags & CRP_EXT_IV) { 985 dst = (uint32_t)mv_p->pkt_header_map->dm_segs[0].ds_addr; 986 src = (uint32_t)MVXPSEC_SRAM_PKT_HDR_PA(sc); 987 len = sizeof(mv_p->pkt_header); 988 err = mvxpsec_dma_copy(sc, &mv_p->dma_ring, dst, src, len); 989 if (__predict_false(err)) 990 return err; 991 } 992 993 return 0; 994} 995 996INLINE int 997mvxpsec_dma_sync_packet(struct mvxpsec_softc *sc, struct mvxpsec_packet *mv_p) 998{ 999 /* sync packet header */ 1000 bus_dmamap_sync(sc->sc_dmat, 1001 mv_p->pkt_header_map, 0, sizeof(mv_p->pkt_header), 1002 BUS_DMASYNC_PREWRITE); 1003 1004#ifdef MVXPSEC_DEBUG 1005 /* sync session header */ 1006 if (mvxpsec_debug != 0) { 1007 struct mvxpsec_session *mv_s = mv_p->mv_s; 1008 1009 /* only debug code touch the session header after newsession */ 1010 bus_dmamap_sync(sc->sc_dmat, 1011 mv_s->session_header_map, 1012 0, sizeof(mv_s->session_header), 1013 BUS_DMASYNC_PREWRITE); 1014 } 1015#endif 1016 1017 /* sync packet buffer */ 1018 bus_dmamap_sync(sc->sc_dmat, 1019 mv_p->data_map, 0, mv_p->data_len, 1020 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1021 1022 return 0; 1023} 1024 1025/* 1026 * Initialize MVXPSEC Internal SRAM 1027 * 1028 * - must be called after DMA initialization. 1029 * - make VM mapping for SRAM area on MBus. 1030 */ 1031STATIC int 1032mvxpsec_init_sram(struct mvxpsec_softc *sc) 1033{ 1034 uint32_t tag, target, attr, base, size; 1035 vaddr_t va; 1036 int window; 1037 1038 switch (device_unit(sc->sc_dev)) { 1039 case 0: 1040 tag = ARMADAXP_TAG_CRYPT0; 1041 break; 1042 case 1: 1043 tag = ARMADAXP_TAG_CRYPT1; 1044 break; 1045 default: 1046 aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n"); 1047 return -1; 1048 } 1049 1050 window = mvsoc_target(tag, &target, &attr, &base, &size); 1051 if (window >= nwindow) { 1052 aprint_error_dev(sc->sc_dev, "no internal SRAM mapping\n"); 1053 return -1; 1054 } 1055 1056 if (sizeof(struct mvxpsec_crypt_sram) > size) { 1057 aprint_error_dev(sc->sc_dev, 1058 "SRAM Data Structure Excceeds SRAM window size.\n"); 1059 return -1; 1060 } 1061 1062 aprint_normal_dev(sc->sc_dev, 1063 "internal SRAM window at 0x%08x-0x%08x", 1064 base, base + size - 1); 1065 sc->sc_sram_pa = base; 1066 1067 /* get vmspace to read/write device internal SRAM */ 1068 va = uvm_km_alloc(kernel_map, PAGE_SIZE, PAGE_SIZE, 1069 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1070 if (va == 0) { 1071 aprint_error_dev(sc->sc_dev, "cannot map SRAM window\n"); 1072 sc->sc_sram_va = NULL; 1073 aprint_normal("\n"); 1074 return 0; 1075 } 1076 /* XXX: not working. PMAP_NOCACHE is not affected? */ 1077 pmap_kenter_pa(va, base, VM_PROT_READ|VM_PROT_WRITE, PMAP_NOCACHE); 1078 pmap_update(pmap_kernel()); 1079 sc->sc_sram_va = (void *)va; 1080 aprint_normal(" va %p\n", sc->sc_sram_va); 1081 memset(sc->sc_sram_va, 0xff, MV_ACC_SRAM_SIZE); 1082 1083 return 0; 1084} 1085 1086/* 1087 * Initialize TDMA engine. 1088 */ 1089STATIC int 1090mvxpsec_init_dma(struct mvxpsec_softc *sc, struct marvell_attach_args *mva) 1091{ 1092 struct mvxpsec_descriptor_handle *dh; 1093 uint8_t *va; 1094 paddr_t pa; 1095 off_t va_off, pa_off; 1096 int i, n, seg, ndh; 1097 1098 /* Init Deviced's control parameters (disabled yet) */ 1099 MVXPSEC_WRITE(sc, MV_TDMA_CONTROL, MV_TDMA_DEFAULT_CONTROL); 1100 1101 /* Init Software DMA Handlers */ 1102 sc->sc_devmem_desc = 1103 mvxpsec_alloc_devmem(sc, 0, PAGE_SIZE * MVXPSEC_DMA_DESC_PAGES); 1104 ndh = (PAGE_SIZE / sizeof(struct mvxpsec_descriptor)) 1105 * MVXPSEC_DMA_DESC_PAGES; 1106 sc->sc_desc_ring = 1107 kmem_alloc(sizeof(struct mvxpsec_descriptor_handle) * ndh, 1108 KM_SLEEP); 1109 aprint_normal_dev(sc->sc_dev, "%d DMA handles in %zu bytes array\n", 1110 ndh, sizeof(struct mvxpsec_descriptor_handle) * ndh); 1111 1112 ndh = 0; 1113 for (seg = 0; seg < devmem_nseg(sc->sc_devmem_desc); seg++) { 1114 va = devmem_va(sc->sc_devmem_desc); 1115 pa = devmem_pa(sc->sc_devmem_desc, seg); 1116 n = devmem_palen(sc->sc_devmem_desc, seg) / 1117 sizeof(struct mvxpsec_descriptor); 1118 va_off = (PAGE_SIZE * seg); 1119 pa_off = 0; 1120 for (i = 0; i < n; i++) { 1121 dh = &sc->sc_desc_ring[ndh]; 1122 dh->map = devmem_map(sc->sc_devmem_desc); 1123 dh->off = va_off + pa_off; 1124 dh->_desc = (void *)(va + va_off + pa_off); 1125 dh->phys_addr = pa + pa_off; 1126 pa_off += sizeof(struct mvxpsec_descriptor); 1127 ndh++; 1128 } 1129 } 1130 sc->sc_desc_ring_size = ndh; 1131 sc->sc_desc_ring_prod = 0; 1132 sc->sc_desc_ring_cons = sc->sc_desc_ring_size - 1; 1133 1134 return 0; 1135} 1136 1137/* 1138 * Wait for TDMA controller become idle 1139 */ 1140INLINE int 1141mvxpsec_dma_wait(struct mvxpsec_softc *sc) 1142{ 1143 int retry = 0; 1144 1145 while (MVXPSEC_READ(sc, MV_TDMA_CONTROL) & MV_TDMA_CONTROL_ACT) { 1146 delay(mvxpsec_wait_interval); 1147 if (retry++ >= mvxpsec_wait_retry) 1148 return -1; 1149 } 1150 return 0; 1151} 1152 1153/* 1154 * Wait for Security Accelerator become idle 1155 */ 1156INLINE int 1157mvxpsec_acc_wait(struct mvxpsec_softc *sc) 1158{ 1159 int retry = 0; 1160 1161 while (MVXPSEC_READ(sc, MV_ACC_COMMAND) & MV_ACC_COMMAND_ACT) { 1162 delay(mvxpsec_wait_interval); 1163 if (++retry >= mvxpsec_wait_retry) 1164 return -1; 1165 } 1166 return 0; 1167} 1168 1169/* 1170 * Entry of interrupt handler 1171 * 1172 * register this to kernel via marvell_intr_establish() 1173 */ 1174int 1175mvxpsec_intr(void *arg) 1176{ 1177 struct mvxpsec_softc *sc = arg; 1178 uint32_t v; 1179 1180 /* IPL_NET */ 1181 while ((v = mvxpsec_intr_ack(sc)) != 0) { 1182 mvxpsec_intr_cnt(sc, v); 1183 MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "MVXPSEC Intr 0x%08x\n", v); 1184 MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "%s\n", s_xpsecintr(v)); 1185#ifdef MVXPSEC_DEBUG 1186 mvxpsec_dump_reg(sc); 1187#endif 1188 1189 /* call high-level handlers */ 1190 if (v & MVXPSEC_INT_ACCTDMA) 1191 mvxpsec_done(sc); 1192 } 1193 1194 return 0; 1195} 1196 1197INLINE void 1198mvxpsec_intr_cleanup(struct mvxpsec_softc *sc) 1199{ 1200 struct mvxpsec_packet *mv_p; 1201 1202 /* must called with sc->sc_dma_mtx held */ 1203 KASSERT(mutex_owned(&sc->sc_dma_mtx)); 1204 1205 /* 1206 * there is only one intr for run_queue. 1207 * no one touch sc_run_queue. 1208 */ 1209 SIMPLEQ_FOREACH(mv_p, &sc->sc_run_queue, queue) 1210 mvxpsec_dma_free(sc, &mv_p->dma_ring); 1211} 1212 1213/* 1214 * Acknowledge to interrupt 1215 * 1216 * read cause bits, clear it, and return it. 1217 * NOTE: multiple cause bits may be returned at once. 1218 */ 1219STATIC uint32_t 1220mvxpsec_intr_ack(struct mvxpsec_softc *sc) 1221{ 1222 uint32_t reg; 1223 1224 reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE); 1225 reg &= MVXPSEC_DEFAULT_INT; 1226 MVXPSEC_WRITE(sc, MVXPSEC_INT_CAUSE, ~reg); 1227 MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg)); 1228 1229 return reg; 1230} 1231 1232/* 1233 * Entry of TDMA error interrupt handler 1234 * 1235 * register this to kernel via marvell_intr_establish() 1236 */ 1237int 1238mvxpsec_eintr(void *arg) 1239{ 1240 struct mvxpsec_softc *sc = arg; 1241 uint32_t err; 1242 1243 /* IPL_NET */ 1244again: 1245 err = mvxpsec_eintr_ack(sc); 1246 if (err == 0) 1247 goto done; 1248 1249 log(LOG_ERR, "%s: DMA Error Interrupt: %s\n", __func__, 1250 s_errreg(err)); 1251#ifdef MVXPSEC_DEBUG 1252 mvxpsec_dump_reg(sc); 1253#endif 1254 1255 goto again; 1256done: 1257 return 0; 1258} 1259 1260/* 1261 * Acknowledge to TDMA error interrupt 1262 * 1263 * read cause bits, clear it, and return it. 1264 * NOTE: multiple cause bits may be returned at once. 1265 */ 1266STATIC uint32_t 1267mvxpsec_eintr_ack(struct mvxpsec_softc *sc) 1268{ 1269 uint32_t reg; 1270 1271 reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE); 1272 reg &= MVXPSEC_DEFAULT_ERR; 1273 MVXPSEC_WRITE(sc, MV_TDMA_ERR_CAUSE, ~reg); 1274 MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, "Int: %s\n", s_xpsecintr(reg)); 1275 1276 return reg; 1277} 1278 1279/* 1280 * Interrupt statistics 1281 * 1282 * this is NOT a statistics of how many times the events 'occurred'. 1283 * this ONLY means how many times the events 'handled'. 1284 */ 1285INLINE void 1286mvxpsec_intr_cnt(struct mvxpsec_softc *sc, int cause) 1287{ 1288 MVXPSEC_EVCNT_INCR(sc, intr_all); 1289 if (cause & MVXPSEC_INT_AUTH) 1290 MVXPSEC_EVCNT_INCR(sc, intr_auth); 1291 if (cause & MVXPSEC_INT_DES) 1292 MVXPSEC_EVCNT_INCR(sc, intr_des); 1293 if (cause & MVXPSEC_INT_AES_ENC) 1294 MVXPSEC_EVCNT_INCR(sc, intr_aes_enc); 1295 if (cause & MVXPSEC_INT_AES_DEC) 1296 MVXPSEC_EVCNT_INCR(sc, intr_aes_dec); 1297 if (cause & MVXPSEC_INT_ENC) 1298 MVXPSEC_EVCNT_INCR(sc, intr_enc); 1299 if (cause & MVXPSEC_INT_SA) 1300 MVXPSEC_EVCNT_INCR(sc, intr_sa); 1301 if (cause & MVXPSEC_INT_ACCTDMA) 1302 MVXPSEC_EVCNT_INCR(sc, intr_acctdma); 1303 if (cause & MVXPSEC_INT_TDMA_COMP) 1304 MVXPSEC_EVCNT_INCR(sc, intr_comp); 1305 if (cause & MVXPSEC_INT_TDMA_OWN) 1306 MVXPSEC_EVCNT_INCR(sc, intr_own); 1307 if (cause & MVXPSEC_INT_ACCTDMA_CONT) 1308 MVXPSEC_EVCNT_INCR(sc, intr_acctdma_cont); 1309} 1310 1311/* 1312 * Setup MVXPSEC header structure. 1313 * 1314 * the header contains descriptor of security accelerator, 1315 * key material of ciphers, iv of ciphers and macs, ... 1316 * 1317 * the header is transferred to MVXPSEC Internal SRAM by TDMA, 1318 * and parsed by MVXPSEC H/W. 1319 */ 1320STATIC int 1321mvxpsec_header_finalize(struct mvxpsec_packet *mv_p) 1322{ 1323 struct mvxpsec_acc_descriptor *desc = &mv_p->pkt_header.desc; 1324 int enc_start, enc_len, iv_offset; 1325 int mac_start, mac_len, mac_offset; 1326 1327 /* offset -> device address */ 1328 enc_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_off); 1329 enc_len = mv_p->enc_len; 1330 if (mv_p->flags & CRP_EXT_IV) 1331 iv_offset = mv_p->enc_ivoff; 1332 else 1333 iv_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->enc_ivoff); 1334 mac_start = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_off); 1335 mac_len = mv_p->mac_len; 1336 mac_offset = MVXPSEC_SRAM_PAYLOAD_DA(mv_p->mac_dst); 1337 1338 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 1339 "PAYLOAD at 0x%08x\n", (int)MVXPSEC_SRAM_PAYLOAD_OFF); 1340 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 1341 "ENC from 0x%08x\n", enc_start); 1342 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 1343 "MAC from 0x%08x\n", mac_start); 1344 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 1345 "MAC to 0x%08x\n", mac_offset); 1346 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 1347 "ENC IV at 0x%08x\n", iv_offset); 1348 1349 /* setup device addresses in Security Accelerator Descriptors */ 1350 desc->acc_encdata = MV_ACC_DESC_ENC_DATA(enc_start, enc_start); 1351 desc->acc_enclen = MV_ACC_DESC_ENC_LEN(enc_len); 1352 if (desc->acc_config & MV_ACC_CRYPTO_DECRYPT) 1353 desc->acc_enckey = 1354 MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_D_DA); 1355 else 1356 desc->acc_enckey = 1357 MV_ACC_DESC_ENC_KEY(MVXPSEC_SRAM_KEY_DA); 1358 desc->acc_enciv = 1359 MV_ACC_DESC_ENC_IV(MVXPSEC_SRAM_IV_WORK_DA, iv_offset); 1360 1361 desc->acc_macsrc = MV_ACC_DESC_MAC_SRC(mac_start, mac_len); 1362 desc->acc_macdst = MV_ACC_DESC_MAC_DST(mac_offset, mac_len); 1363 desc->acc_maciv = 1364 MV_ACC_DESC_MAC_IV(MVXPSEC_SRAM_MIV_IN_DA, 1365 MVXPSEC_SRAM_MIV_OUT_DA); 1366 1367 return 0; 1368} 1369 1370/* 1371 * constractor of session structure. 1372 * 1373 * this constrator will be called by pool_cache framework. 1374 */ 1375STATIC int 1376mvxpsec_session_ctor(void *arg, void *obj, int flags) 1377{ 1378 struct mvxpsec_softc *sc = arg; 1379 struct mvxpsec_session *mv_s = obj; 1380 1381 /* pool is owned by softc */ 1382 mv_s->sc = sc; 1383 1384 /* Create and load DMA map for session header */ 1385 mv_s->session_header_map = 0; 1386 if (bus_dmamap_create(sc->sc_dmat, 1387 sizeof(mv_s->session_header), 1, 1388 sizeof(mv_s->session_header), 0, 1389 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 1390 &mv_s->session_header_map)) { 1391 log(LOG_ERR, "%s: cannot create DMA map\n", __func__); 1392 goto fail; 1393 } 1394 if (bus_dmamap_load(sc->sc_dmat, mv_s->session_header_map, 1395 &mv_s->session_header, sizeof(mv_s->session_header), 1396 NULL, BUS_DMA_NOWAIT)) { 1397 log(LOG_ERR, "%s: cannot load header\n", __func__); 1398 goto fail; 1399 } 1400 1401 return 0; 1402fail: 1403 if (mv_s->session_header_map) 1404 bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map); 1405 return ENOMEM; 1406} 1407 1408/* 1409 * destractor of session structure. 1410 * 1411 * this destrator will be called by pool_cache framework. 1412 */ 1413STATIC void 1414mvxpsec_session_dtor(void *arg, void *obj) 1415{ 1416 struct mvxpsec_softc *sc = arg; 1417 struct mvxpsec_session *mv_s = obj; 1418 1419 if (mv_s->sc != sc) 1420 panic("inconsitent context\n"); 1421 1422 bus_dmamap_destroy(sc->sc_dmat, mv_s->session_header_map); 1423} 1424 1425/* 1426 * constructor of packet structure. 1427 */ 1428STATIC int 1429mvxpsec_packet_ctor(void *arg, void *obj, int flags) 1430{ 1431 struct mvxpsec_softc *sc = arg; 1432 struct mvxpsec_packet *mv_p = obj; 1433 1434 mv_p->dma_ring.dma_head = NULL; 1435 mv_p->dma_ring.dma_last = NULL; 1436 mv_p->dma_ring.dma_size = 0; 1437 1438 /* Create and load DMA map for packet header */ 1439 mv_p->pkt_header_map = 0; 1440 if (bus_dmamap_create(sc->sc_dmat, 1441 sizeof(mv_p->pkt_header), 1, sizeof(mv_p->pkt_header), 0, 1442 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 1443 &mv_p->pkt_header_map)) { 1444 log(LOG_ERR, "%s: cannot create DMA map\n", __func__); 1445 goto fail; 1446 } 1447 if (bus_dmamap_load(sc->sc_dmat, mv_p->pkt_header_map, 1448 &mv_p->pkt_header, sizeof(mv_p->pkt_header), 1449 NULL, BUS_DMA_NOWAIT)) { 1450 log(LOG_ERR, "%s: cannot load header\n", __func__); 1451 goto fail; 1452 } 1453 1454 /* Create DMA map for session data. */ 1455 mv_p->data_map = 0; 1456 if (bus_dmamap_create(sc->sc_dmat, 1457 MVXPSEC_DMA_MAX_SIZE, MVXPSEC_DMA_MAX_SEGS, MVXPSEC_DMA_MAX_SIZE, 1458 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &mv_p->data_map)) { 1459 log(LOG_ERR, "%s: cannot create DMA map\n", __func__); 1460 goto fail; 1461 } 1462 1463 return 0; 1464fail: 1465 if (mv_p->pkt_header_map) 1466 bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map); 1467 if (mv_p->data_map) 1468 bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map); 1469 return ENOMEM; 1470} 1471 1472/* 1473 * destractor of packet structure. 1474 */ 1475STATIC void 1476mvxpsec_packet_dtor(void *arg, void *obj) 1477{ 1478 struct mvxpsec_softc *sc = arg; 1479 struct mvxpsec_packet *mv_p = obj; 1480 1481 mutex_enter(&sc->sc_dma_mtx); 1482 mvxpsec_dma_free(sc, &mv_p->dma_ring); 1483 mutex_exit(&sc->sc_dma_mtx); 1484 bus_dmamap_destroy(sc->sc_dmat, mv_p->pkt_header_map); 1485 bus_dmamap_destroy(sc->sc_dmat, mv_p->data_map); 1486} 1487 1488/* 1489 * allocate new session structure. 1490 */ 1491STATIC struct mvxpsec_session * 1492mvxpsec_session_alloc(struct mvxpsec_softc *sc) 1493{ 1494 struct mvxpsec_session *mv_s; 1495 1496 mv_s = pool_cache_get(sc->sc_session_pool, PR_NOWAIT); 1497 if (mv_s == NULL) { 1498 log(LOG_ERR, "%s: cannot allocate memory\n", __func__); 1499 return NULL; 1500 } 1501 mv_s->refs = 1; /* 0 means session is already invalid */ 1502 mv_s->sflags = 0; 1503 1504 return mv_s; 1505} 1506 1507/* 1508 * deallocate session structure. 1509 */ 1510STATIC void 1511mvxpsec_session_dealloc(struct mvxpsec_session *mv_s) 1512{ 1513 struct mvxpsec_softc *sc = mv_s->sc; 1514 1515 mv_s->sflags |= DELETED; 1516 mvxpsec_session_unref(mv_s); 1517 crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); 1518 1519 return; 1520} 1521 1522STATIC int 1523mvxpsec_session_ref(struct mvxpsec_session *mv_s) 1524{ 1525 uint32_t refs; 1526 1527 if (mv_s->sflags & DELETED) { 1528 log(LOG_ERR, 1529 "%s: session is already deleted.\n", __func__); 1530 return -1; 1531 } 1532 1533 refs = atomic_inc_32_nv(&mv_s->refs); 1534 if (refs == 1) { 1535 /* 1536 * a session with refs == 0 is 1537 * already invalidated. revert it. 1538 * XXX: use CAS ? 1539 */ 1540 atomic_dec_32(&mv_s->refs); 1541 log(LOG_ERR, 1542 "%s: session is already invalidated.\n", __func__); 1543 return -1; 1544 } 1545 1546 return 0; 1547} 1548 1549STATIC void 1550mvxpsec_session_unref(struct mvxpsec_session *mv_s) 1551{ 1552 uint32_t refs; 1553 1554 membar_release(); 1555 refs = atomic_dec_32_nv(&mv_s->refs); 1556 if (refs == 0) { 1557 membar_acquire(); 1558 pool_cache_put(mv_s->sc->sc_session_pool, mv_s); 1559 } 1560} 1561 1562/* 1563 * look for session is exist or not 1564 */ 1565INLINE struct mvxpsec_session * 1566mvxpsec_session_lookup(struct mvxpsec_softc *sc, int sid) 1567{ 1568 struct mvxpsec_session *mv_s; 1569 int session; 1570 1571 /* must called sc->sc_session_mtx held */ 1572 KASSERT(mutex_owned(&sc->sc_session_mtx)); 1573 1574 session = MVXPSEC_SESSION(sid); 1575 if (__predict_false(session > MVXPSEC_MAX_SESSIONS)) { 1576 log(LOG_ERR, "%s: session number too large %d\n", 1577 __func__, session); 1578 return NULL; 1579 } 1580 if (__predict_false( (mv_s = sc->sc_sessions[session]) == NULL)) { 1581 log(LOG_ERR, "%s: invalid session %d\n", 1582 __func__, session); 1583 return NULL; 1584 } 1585 1586 KASSERT(mv_s->sid == session); 1587 1588 return mv_s; 1589} 1590 1591/* 1592 * allocation new packet structure. 1593 */ 1594STATIC struct mvxpsec_packet * 1595mvxpsec_packet_alloc(struct mvxpsec_session *mv_s) 1596{ 1597 struct mvxpsec_softc *sc = mv_s->sc; 1598 struct mvxpsec_packet *mv_p; 1599 1600 /* must be called mv_queue_mtx held. */ 1601 KASSERT(mutex_owned(&sc->sc_queue_mtx)); 1602 /* must be called mv_session_mtx held. */ 1603 KASSERT(mutex_owned(&sc->sc_session_mtx)); 1604 1605 if (mvxpsec_session_ref(mv_s) < 0) { 1606 log(LOG_ERR, "%s: invalid session.\n", __func__); 1607 return NULL; 1608 } 1609 1610 if ( (mv_p = SLIST_FIRST(&sc->sc_free_list)) != NULL) { 1611 SLIST_REMOVE_HEAD(&sc->sc_free_list, free_list); 1612 sc->sc_free_qlen--; 1613 } 1614 else { 1615 mv_p = pool_cache_get(sc->sc_packet_pool, PR_NOWAIT); 1616 if (mv_p == NULL) { 1617 log(LOG_ERR, "%s: cannot allocate memory\n", 1618 __func__); 1619 mvxpsec_session_unref(mv_s); 1620 return NULL; 1621 } 1622 } 1623 mv_p->mv_s = mv_s; 1624 mv_p->flags = 0; 1625 mv_p->data_ptr = NULL; 1626 1627 return mv_p; 1628} 1629 1630/* 1631 * free packet structure. 1632 */ 1633STATIC void 1634mvxpsec_packet_dealloc(struct mvxpsec_packet *mv_p) 1635{ 1636 struct mvxpsec_session *mv_s = mv_p->mv_s; 1637 struct mvxpsec_softc *sc = mv_s->sc; 1638 1639 /* must called with sc->sc_queue_mtx held */ 1640 KASSERT(mutex_owned(&sc->sc_queue_mtx)); 1641 1642 if (mv_p->dma_ring.dma_size != 0) { 1643 sc->sc_desc_ring_cons += mv_p->dma_ring.dma_size; 1644 } 1645 mv_p->dma_ring.dma_head = NULL; 1646 mv_p->dma_ring.dma_last = NULL; 1647 mv_p->dma_ring.dma_size = 0; 1648 1649 if (mv_p->data_map) { 1650 if (mv_p->flags & RDY_DATA) { 1651 bus_dmamap_unload(sc->sc_dmat, mv_p->data_map); 1652 mv_p->flags &= ~RDY_DATA; 1653 } 1654 } 1655 1656 if (sc->sc_free_qlen > sc->sc_wait_qlimit) 1657 pool_cache_put(sc->sc_packet_pool, mv_p); 1658 else { 1659 SLIST_INSERT_HEAD(&sc->sc_free_list, mv_p, free_list); 1660 sc->sc_free_qlen++; 1661 } 1662 mvxpsec_session_unref(mv_s); 1663} 1664 1665INLINE void 1666mvxpsec_packet_enqueue(struct mvxpsec_packet *mv_p) 1667{ 1668 struct mvxpsec_softc *sc = mv_p->mv_s->sc; 1669 struct mvxpsec_packet *last_packet; 1670 struct mvxpsec_descriptor_handle *cur_dma, *prev_dma; 1671 1672 /* must called with sc->sc_queue_mtx held */ 1673 KASSERT(mutex_owned(&sc->sc_queue_mtx)); 1674 1675 if (sc->sc_wait_qlen == 0) { 1676 SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue); 1677 sc->sc_wait_qlen++; 1678 mv_p->flags |= SETUP_DONE; 1679 return; 1680 } 1681 1682 last_packet = SIMPLEQ_LAST(&sc->sc_wait_queue, mvxpsec_packet, queue); 1683 SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue); 1684 sc->sc_wait_qlen++; 1685 1686 /* chain the DMA */ 1687 cur_dma = mv_p->dma_ring.dma_head; 1688 prev_dma = last_packet->dma_ring.dma_last; 1689 mvxpsec_dma_cat(sc, prev_dma, cur_dma); 1690 mv_p->flags |= SETUP_DONE; 1691} 1692 1693/* 1694 * called by interrupt handler 1695 */ 1696STATIC int 1697mvxpsec_done_packet(struct mvxpsec_packet *mv_p) 1698{ 1699 struct mvxpsec_session *mv_s = mv_p->mv_s; 1700 struct mvxpsec_softc *sc = mv_s->sc; 1701 1702 KASSERT((mv_p->flags & RDY_DATA)); 1703 KASSERT((mv_p->flags & SETUP_DONE)); 1704 1705 /* unload data */ 1706 bus_dmamap_sync(sc->sc_dmat, mv_p->data_map, 1707 0, mv_p->data_len, 1708 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 1709 bus_dmamap_unload(sc->sc_dmat, mv_p->data_map); 1710 mv_p->flags &= ~RDY_DATA; 1711 1712#ifdef MVXPSEC_DEBUG 1713 if (mvxpsec_debug != 0) { 1714 int s; 1715 1716 bus_dmamap_sync(sc->sc_dmat, mv_p->pkt_header_map, 1717 0, sizeof(mv_p->pkt_header), 1718 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 1719 bus_dmamap_sync(sc->sc_dmat, mv_s->session_header_map, 1720 0, sizeof(mv_s->session_header), 1721 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 1722 1723 if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) { 1724 char buf[1500]; 1725 struct mbuf *m; 1726 struct uio *uio; 1727 size_t len; 1728 1729 switch (mv_p->data_type) { 1730 case MVXPSEC_DATA_MBUF: 1731 m = mv_p->data_mbuf; 1732 len = m->m_pkthdr.len; 1733 if (len > sizeof(buf)) 1734 len = sizeof(buf); 1735 m_copydata(m, 0, len, buf); 1736 break; 1737 case MVXPSEC_DATA_UIO: 1738 uio = mv_p->data_uio; 1739 len = uio->uio_resid; 1740 if (len > sizeof(buf)) 1741 len = sizeof(buf); 1742 cuio_copydata(uio, 0, len, buf); 1743 break; 1744 default: 1745 len = 0; 1746 } 1747 if (len > 0) 1748 mvxpsec_dump_data(__func__, buf, len); 1749 } 1750 1751 if (mvxpsec_debug & MVXPSEC_DEBUG_PAYLOAD) { 1752 MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD, 1753 "%s: session_descriptor:\n", __func__); 1754 mvxpsec_dump_packet_desc(__func__, mv_p); 1755 MVXPSEC_PRINTF(MVXPSEC_DEBUG_PAYLOAD, 1756 "%s: session_data:\n", __func__); 1757 mvxpsec_dump_packet_data(__func__, mv_p); 1758 } 1759 1760 if (mvxpsec_debug & MVXPSEC_DEBUG_SRAM) { 1761 MVXPSEC_PRINTF(MVXPSEC_DEBUG_SRAM, 1762 "%s: SRAM\n", __func__); 1763 mvxpsec_dump_sram(__func__, sc, 2000); 1764 } 1765 1766 s = MVXPSEC_READ(sc, MV_ACC_STATUS); 1767 if (s & MV_ACC_STATUS_MAC_ERR) { 1768 MVXPSEC_PRINTF(MVXPSEC_DEBUG_INTR, 1769 "%s: Message Authentication Failed.\n", __func__); 1770 } 1771 } 1772#endif 1773 1774 /* copy back IV */ 1775 if (mv_p->flags & CRP_EXT_IV) { 1776 memcpy(mv_p->ext_iv, 1777 &mv_p->pkt_header.crp_iv_ext, mv_p->ext_ivlen); 1778 mv_p->ext_iv = NULL; 1779 mv_p->ext_ivlen = 0; 1780 } 1781 1782 /* notify opencrypto */ 1783 mv_p->crp->crp_etype = 0; 1784 crypto_done(mv_p->crp); 1785 mv_p->crp = NULL; 1786 1787 /* unblock driver */ 1788 mvxpsec_packet_dealloc(mv_p); 1789 crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); 1790 1791 MVXPSEC_EVCNT_INCR(sc, packet_ok); 1792 1793 return 0; 1794} 1795 1796 1797/* 1798 * Opencrypto API registration 1799 */ 1800int 1801mvxpsec_register(struct mvxpsec_softc *sc) 1802{ 1803 int oplen = SRAM_PAYLOAD_SIZE; 1804 int flags = 0; 1805 int err; 1806 1807 sc->sc_nsessions = 0; 1808 sc->sc_cid = crypto_get_driverid(0); 1809 if (sc->sc_cid < 0) { 1810 log(LOG_ERR, 1811 "%s: crypto_get_driverid() failed.\n", __func__); 1812 err = EINVAL; 1813 goto done; 1814 } 1815 1816 /* Ciphers */ 1817 err = crypto_register(sc->sc_cid, CRYPTO_DES_CBC, oplen, flags, 1818 mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); 1819 if (err) 1820 goto done; 1821 1822 err = crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, oplen, flags, 1823 mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); 1824 if (err) 1825 goto done; 1826 1827 err = crypto_register(sc->sc_cid, CRYPTO_AES_CBC, oplen, flags, 1828 mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); 1829 if (err) 1830 goto done; 1831 1832 /* MACs */ 1833 err = crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 1834 oplen, flags, 1835 mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); 1836 if (err) 1837 goto done; 1838 1839 err = crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 1840 oplen, flags, 1841 mvxpsec_newsession, mvxpsec_freesession, mvxpsec_dispatch, sc); 1842 if (err) 1843 goto done; 1844 1845#ifdef DEBUG 1846 log(LOG_DEBUG, 1847 "%s: registered to opencrypto(max data = %d bytes)\n", 1848 device_xname(sc->sc_dev), oplen); 1849#endif 1850 1851 err = 0; 1852done: 1853 return err; 1854} 1855 1856/* 1857 * Create new opencrypto session 1858 * 1859 * - register cipher key, mac key. 1860 * - initialize mac internal state. 1861 */ 1862int 1863mvxpsec_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri) 1864{ 1865 struct mvxpsec_softc *sc = arg; 1866 struct mvxpsec_session *mv_s = NULL; 1867 struct cryptoini *c; 1868 static int hint = 0; 1869 int session = -1; 1870 int sid; 1871 int err; 1872 int i; 1873 1874 /* allocate driver session context */ 1875 mv_s = mvxpsec_session_alloc(sc); 1876 if (mv_s == NULL) 1877 return ENOMEM; 1878 1879 /* 1880 * lookup opencrypto session table 1881 * 1882 * we have sc_session_mtx after here. 1883 */ 1884 mutex_enter(&sc->sc_session_mtx); 1885 if (sc->sc_nsessions >= MVXPSEC_MAX_SESSIONS) { 1886 mutex_exit(&sc->sc_session_mtx); 1887 log(LOG_ERR, "%s: too many IPsec SA(max %d)\n", 1888 __func__, MVXPSEC_MAX_SESSIONS); 1889 mvxpsec_session_dealloc(mv_s); 1890 return ENOMEM; 1891 } 1892 for (i = hint; i < MVXPSEC_MAX_SESSIONS; i++) { 1893 if (sc->sc_sessions[i]) 1894 continue; 1895 session = i; 1896 hint = session + 1; 1897 break; 1898 } 1899 if (session < 0) { 1900 for (i = 0; i < hint; i++) { 1901 if (sc->sc_sessions[i]) 1902 continue; 1903 session = i; 1904 hint = session + 1; 1905 break; 1906 } 1907 if (session < 0) { 1908 mutex_exit(&sc->sc_session_mtx); 1909 /* session full */ 1910 log(LOG_ERR, "%s: too many IPsec SA(max %d)\n", 1911 __func__, MVXPSEC_MAX_SESSIONS); 1912 mvxpsec_session_dealloc(mv_s); 1913 hint = 0; 1914 return ENOMEM; 1915 } 1916 } 1917 if (hint >= MVXPSEC_MAX_SESSIONS) 1918 hint = 0; 1919 sc->sc_nsessions++; 1920 sc->sc_sessions[session] = mv_s; 1921#ifdef DEBUG 1922 log(LOG_DEBUG, "%s: new session %d allocated\n", __func__, session); 1923#endif 1924 1925 sid = MVXPSEC_SID(device_unit(sc->sc_dev), session); 1926 mv_s->sid = sid; 1927 1928 /* setup the session key ... */ 1929 for (c = cri; c; c = c->cri_next) { 1930 switch (c->cri_alg) { 1931 case CRYPTO_DES_CBC: 1932 case CRYPTO_3DES_CBC: 1933 case CRYPTO_AES_CBC: 1934 /* key */ 1935 if (mvxpsec_key_precomp(c->cri_alg, 1936 c->cri_key, c->cri_klen, 1937 &mv_s->session_header.crp_key, 1938 &mv_s->session_header.crp_key_d)) { 1939 log(LOG_ERR, 1940 "%s: Invalid HMAC key for %s.\n", 1941 __func__, s_ctlalg(c->cri_alg)); 1942 err = EINVAL; 1943 goto fail; 1944 } 1945 if (mv_s->sflags & RDY_CRP_KEY) { 1946 log(LOG_WARNING, 1947 "%s: overwrite cipher: %s->%s.\n", 1948 __func__, 1949 s_ctlalg(mv_s->cipher_alg), 1950 s_ctlalg(c->cri_alg)); 1951 } 1952 mv_s->sflags |= RDY_CRP_KEY; 1953 mv_s->enc_klen = c->cri_klen; 1954 mv_s->cipher_alg = c->cri_alg; 1955 /* create per session IV (compatible with KAME IPsec) */ 1956 cprng_fast(&mv_s->session_iv, sizeof(mv_s->session_iv)); 1957 mv_s->sflags |= RDY_CRP_IV; 1958 break; 1959 case CRYPTO_SHA1_HMAC_96: 1960 case CRYPTO_MD5_HMAC_96: 1961 /* key */ 1962 if (mvxpsec_hmac_precomp(c->cri_alg, 1963 c->cri_key, c->cri_klen, 1964 (uint32_t *)&mv_s->session_header.miv_in, 1965 (uint32_t *)&mv_s->session_header.miv_out)) { 1966 log(LOG_ERR, 1967 "%s: Invalid MAC key\n", __func__); 1968 err = EINVAL; 1969 goto fail; 1970 } 1971 if (mv_s->sflags & RDY_MAC_KEY || 1972 mv_s->sflags & RDY_MAC_IV) { 1973 log(LOG_ERR, 1974 "%s: overwrite HMAC: %s->%s.\n", 1975 __func__, s_ctlalg(mv_s->hmac_alg), 1976 s_ctlalg(c->cri_alg)); 1977 } 1978 mv_s->sflags |= RDY_MAC_KEY; 1979 mv_s->sflags |= RDY_MAC_IV; 1980 1981 mv_s->mac_klen = c->cri_klen; 1982 mv_s->hmac_alg = c->cri_alg; 1983 break; 1984 default: 1985 log(LOG_ERR, "%s: Unknown algorithm %d\n", 1986 __func__, c->cri_alg); 1987 err = EINVAL; 1988 goto fail; 1989 } 1990 } 1991 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 1992 "H/W Crypto session (id:%u) added.\n", session); 1993 1994 *sidp = sid; 1995 MVXPSEC_EVCNT_INCR(sc, session_new); 1996 mutex_exit(&sc->sc_session_mtx); 1997 1998 /* sync session header(it's never touched after here) */ 1999 bus_dmamap_sync(sc->sc_dmat, 2000 mv_s->session_header_map, 2001 0, sizeof(mv_s->session_header), 2002 BUS_DMASYNC_PREWRITE); 2003 2004 return 0; 2005 2006fail: 2007 sc->sc_nsessions--; 2008 sc->sc_sessions[session] = NULL; 2009 hint = session; 2010 if (mv_s) 2011 mvxpsec_session_dealloc(mv_s); 2012 log(LOG_WARNING, 2013 "%s: Failed to add H/W crypto session (id:%u): err=%d\n", 2014 __func__, session, err); 2015 2016 mutex_exit(&sc->sc_session_mtx); 2017 return err; 2018} 2019 2020/* 2021 * remove opencrypto session 2022 */ 2023void 2024mvxpsec_freesession(void *arg, uint64_t tid) 2025{ 2026 struct mvxpsec_softc *sc = arg; 2027 struct mvxpsec_session *mv_s; 2028 int session; 2029 uint32_t sid = ((uint32_t)tid) & 0xffffffff; 2030 2031 session = MVXPSEC_SESSION(sid); 2032 KASSERTMSG(session >= 0, "session=%d", session); 2033 KASSERTMSG(session < MVXPSEC_MAX_SESSIONS, "session=%d max=%d", 2034 session, MVXPSEC_MAX_SESSIONS); 2035 2036 mutex_enter(&sc->sc_session_mtx); 2037 mv_s = sc->sc_sessions[session]; 2038 KASSERT(mv_s != NULL); 2039 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2040 "%s: inactivate session %d\n", __func__, session); 2041 2042 /* inactivate mvxpsec session */ 2043 sc->sc_sessions[session] = NULL; 2044 sc->sc_nsessions--; 2045 sc->sc_last_session = NULL; 2046 mutex_exit(&sc->sc_session_mtx); 2047 2048 KASSERT(sc->sc_nsessions >= 0); 2049 KASSERT(mv_s->sid == sid); 2050 2051 mvxpsec_session_dealloc(mv_s); 2052 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2053 "H/W Crypto session (id: %d) deleted.\n", session); 2054 2055 /* force unblock opencrypto */ 2056 crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); 2057 2058 MVXPSEC_EVCNT_INCR(sc, session_free); 2059} 2060 2061/* 2062 * process data with existing session 2063 */ 2064int 2065mvxpsec_dispatch(void *arg, struct cryptop *crp, int hint) 2066{ 2067 struct mvxpsec_softc *sc = arg; 2068 struct mvxpsec_session *mv_s; 2069 struct mvxpsec_packet *mv_p; 2070 int q_full; 2071 int running; 2072 int err; 2073 2074 mutex_enter(&sc->sc_queue_mtx); 2075 2076 /* 2077 * lookup session 2078 */ 2079 mutex_enter(&sc->sc_session_mtx); 2080 mv_s = mvxpsec_session_lookup(sc, crp->crp_sid); 2081 if (__predict_false(mv_s == NULL)) { 2082 err = EINVAL; 2083 mv_p = NULL; 2084 mutex_exit(&sc->sc_session_mtx); 2085 goto fail; 2086 } 2087 mv_p = mvxpsec_packet_alloc(mv_s); 2088 if (__predict_false(mv_p == NULL)) { 2089 mutex_exit(&sc->sc_session_mtx); 2090 mutex_exit(&sc->sc_queue_mtx); 2091 return ERESTART; /* => queued in opencrypto layer */ 2092 } 2093 mutex_exit(&sc->sc_session_mtx); 2094 2095 /* 2096 * check queue status 2097 */ 2098#ifdef MVXPSEC_MULTI_PACKET 2099 q_full = (sc->sc_wait_qlen >= sc->sc_wait_qlimit) ? 1 : 0; 2100#else 2101 q_full = (sc->sc_wait_qlen != 0) ? 1 : 0; 2102#endif 2103 running = (sc->sc_flags & HW_RUNNING) ? 1: 0; 2104 if (q_full) { 2105 /* input queue is full. */ 2106 if (!running && sc->sc_wait_qlen > 0) 2107 mvxpsec_dispatch_queue(sc); 2108 MVXPSEC_EVCNT_INCR(sc, queue_full); 2109 mvxpsec_packet_dealloc(mv_p); 2110 mutex_exit(&sc->sc_queue_mtx); 2111 return ERESTART; /* => queued in opencrypto layer */ 2112 } 2113 2114 /* 2115 * Load and setup packet data 2116 */ 2117 err = mvxpsec_packet_setcrp(mv_p, crp); 2118 if (__predict_false(err)) 2119 goto fail; 2120 2121 /* 2122 * Setup DMA descriptor chains 2123 */ 2124 mutex_enter(&sc->sc_dma_mtx); 2125 err = mvxpsec_dma_copy_packet(sc, mv_p); 2126 mutex_exit(&sc->sc_dma_mtx); 2127 if (__predict_false(err)) 2128 goto fail; 2129 2130#ifdef MVXPSEC_DEBUG 2131 mvxpsec_dump_packet(__func__, mv_p); 2132#endif 2133 2134 /* 2135 * Sync/inval the data cache 2136 */ 2137 err = mvxpsec_dma_sync_packet(sc, mv_p); 2138 if (__predict_false(err)) 2139 goto fail; 2140 2141 /* 2142 * Enqueue the packet 2143 */ 2144 MVXPSEC_EVCNT_INCR(sc, dispatch_packets); 2145#ifdef MVXPSEC_MULTI_PACKET 2146 mvxpsec_packet_enqueue(mv_p); 2147 if (!running) 2148 mvxpsec_dispatch_queue(sc); 2149#else 2150 SIMPLEQ_INSERT_TAIL(&sc->sc_wait_queue, mv_p, queue); 2151 sc->sc_wait_qlen++; 2152 mv_p->flags |= SETUP_DONE; 2153 if (!running) 2154 mvxpsec_dispatch_queue(sc); 2155#endif 2156 mutex_exit(&sc->sc_queue_mtx); 2157 return 0; 2158 2159fail: 2160 /* Drop the incoming packet */ 2161 mvxpsec_drop(sc, crp, mv_p, err); 2162 mutex_exit(&sc->sc_queue_mtx); 2163 return 0; 2164} 2165 2166/* 2167 * back the packet to the IP stack 2168 */ 2169void 2170mvxpsec_done(void *arg) 2171{ 2172 struct mvxpsec_softc *sc = arg; 2173 struct mvxpsec_packet *mv_p; 2174 mvxpsec_queue_t ret_queue; 2175 int ndone; 2176 2177 mutex_enter(&sc->sc_queue_mtx); 2178 2179 /* stop wdog timer */ 2180 callout_stop(&sc->sc_timeout); 2181 2182 /* refill MVXPSEC */ 2183 ret_queue = sc->sc_run_queue; 2184 SIMPLEQ_INIT(&sc->sc_run_queue); 2185 sc->sc_flags &= ~HW_RUNNING; 2186 if (sc->sc_wait_qlen > 0) 2187 mvxpsec_dispatch_queue(sc); 2188 2189 ndone = 0; 2190 while ( (mv_p = SIMPLEQ_FIRST(&ret_queue)) != NULL) { 2191 SIMPLEQ_REMOVE_HEAD(&ret_queue, queue); 2192 mvxpsec_dma_free(sc, &mv_p->dma_ring); 2193 mvxpsec_done_packet(mv_p); 2194 ndone++; 2195 } 2196 MVXPSEC_EVCNT_MAX(sc, max_done, ndone); 2197 2198 mutex_exit(&sc->sc_queue_mtx); 2199} 2200 2201/* 2202 * drop the packet 2203 */ 2204INLINE void 2205mvxpsec_drop(struct mvxpsec_softc *sc, struct cryptop *crp, 2206 struct mvxpsec_packet *mv_p, int err) 2207{ 2208 /* must called with sc->sc_queue_mtx held */ 2209 KASSERT(mutex_owned(&sc->sc_queue_mtx)); 2210 2211 if (mv_p) 2212 mvxpsec_packet_dealloc(mv_p); 2213 if (err < 0) 2214 err = EINVAL; 2215 crp->crp_etype = err; 2216 crypto_done(crp); 2217 MVXPSEC_EVCNT_INCR(sc, packet_err); 2218 2219 /* dispatch other packets in queue */ 2220 if (sc->sc_wait_qlen > 0 && 2221 !(sc->sc_flags & HW_RUNNING)) 2222 mvxpsec_dispatch_queue(sc); 2223 2224 /* unblock driver for dropped packet */ 2225 crypto_unblock(sc->sc_cid, CRYPTO_SYMQ|CRYPTO_ASYMQ); 2226} 2227 2228/* move wait queue entry to run queue */ 2229STATIC int 2230mvxpsec_dispatch_queue(struct mvxpsec_softc *sc) 2231{ 2232 struct mvxpsec_packet *mv_p; 2233 paddr_t head; 2234 int ndispatch = 0; 2235 2236 /* must called with sc->sc_queue_mtx held */ 2237 KASSERT(mutex_owned(&sc->sc_queue_mtx)); 2238 2239 /* check there is any task */ 2240 if (__predict_false(sc->sc_flags & HW_RUNNING)) { 2241 log(LOG_WARNING, 2242 "%s: another packet already exist.\n", __func__); 2243 return 0; 2244 } 2245 if (__predict_false(SIMPLEQ_EMPTY(&sc->sc_wait_queue))) { 2246 log(LOG_WARNING, 2247 "%s: no waiting packet yet(qlen=%d).\n", 2248 __func__, sc->sc_wait_qlen); 2249 return 0; 2250 } 2251 2252 /* move queue */ 2253 sc->sc_run_queue = sc->sc_wait_queue; 2254 sc->sc_flags |= HW_RUNNING; /* dropped by intr or timeout */ 2255 SIMPLEQ_INIT(&sc->sc_wait_queue); 2256 ndispatch = sc->sc_wait_qlen; 2257 sc->sc_wait_qlen = 0; 2258 2259 /* get 1st DMA descriptor */ 2260 mv_p = SIMPLEQ_FIRST(&sc->sc_run_queue); 2261 head = mv_p->dma_ring.dma_head->phys_addr; 2262 2263 /* terminate last DMA descriptor */ 2264 mv_p = SIMPLEQ_LAST(&sc->sc_run_queue, mvxpsec_packet, queue); 2265 mvxpsec_dma_finalize(sc, &mv_p->dma_ring); 2266 2267 /* configure TDMA */ 2268 if (mvxpsec_dma_wait(sc) < 0) { 2269 log(LOG_ERR, "%s: DMA DEVICE not responding", __func__); 2270 callout_schedule(&sc->sc_timeout, hz); 2271 return 0; 2272 } 2273 MVXPSEC_WRITE(sc, MV_TDMA_NXT, head); 2274 2275 /* trigger ACC */ 2276 if (mvxpsec_acc_wait(sc) < 0) { 2277 log(LOG_ERR, "%s: MVXPSEC not responding", __func__); 2278 callout_schedule(&sc->sc_timeout, hz); 2279 return 0; 2280 } 2281 MVXPSEC_WRITE(sc, MV_ACC_COMMAND, MV_ACC_COMMAND_ACT); 2282 2283 MVXPSEC_EVCNT_MAX(sc, max_dispatch, ndispatch); 2284 MVXPSEC_EVCNT_INCR(sc, dispatch_queue); 2285 callout_schedule(&sc->sc_timeout, hz); 2286 return 0; 2287} 2288 2289/* 2290 * process opencrypto operations(cryptop) for packets. 2291 */ 2292INLINE int 2293mvxpsec_parse_crd(struct mvxpsec_packet *mv_p, struct cryptodesc *crd) 2294{ 2295 int ivlen; 2296 2297 KASSERT(mv_p->flags & RDY_DATA); 2298 2299 /* MAC & Ciphers: set data location and operation */ 2300 switch (crd->crd_alg) { 2301 case CRYPTO_SHA1_HMAC_96: 2302 mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96; 2303 /* fall through */ 2304 case CRYPTO_SHA1_HMAC: 2305 mv_p->mac_dst = crd->crd_inject; 2306 mv_p->mac_off = crd->crd_skip; 2307 mv_p->mac_len = crd->crd_len; 2308 MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config, 2309 MV_ACC_CRYPTO_MAC_HMAC_SHA1); 2310 mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC); 2311 /* No more setup for MAC */ 2312 return 0; 2313 case CRYPTO_MD5_HMAC_96: 2314 mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_MAC_96; 2315 /* fall through */ 2316 case CRYPTO_MD5_HMAC: 2317 mv_p->mac_dst = crd->crd_inject; 2318 mv_p->mac_off = crd->crd_skip; 2319 mv_p->mac_len = crd->crd_len; 2320 MV_ACC_CRYPTO_MAC_SET(mv_p->pkt_header.desc.acc_config, 2321 MV_ACC_CRYPTO_MAC_HMAC_MD5); 2322 mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_MAC); 2323 /* No more setup for MAC */ 2324 return 0; 2325 case CRYPTO_DES_CBC: 2326 mv_p->enc_ivoff = crd->crd_inject; 2327 mv_p->enc_off = crd->crd_skip; 2328 mv_p->enc_len = crd->crd_len; 2329 ivlen = 8; 2330 MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config, 2331 MV_ACC_CRYPTO_ENC_DES); 2332 mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC; 2333 mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC); 2334 break; 2335 case CRYPTO_3DES_CBC: 2336 mv_p->enc_ivoff = crd->crd_inject; 2337 mv_p->enc_off = crd->crd_skip; 2338 mv_p->enc_len = crd->crd_len; 2339 ivlen = 8; 2340 MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config, 2341 MV_ACC_CRYPTO_ENC_3DES); 2342 mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC; 2343 mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_3DES_EDE; 2344 mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC); 2345 break; 2346 case CRYPTO_AES_CBC: 2347 mv_p->enc_ivoff = crd->crd_inject; 2348 mv_p->enc_off = crd->crd_skip; 2349 mv_p->enc_len = crd->crd_len; 2350 ivlen = 16; 2351 MV_ACC_CRYPTO_ENC_SET(mv_p->pkt_header.desc.acc_config, 2352 MV_ACC_CRYPTO_ENC_AES); 2353 MV_ACC_CRYPTO_AES_KLEN_SET( 2354 mv_p->pkt_header.desc.acc_config, 2355 mvxpsec_aesklen(mv_p->mv_s->enc_klen)); 2356 mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_CBC; 2357 mvxpsec_packet_update_op_order(mv_p, MV_ACC_CRYPTO_OP_ENC); 2358 break; 2359 default: 2360 log(LOG_ERR, "%s: Unknown algorithm %d\n", 2361 __func__, crd->crd_alg); 2362 return EINVAL; 2363 } 2364 2365 /* Operations only for Cipher, not MAC */ 2366 if (crd->crd_flags & CRD_F_ENCRYPT) { 2367 /* Ciphers: Originate IV for Encryption.*/ 2368 mv_p->pkt_header.desc.acc_config &= ~MV_ACC_CRYPTO_DECRYPT; 2369 mv_p->flags |= DIR_ENCRYPT; 2370 2371 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2372 MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "EXPLICIT IV\n"); 2373 mv_p->flags |= CRP_EXT_IV; 2374 mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen); 2375 mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF; 2376 } 2377 else if (crd->crd_flags & CRD_F_IV_PRESENT) { 2378 MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "IV is present\n"); 2379 mvxpsec_packet_copy_iv(mv_p, crd->crd_inject, ivlen); 2380 } 2381 else { 2382 MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, "Create New IV\n"); 2383 mvxpsec_packet_write_iv(mv_p, NULL, ivlen); 2384 } 2385 } 2386 else { 2387 /* Ciphers: IV is loadded from crd_inject when it's present */ 2388 mv_p->pkt_header.desc.acc_config |= MV_ACC_CRYPTO_DECRYPT; 2389 mv_p->flags |= DIR_DECRYPT; 2390 2391 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2392#ifdef MVXPSEC_DEBUG 2393 if (mvxpsec_debug & MVXPSEC_DEBUG_ENC_IV) { 2394 MVXPSEC_PRINTF(MVXPSEC_DEBUG_ENC_IV, 2395 "EXPLICIT IV(Decrypt)\n"); 2396 mvxpsec_dump_data(__func__, crd->crd_iv, ivlen); 2397 } 2398#endif 2399 mv_p->flags |= CRP_EXT_IV; 2400 mvxpsec_packet_write_iv(mv_p, crd->crd_iv, ivlen); 2401 mv_p->enc_ivoff = MVXPSEC_SRAM_IV_EXT_OFF; 2402 } 2403 } 2404 2405 KASSERT(!((mv_p->flags & DIR_ENCRYPT) && (mv_p->flags & DIR_DECRYPT))); 2406 2407 return 0; 2408} 2409 2410INLINE int 2411mvxpsec_parse_crp(struct mvxpsec_packet *mv_p) 2412{ 2413 struct cryptop *crp = mv_p->crp; 2414 struct cryptodesc *crd; 2415 int err; 2416 2417 KASSERT(crp); 2418 2419 mvxpsec_packet_reset_op(mv_p); 2420 2421 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2422 err = mvxpsec_parse_crd(mv_p, crd); 2423 if (err) 2424 return err; 2425 } 2426 2427 return 0; 2428} 2429 2430INLINE int 2431mvxpsec_packet_setcrp(struct mvxpsec_packet *mv_p, struct cryptop *crp) 2432{ 2433 int err = EINVAL; 2434 2435 /* register crp to the MVXPSEC packet */ 2436 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2437 err = mvxpsec_packet_setmbuf(mv_p, 2438 (struct mbuf *)crp->crp_buf); 2439 mv_p->crp = crp; 2440 } 2441 else if (crp->crp_flags & CRYPTO_F_IOV) { 2442 err = mvxpsec_packet_setuio(mv_p, 2443 (struct uio *)crp->crp_buf); 2444 mv_p->crp = crp; 2445 } 2446 else { 2447 err = mvxpsec_packet_setdata(mv_p, 2448 (struct mbuf *)crp->crp_buf, crp->crp_ilen); 2449 mv_p->crp = crp; 2450 } 2451 if (__predict_false(err)) 2452 return err; 2453 2454 /* parse crp and setup MVXPSEC registers/descriptors */ 2455 err = mvxpsec_parse_crp(mv_p); 2456 if (__predict_false(err)) 2457 return err; 2458 2459 /* fixup data offset to fit MVXPSEC internal SRAM */ 2460 err = mvxpsec_header_finalize(mv_p); 2461 if (__predict_false(err)) 2462 return err; 2463 2464 return 0; 2465} 2466 2467/* 2468 * load data for encrypt/decrypt/authentication 2469 * 2470 * data is raw kernel memory area. 2471 */ 2472STATIC int 2473mvxpsec_packet_setdata(struct mvxpsec_packet *mv_p, 2474 void *data, uint32_t data_len) 2475{ 2476 struct mvxpsec_session *mv_s = mv_p->mv_s; 2477 struct mvxpsec_softc *sc = mv_s->sc; 2478 2479 if (bus_dmamap_load(sc->sc_dmat, mv_p->data_map, data, data_len, 2480 NULL, BUS_DMA_NOWAIT)) { 2481 log(LOG_ERR, "%s: cannot load data\n", __func__); 2482 return -1; 2483 } 2484 mv_p->data_type = MVXPSEC_DATA_RAW; 2485 mv_p->data_raw = data; 2486 mv_p->data_len = data_len; 2487 mv_p->flags |= RDY_DATA; 2488 2489 return 0; 2490} 2491 2492/* 2493 * load data for encrypt/decrypt/authentication 2494 * 2495 * data is mbuf based network data. 2496 */ 2497STATIC int 2498mvxpsec_packet_setmbuf(struct mvxpsec_packet *mv_p, struct mbuf *m) 2499{ 2500 struct mvxpsec_session *mv_s = mv_p->mv_s; 2501 struct mvxpsec_softc *sc = mv_s->sc; 2502 size_t pktlen = 0; 2503 2504 if (__predict_true(m->m_flags & M_PKTHDR)) 2505 pktlen = m->m_pkthdr.len; 2506 else { 2507 struct mbuf *mp = m; 2508 2509 while (mp != NULL) { 2510 pktlen += m->m_len; 2511 mp = mp->m_next; 2512 } 2513 } 2514 if (pktlen > SRAM_PAYLOAD_SIZE) { 2515#if NIPSEC > 0 2516 extern percpu_t *espstat_percpu; 2517 /* XXX: 2518 * layer violation. opencrypto knows our max packet size 2519 * from crypto_register(9) API. 2520 */ 2521 2522 _NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG); 2523#endif 2524 log(LOG_ERR, 2525 "%s: ESP Packet too large: %zu [oct.] > %zu [oct.]\n", 2526 device_xname(sc->sc_dev), 2527 (size_t)pktlen, SRAM_PAYLOAD_SIZE); 2528 mv_p->data_type = MVXPSEC_DATA_NONE; 2529 mv_p->data_mbuf = NULL; 2530 return -1; 2531 } 2532 2533 if (bus_dmamap_load_mbuf(sc->sc_dmat, mv_p->data_map, m, 2534 BUS_DMA_NOWAIT)) { 2535 mv_p->data_type = MVXPSEC_DATA_NONE; 2536 mv_p->data_mbuf = NULL; 2537 log(LOG_ERR, "%s: cannot load mbuf\n", __func__); 2538 return -1; 2539 } 2540 2541 /* set payload buffer */ 2542 mv_p->data_type = MVXPSEC_DATA_MBUF; 2543 mv_p->data_mbuf = m; 2544 if (m->m_flags & M_PKTHDR) { 2545 mv_p->data_len = m->m_pkthdr.len; 2546 } 2547 else { 2548 mv_p->data_len = 0; 2549 while (m) { 2550 mv_p->data_len += m->m_len; 2551 m = m->m_next; 2552 } 2553 } 2554 mv_p->flags |= RDY_DATA; 2555 2556 return 0; 2557} 2558 2559STATIC int 2560mvxpsec_packet_setuio(struct mvxpsec_packet *mv_p, struct uio *uio) 2561{ 2562 struct mvxpsec_session *mv_s = mv_p->mv_s; 2563 struct mvxpsec_softc *sc = mv_s->sc; 2564 2565 if (uio->uio_resid > SRAM_PAYLOAD_SIZE) { 2566#if NIPSEC > 0 2567 extern percpu_t *espstat_percpu; 2568 /* XXX: 2569 * layer violation. opencrypto knows our max packet size 2570 * from crypto_register(9) API. 2571 */ 2572 2573 _NET_STATINC(espstat_percpu, ESP_STAT_TOOBIG); 2574#endif 2575 log(LOG_ERR, 2576 "%s: uio request too large: %zu [oct.] > %zu [oct.]\n", 2577 device_xname(sc->sc_dev), 2578 uio->uio_resid, SRAM_PAYLOAD_SIZE); 2579 mv_p->data_type = MVXPSEC_DATA_NONE; 2580 mv_p->data_mbuf = NULL; 2581 return -1; 2582 } 2583 2584 if (bus_dmamap_load_uio(sc->sc_dmat, mv_p->data_map, uio, 2585 BUS_DMA_NOWAIT)) { 2586 mv_p->data_type = MVXPSEC_DATA_NONE; 2587 mv_p->data_mbuf = NULL; 2588 log(LOG_ERR, "%s: cannot load uio buf\n", __func__); 2589 return -1; 2590 } 2591 2592 /* set payload buffer */ 2593 mv_p->data_type = MVXPSEC_DATA_UIO; 2594 mv_p->data_uio = uio; 2595 mv_p->data_len = uio->uio_resid; 2596 mv_p->flags |= RDY_DATA; 2597 2598 return 0; 2599} 2600 2601STATIC int 2602mvxpsec_packet_rdata(struct mvxpsec_packet *mv_p, 2603 int off, int len, void *cp) 2604{ 2605 uint8_t *p; 2606 2607 if (mv_p->data_type == MVXPSEC_DATA_RAW) { 2608 p = (uint8_t *)mv_p->data_raw + off; 2609 memcpy(cp, p, len); 2610 } 2611 else if (mv_p->data_type == MVXPSEC_DATA_MBUF) { 2612 m_copydata(mv_p->data_mbuf, off, len, cp); 2613 } 2614 else if (mv_p->data_type == MVXPSEC_DATA_UIO) { 2615 cuio_copydata(mv_p->data_uio, off, len, cp); 2616 } 2617 else 2618 return -1; 2619 2620 return 0; 2621} 2622 2623STATIC int 2624mvxpsec_packet_wdata(struct mvxpsec_packet *mv_p, 2625 int off, int len, void *cp) 2626{ 2627 uint8_t *p; 2628 2629 if (mv_p->data_type == MVXPSEC_DATA_RAW) { 2630 p = (uint8_t *)mv_p->data_raw + off; 2631 memcpy(p, cp, len); 2632 } 2633 else if (mv_p->data_type == MVXPSEC_DATA_MBUF) { 2634 m_copyback(mv_p->data_mbuf, off, len, cp); 2635 } 2636 else if (mv_p->data_type == MVXPSEC_DATA_UIO) { 2637 cuio_copyback(mv_p->data_uio, off, len, cp); 2638 } 2639 else 2640 return -1; 2641 2642 return 0; 2643} 2644 2645/* 2646 * Set initial vector of cipher to the session. 2647 */ 2648STATIC int 2649mvxpsec_packet_write_iv(struct mvxpsec_packet *mv_p, void *iv, int ivlen) 2650{ 2651 uint8_t ivbuf[16]; 2652 2653 KASSERT(ivlen == 8 || ivlen == 16); 2654 2655 if (iv == NULL) { 2656 if (mv_p->mv_s->sflags & RDY_CRP_IV) { 2657 /* use per session IV (compatible with KAME IPsec) */ 2658 mv_p->pkt_header.crp_iv_work = mv_p->mv_s->session_iv; 2659 mv_p->flags |= RDY_CRP_IV; 2660 return 0; 2661 } 2662 cprng_fast(ivbuf, ivlen); 2663 iv = ivbuf; 2664 } 2665 memcpy(&mv_p->pkt_header.crp_iv_work, iv, ivlen); 2666 if (mv_p->flags & CRP_EXT_IV) { 2667 memcpy(&mv_p->pkt_header.crp_iv_ext, iv, ivlen); 2668 mv_p->ext_iv = iv; 2669 mv_p->ext_ivlen = ivlen; 2670 } 2671 mv_p->flags |= RDY_CRP_IV; 2672 2673 return 0; 2674} 2675 2676STATIC int 2677mvxpsec_packet_copy_iv(struct mvxpsec_packet *mv_p, int off, int ivlen) 2678{ 2679 mvxpsec_packet_rdata(mv_p, off, ivlen, 2680 &mv_p->pkt_header.crp_iv_work); 2681 mv_p->flags |= RDY_CRP_IV; 2682 2683 return 0; 2684} 2685 2686/* 2687 * set a encryption or decryption key to the session 2688 * 2689 * Input key material is big endian. 2690 */ 2691STATIC int 2692mvxpsec_key_precomp(int alg, void *keymat, int kbitlen, 2693 void *key_encrypt, void *key_decrypt) 2694{ 2695 uint32_t *kp = keymat; 2696 uint32_t *ekp = key_encrypt; 2697 uint32_t *dkp = key_decrypt; 2698 int i; 2699 2700 switch (alg) { 2701 case CRYPTO_DES_CBC: 2702 if (kbitlen < 64 || (kbitlen % 8) != 0) { 2703 log(LOG_WARNING, 2704 "mvxpsec: invalid DES keylen %d\n", kbitlen); 2705 return EINVAL; 2706 } 2707 for (i = 0; i < 2; i++) 2708 dkp[i] = ekp[i] = kp[i]; 2709 for (; i < 8; i++) 2710 dkp[i] = ekp[i] = 0; 2711 break; 2712 case CRYPTO_3DES_CBC: 2713 if (kbitlen < 192 || (kbitlen % 8) != 0) { 2714 log(LOG_WARNING, 2715 "mvxpsec: invalid 3DES keylen %d\n", kbitlen); 2716 return EINVAL; 2717 } 2718 for (i = 0; i < 8; i++) 2719 dkp[i] = ekp[i] = kp[i]; 2720 break; 2721 case CRYPTO_AES_CBC: 2722 if (kbitlen < 128) { 2723 log(LOG_WARNING, 2724 "mvxpsec: invalid AES keylen %d\n", kbitlen); 2725 return EINVAL; 2726 } 2727 else if (kbitlen < 192) { 2728 /* AES-128 */ 2729 for (i = 0; i < 4; i++) 2730 ekp[i] = kp[i]; 2731 for (; i < 8; i++) 2732 ekp[i] = 0; 2733 } 2734 else if (kbitlen < 256) { 2735 /* AES-192 */ 2736 for (i = 0; i < 6; i++) 2737 ekp[i] = kp[i]; 2738 for (; i < 8; i++) 2739 ekp[i] = 0; 2740 } 2741 else { 2742 /* AES-256 */ 2743 for (i = 0; i < 8; i++) 2744 ekp[i] = kp[i]; 2745 } 2746 /* make decryption key */ 2747 mv_aes_deckey((uint8_t *)dkp, (uint8_t *)ekp, kbitlen); 2748 break; 2749 default: 2750 for (i = 0; i < 8; i++) 2751 ekp[0] = dkp[0] = 0; 2752 break; 2753 } 2754 2755#ifdef MVXPSEC_DEBUG 2756 if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) { 2757 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2758 "%s: keyregistered\n", __func__); 2759 mvxpsec_dump_data(__func__, ekp, 32); 2760 } 2761#endif 2762 2763 return 0; 2764} 2765 2766/* 2767 * set MAC key to the session 2768 * 2769 * MAC engine has no register for key itself, but the engine has 2770 * inner and outer IV register. software must compute IV before 2771 * enable the engine. 2772 * 2773 * IV is a hash of ipad/opad. these are defined by FIPS-198a 2774 * standard. 2775 */ 2776STATIC int 2777mvxpsec_hmac_precomp(int alg, void *key, int kbitlen, 2778 void *iv_inner, void *iv_outer) 2779{ 2780 SHA1_CTX sha1; 2781 MD5_CTX md5; 2782 uint8_t *key8 = key; 2783 uint8_t kbuf[64]; 2784 uint8_t ipad[64]; 2785 uint8_t opad[64]; 2786 uint32_t *iv_in = iv_inner; 2787 uint32_t *iv_out = iv_outer; 2788 int kbytelen; 2789 int i; 2790#define HMAC_IPAD 0x36 2791#define HMAC_OPAD 0x5c 2792 2793 kbytelen = kbitlen / 8; 2794 KASSERT(kbitlen == kbytelen * 8); 2795 if (kbytelen > 64) { 2796 SHA1Init(&sha1); 2797 SHA1Update(&sha1, key, kbytelen); 2798 SHA1Final(kbuf, &sha1); 2799 key8 = kbuf; 2800 kbytelen = 64; 2801 } 2802 2803 /* make initial 64 oct. string */ 2804 switch (alg) { 2805 case CRYPTO_SHA1_HMAC_96: 2806 case CRYPTO_SHA1_HMAC: 2807 case CRYPTO_MD5_HMAC_96: 2808 case CRYPTO_MD5_HMAC: 2809 for (i = 0; i < kbytelen; i++) { 2810 ipad[i] = (key8[i] ^ HMAC_IPAD); 2811 opad[i] = (key8[i] ^ HMAC_OPAD); 2812 } 2813 for (; i < 64; i++) { 2814 ipad[i] = HMAC_IPAD; 2815 opad[i] = HMAC_OPAD; 2816 } 2817 break; 2818 default: 2819 break; 2820 } 2821#ifdef MVXPSEC_DEBUG 2822 if (mvxpsec_debug & MVXPSEC_DEBUG_OPENCRYPTO) { 2823 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2824 "%s: HMAC-KEY Pre-comp:\n", __func__); 2825 mvxpsec_dump_data(__func__, key, 64); 2826 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2827 "%s: ipad:\n", __func__); 2828 mvxpsec_dump_data(__func__, ipad, sizeof(ipad)); 2829 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2830 "%s: opad:\n", __func__); 2831 mvxpsec_dump_data(__func__, opad, sizeof(opad)); 2832 } 2833#endif 2834 2835 /* make iv from string */ 2836 switch (alg) { 2837 case CRYPTO_SHA1_HMAC_96: 2838 case CRYPTO_SHA1_HMAC: 2839 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2840 "%s: Generate iv_in(SHA1)\n", __func__); 2841 SHA1Init(&sha1); 2842 SHA1Update(&sha1, ipad, 64); 2843 /* XXX: private state... (LE) */ 2844 iv_in[0] = htobe32(sha1.state[0]); 2845 iv_in[1] = htobe32(sha1.state[1]); 2846 iv_in[2] = htobe32(sha1.state[2]); 2847 iv_in[3] = htobe32(sha1.state[3]); 2848 iv_in[4] = htobe32(sha1.state[4]); 2849 2850 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2851 "%s: Generate iv_out(SHA1)\n", __func__); 2852 SHA1Init(&sha1); 2853 SHA1Update(&sha1, opad, 64); 2854 /* XXX: private state... (LE) */ 2855 iv_out[0] = htobe32(sha1.state[0]); 2856 iv_out[1] = htobe32(sha1.state[1]); 2857 iv_out[2] = htobe32(sha1.state[2]); 2858 iv_out[3] = htobe32(sha1.state[3]); 2859 iv_out[4] = htobe32(sha1.state[4]); 2860 break; 2861 case CRYPTO_MD5_HMAC_96: 2862 case CRYPTO_MD5_HMAC: 2863 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2864 "%s: Generate iv_in(MD5)\n", __func__); 2865 MD5Init(&md5); 2866 MD5Update(&md5, ipad, sizeof(ipad)); 2867 /* XXX: private state... (LE) */ 2868 iv_in[0] = htobe32(md5.state[0]); 2869 iv_in[1] = htobe32(md5.state[1]); 2870 iv_in[2] = htobe32(md5.state[2]); 2871 iv_in[3] = htobe32(md5.state[3]); 2872 iv_in[4] = 0; 2873 2874 MVXPSEC_PRINTF(MVXPSEC_DEBUG_OPENCRYPTO, 2875 "%s: Generate iv_out(MD5)\n", __func__); 2876 MD5Init(&md5); 2877 MD5Update(&md5, opad, sizeof(opad)); 2878 /* XXX: private state... (LE) */ 2879 iv_out[0] = htobe32(md5.state[0]); 2880 iv_out[1] = htobe32(md5.state[1]); 2881 iv_out[2] = htobe32(md5.state[2]); 2882 iv_out[3] = htobe32(md5.state[3]); 2883 iv_out[4] = 0; 2884 break; 2885 default: 2886 break; 2887 } 2888 2889#ifdef MVXPSEC_DEBUG 2890 if (mvxpsec_debug & MVXPSEC_DEBUG_HASH_IV) { 2891 MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV, 2892 "%s: HMAC IV-IN\n", __func__); 2893 mvxpsec_dump_data(__func__, (uint8_t *)iv_in, 20); 2894 MVXPSEC_PRINTF(MVXPSEC_DEBUG_HASH_IV, 2895 "%s: HMAC IV-OUT\n", __func__); 2896 mvxpsec_dump_data(__func__, (uint8_t *)iv_out, 20); 2897 } 2898#endif 2899 2900 return 0; 2901#undef HMAC_IPAD 2902#undef HMAC_OPAD 2903} 2904 2905/* 2906 * AES Support routine 2907 */ 2908static uint8_t AES_SBOX[256] = { 2909 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 2910 171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 2911 156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 2912 229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154, 2913 7, 18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110, 2914 90, 160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237, 2915 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239, 2916 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168, 2917 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 2918 243, 210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 2919 100, 93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 2920 184, 20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92, 2921 194, 211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213, 2922 78, 169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46, 2923 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62, 2924 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158, 2925 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 2926 40, 223, 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 2927 176, 84, 187, 22 2928}; 2929 2930static uint32_t AES_RCON[30] = { 2931 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 2932 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 2933 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91 2934}; 2935 2936STATIC int 2937mv_aes_ksched(uint8_t k[4][MAXKC], int keyBits, 2938 uint8_t W[MAXROUNDS+1][4][MAXBC]) 2939{ 2940 int KC, BC, ROUNDS; 2941 int i, j, t, rconpointer = 0; 2942 uint8_t tk[4][MAXKC]; 2943 2944 switch (keyBits) { 2945 case 128: 2946 ROUNDS = 10; 2947 KC = 4; 2948 break; 2949 case 192: 2950 ROUNDS = 12; 2951 KC = 6; 2952 break; 2953 case 256: 2954 ROUNDS = 14; 2955 KC = 8; 2956 break; 2957 default: 2958 return (-1); 2959 } 2960 BC = 4; /* 128 bits */ 2961 2962 for(j = 0; j < KC; j++) 2963 for(i = 0; i < 4; i++) 2964 tk[i][j] = k[i][j]; 2965 t = 0; 2966 2967 /* copy values into round key array */ 2968 for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++) 2969 for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j]; 2970 2971 while (t < (ROUNDS+1)*BC) { /* while not enough round key material calculated */ 2972 /* calculate new values */ 2973 for(i = 0; i < 4; i++) 2974 tk[i][0] ^= AES_SBOX[tk[(i+1)%4][KC-1]]; 2975 tk[0][0] ^= AES_RCON[rconpointer++]; 2976 2977 if (KC != 8) 2978 for(j = 1; j < KC; j++) 2979 for(i = 0; i < 4; i++) 2980 tk[i][j] ^= tk[i][j-1]; 2981 else { 2982 for(j = 1; j < KC/2; j++) 2983 for(i = 0; i < 4; i++) 2984 tk[i][j] ^= tk[i][j-1]; 2985 for(i = 0; i < 4; i++) 2986 tk[i][KC/2] ^= AES_SBOX[tk[i][KC/2 - 1]]; 2987 for(j = KC/2 + 1; j < KC; j++) 2988 for(i = 0; i < 4; i++) 2989 tk[i][j] ^= tk[i][j-1]; 2990 } 2991 /* copy values into round key array */ 2992 for(j = 0; (j < KC) && (t < (ROUNDS+1)*BC); j++, t++) 2993 for(i = 0; i < 4; i++) W[t / BC][i][t % BC] = tk[i][j]; 2994 } 2995 2996 return 0; 2997} 2998 2999STATIC int 3000mv_aes_deckey(uint8_t *expandedKey, uint8_t *keyMaterial, int keyLen) 3001{ 3002 uint8_t W[MAXROUNDS+1][4][MAXBC]; 3003 uint8_t k[4][MAXKC]; 3004 uint8_t j; 3005 int i, rounds, KC; 3006 3007 if (expandedKey == NULL) 3008 return -1; 3009 3010 if (!((keyLen == 128) || (keyLen == 192) || (keyLen == 256))) 3011 return -1; 3012 3013 if (keyMaterial == NULL) 3014 return -1; 3015 3016 /* initialize key schedule: */ 3017 for (i=0; i<keyLen/8; i++) { 3018 j = keyMaterial[i]; 3019 k[i % 4][i / 4] = j; 3020 } 3021 3022 mv_aes_ksched(k, keyLen, W); 3023 switch (keyLen) { 3024 case 128: 3025 rounds = 10; 3026 KC = 4; 3027 break; 3028 case 192: 3029 rounds = 12; 3030 KC = 6; 3031 break; 3032 case 256: 3033 rounds = 14; 3034 KC = 8; 3035 break; 3036 default: 3037 return -1; 3038 } 3039 3040 for(i=0; i<MAXBC; i++) 3041 for(j=0; j<4; j++) 3042 expandedKey[i*4+j] = W[rounds][j][i]; 3043 for(; i<KC; i++) 3044 for(j=0; j<4; j++) 3045 expandedKey[i*4+j] = W[rounds-1][j][i+MAXBC-KC]; 3046 3047 return 0; 3048} 3049 3050/* 3051 * Clear cipher/mac operation state 3052 */ 3053INLINE void 3054mvxpsec_packet_reset_op(struct mvxpsec_packet *mv_p) 3055{ 3056 mv_p->pkt_header.desc.acc_config = 0; 3057 mv_p->enc_off = mv_p->enc_ivoff = mv_p->enc_len = 0; 3058 mv_p->mac_off = mv_p->mac_dst = mv_p->mac_len = 0; 3059} 3060 3061/* 3062 * update MVXPSEC operation order 3063 */ 3064INLINE void 3065mvxpsec_packet_update_op_order(struct mvxpsec_packet *mv_p, int op) 3066{ 3067 struct mvxpsec_acc_descriptor *acc_desc = &mv_p->pkt_header.desc; 3068 uint32_t cur_op = acc_desc->acc_config & MV_ACC_CRYPTO_OP_MASK; 3069 3070 KASSERT(op == MV_ACC_CRYPTO_OP_MAC || op == MV_ACC_CRYPTO_OP_ENC); 3071 KASSERT((op & MV_ACC_CRYPTO_OP_MASK) == op); 3072 3073 if (cur_op == 0) 3074 acc_desc->acc_config |= op; 3075 else if (cur_op == MV_ACC_CRYPTO_OP_MAC && op == MV_ACC_CRYPTO_OP_ENC) { 3076 acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK; 3077 acc_desc->acc_config |= MV_ACC_CRYPTO_OP_MACENC; 3078 /* MAC then ENC (= decryption) */ 3079 } 3080 else if (cur_op == MV_ACC_CRYPTO_OP_ENC && op == MV_ACC_CRYPTO_OP_MAC) { 3081 acc_desc->acc_config &= ~MV_ACC_CRYPTO_OP_MASK; 3082 acc_desc->acc_config |= MV_ACC_CRYPTO_OP_ENCMAC; 3083 /* ENC then MAC (= encryption) */ 3084 } 3085 else { 3086 log(LOG_ERR, "%s: multiple %s algorithm is not supported.\n", 3087 __func__, 3088 (op == MV_ACC_CRYPTO_OP_ENC) ? "encryption" : "authentication"); 3089 } 3090} 3091 3092/* 3093 * Parameter Conversions 3094 */ 3095INLINE uint32_t 3096mvxpsec_alg2acc(uint32_t alg) 3097{ 3098 uint32_t reg; 3099 3100 switch (alg) { 3101 case CRYPTO_DES_CBC: 3102 reg = MV_ACC_CRYPTO_ENC_DES; 3103 reg |= MV_ACC_CRYPTO_CBC; 3104 break; 3105 case CRYPTO_3DES_CBC: 3106 reg = MV_ACC_CRYPTO_ENC_3DES; 3107 reg |= MV_ACC_CRYPTO_3DES_EDE; 3108 reg |= MV_ACC_CRYPTO_CBC; 3109 break; 3110 case CRYPTO_AES_CBC: 3111 reg = MV_ACC_CRYPTO_ENC_AES; 3112 reg |= MV_ACC_CRYPTO_CBC; 3113 break; 3114 case CRYPTO_SHA1_HMAC_96: 3115 reg = MV_ACC_CRYPTO_MAC_HMAC_SHA1; 3116 reg |= MV_ACC_CRYPTO_MAC_96; 3117 break; 3118 case CRYPTO_MD5_HMAC_96: 3119 reg = MV_ACC_CRYPTO_MAC_HMAC_MD5; 3120 reg |= MV_ACC_CRYPTO_MAC_96; 3121 break; 3122 default: 3123 reg = 0; 3124 break; 3125 } 3126 3127 return reg; 3128} 3129 3130INLINE uint32_t 3131mvxpsec_aesklen(int klen) 3132{ 3133 if (klen < 128) 3134 return 0; 3135 else if (klen < 192) 3136 return MV_ACC_CRYPTO_AES_KLEN_128; 3137 else if (klen < 256) 3138 return MV_ACC_CRYPTO_AES_KLEN_192; 3139 else 3140 return MV_ACC_CRYPTO_AES_KLEN_256; 3141 3142 return 0; 3143} 3144 3145/* 3146 * String Conversions 3147 */ 3148STATIC const char * 3149s_errreg(uint32_t v) 3150{ 3151 static char buf[80]; 3152 3153 snprintf(buf, sizeof(buf), 3154 "%sMiss %sDoubleHit %sBothHit %sDataError", 3155 (v & MV_TDMA_ERRC_MISS) ? "+" : "-", 3156 (v & MV_TDMA_ERRC_DHIT) ? "+" : "-", 3157 (v & MV_TDMA_ERRC_BHIT) ? "+" : "-", 3158 (v & MV_TDMA_ERRC_DERR) ? "+" : "-"); 3159 3160 return (const char *)buf; 3161} 3162 3163STATIC const char * 3164s_winreg(uint32_t v) 3165{ 3166 static char buf[80]; 3167 3168 snprintf(buf, sizeof(buf), 3169 "%s TGT 0x%x ATTR 0x%02x size %u(0x%04x)[64KB]", 3170 (v & MV_TDMA_ATTR_ENABLE) ? "EN" : "DIS", 3171 MV_TDMA_ATTR_GET_TARGET(v), MV_TDMA_ATTR_GET_ATTR(v), 3172 MV_TDMA_ATTR_GET_SIZE(v), MV_TDMA_ATTR_GET_SIZE(v)); 3173 3174 return (const char *)buf; 3175} 3176 3177STATIC const char * 3178s_ctrlreg(uint32_t reg) 3179{ 3180 static char buf[80]; 3181 3182 snprintf(buf, sizeof(buf), 3183 "%s: %sFETCH DBURST-%u SBURST-%u %sOUTS %sCHAIN %sBSWAP %sACT", 3184 (reg & MV_TDMA_CONTROL_ENABLE) ? "ENABLE" : "DISABLE", 3185 (reg & MV_TDMA_CONTROL_FETCH) ? "+" : "-", 3186 MV_TDMA_CONTROL_GET_DST_BURST(reg), 3187 MV_TDMA_CONTROL_GET_SRC_BURST(reg), 3188 (reg & MV_TDMA_CONTROL_OUTS_EN) ? "+" : "-", 3189 (reg & MV_TDMA_CONTROL_CHAIN_DIS) ? "-" : "+", 3190 (reg & MV_TDMA_CONTROL_BSWAP_DIS) ? "-" : "+", 3191 (reg & MV_TDMA_CONTROL_ACT) ? "+" : "-"); 3192 3193 return (const char *)buf; 3194} 3195 3196_STATIC const char * 3197s_xpsecintr(uint32_t v) 3198{ 3199 static char buf[160]; 3200 3201 snprintf(buf, sizeof(buf), 3202 "%sAuth %sDES %sAES-ENC %sAES-DEC %sENC %sSA %sAccAndTDMA " 3203 "%sTDMAComp %sTDMAOwn %sAccAndTDMA_Cont", 3204 (v & MVXPSEC_INT_AUTH) ? "+" : "-", 3205 (v & MVXPSEC_INT_DES) ? "+" : "-", 3206 (v & MVXPSEC_INT_AES_ENC) ? "+" : "-", 3207 (v & MVXPSEC_INT_AES_DEC) ? "+" : "-", 3208 (v & MVXPSEC_INT_ENC) ? "+" : "-", 3209 (v & MVXPSEC_INT_SA) ? "+" : "-", 3210 (v & MVXPSEC_INT_ACCTDMA) ? "+" : "-", 3211 (v & MVXPSEC_INT_TDMA_COMP) ? "+" : "-", 3212 (v & MVXPSEC_INT_TDMA_OWN) ? "+" : "-", 3213 (v & MVXPSEC_INT_ACCTDMA_CONT) ? "+" : "-"); 3214 3215 return (const char *)buf; 3216} 3217 3218STATIC const char * 3219s_ctlalg(uint32_t alg) 3220{ 3221 switch (alg) { 3222 case CRYPTO_SHA1_HMAC_96: 3223 return "HMAC-SHA1-96"; 3224 case CRYPTO_SHA1_HMAC: 3225 return "HMAC-SHA1"; 3226 case CRYPTO_SHA1: 3227 return "SHA1"; 3228 case CRYPTO_MD5_HMAC_96: 3229 return "HMAC-MD5-96"; 3230 case CRYPTO_MD5_HMAC: 3231 return "HMAC-MD5"; 3232 case CRYPTO_MD5: 3233 return "MD5"; 3234 case CRYPTO_DES_CBC: 3235 return "DES-CBC"; 3236 case CRYPTO_3DES_CBC: 3237 return "3DES-CBC"; 3238 case CRYPTO_AES_CBC: 3239 return "AES-CBC"; 3240 default: 3241 break; 3242 } 3243 3244 return "Unknown"; 3245} 3246 3247STATIC const char * 3248s_xpsec_op(uint32_t reg) 3249{ 3250 reg &= MV_ACC_CRYPTO_OP_MASK; 3251 switch (reg) { 3252 case MV_ACC_CRYPTO_OP_ENC: 3253 return "ENC"; 3254 case MV_ACC_CRYPTO_OP_MAC: 3255 return "MAC"; 3256 case MV_ACC_CRYPTO_OP_ENCMAC: 3257 return "ENC-MAC"; 3258 case MV_ACC_CRYPTO_OP_MACENC: 3259 return "MAC-ENC"; 3260 default: 3261 break; 3262 } 3263 3264 return "Unknown"; 3265} 3266 3267STATIC const char * 3268s_xpsec_enc(uint32_t alg) 3269{ 3270 alg <<= MV_ACC_CRYPTO_ENC_SHIFT; 3271 switch (alg) { 3272 case MV_ACC_CRYPTO_ENC_DES: 3273 return "DES"; 3274 case MV_ACC_CRYPTO_ENC_3DES: 3275 return "3DES"; 3276 case MV_ACC_CRYPTO_ENC_AES: 3277 return "AES"; 3278 default: 3279 break; 3280 } 3281 3282 return "Unknown"; 3283} 3284 3285STATIC const char * 3286s_xpsec_mac(uint32_t alg) 3287{ 3288 alg <<= MV_ACC_CRYPTO_MAC_SHIFT; 3289 switch (alg) { 3290 case MV_ACC_CRYPTO_MAC_NONE: 3291 return "Disabled"; 3292 case MV_ACC_CRYPTO_MAC_MD5: 3293 return "MD5"; 3294 case MV_ACC_CRYPTO_MAC_SHA1: 3295 return "SHA1"; 3296 case MV_ACC_CRYPTO_MAC_HMAC_MD5: 3297 return "HMAC-MD5"; 3298 case MV_ACC_CRYPTO_MAC_HMAC_SHA1: 3299 return "HMAC-SHA1"; 3300 default: 3301 break; 3302 } 3303 3304 return "Unknown"; 3305} 3306 3307STATIC const char * 3308s_xpsec_frag(uint32_t frag) 3309{ 3310 frag <<= MV_ACC_CRYPTO_FRAG_SHIFT; 3311 switch (frag) { 3312 case MV_ACC_CRYPTO_NOFRAG: 3313 return "NoFragment"; 3314 case MV_ACC_CRYPTO_FRAG_FIRST: 3315 return "FirstFragment"; 3316 case MV_ACC_CRYPTO_FRAG_MID: 3317 return "MiddleFragment"; 3318 case MV_ACC_CRYPTO_FRAG_LAST: 3319 return "LastFragment"; 3320 default: 3321 break; 3322 } 3323 3324 return "Unknown"; 3325} 3326 3327#ifdef MVXPSEC_DEBUG 3328void 3329mvxpsec_dump_reg(struct mvxpsec_softc *sc) 3330{ 3331 uint32_t reg; 3332 int i; 3333 3334 if ((mvxpsec_debug & MVXPSEC_DEBUG_DESC) == 0) 3335 return; 3336 3337 printf("--- Interrupt Registers ---\n"); 3338 reg = MVXPSEC_READ(sc, MVXPSEC_INT_CAUSE); 3339 printf("MVXPSEC INT CAUSE: 0x%08x\n", reg); 3340 printf("MVXPSEC INT CAUSE: %s\n", s_xpsecintr(reg)); 3341 reg = MVXPSEC_READ(sc, MVXPSEC_INT_MASK); 3342 printf("MVXPSEC INT MASK: 0x%08x\n", reg); 3343 printf("MVXPSEC INT MASKE: %s\n", s_xpsecintr(reg)); 3344 3345 printf("--- DMA Configuration Registers ---\n"); 3346 for (i = 0; i < MV_TDMA_NWINDOW; i++) { 3347 reg = MVXPSEC_READ(sc, MV_TDMA_BAR(i)); 3348 printf("TDMA BAR%d: 0x%08x\n", i, reg); 3349 reg = MVXPSEC_READ(sc, MV_TDMA_ATTR(i)); 3350 printf("TDMA ATTR%d: 0x%08x\n", i, reg); 3351 printf(" -> %s\n", s_winreg(reg)); 3352 } 3353 3354 printf("--- DMA Control Registers ---\n"); 3355 3356 reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL); 3357 printf("TDMA CONTROL: 0x%08x\n", reg); 3358 printf(" -> %s\n", s_ctrlreg(reg)); 3359 3360 printf("--- DMA Current Command Descriptors ---\n"); 3361 3362 reg = MVXPSEC_READ(sc, MV_TDMA_ERR_CAUSE); 3363 printf("TDMA ERR CAUSE: 0x%08x\n", reg); 3364 3365 reg = MVXPSEC_READ(sc, MV_TDMA_ERR_MASK); 3366 printf("TDMA ERR MASK: 0x%08x\n", reg); 3367 3368 reg = MVXPSEC_READ(sc, MV_TDMA_CNT); 3369 printf("TDMA DATA OWNER: %s\n", 3370 (reg & MV_TDMA_CNT_OWN) ? "DMAC" : "CPU"); 3371 printf("TDMA DATA COUNT: %d(0x%x)\n", 3372 (reg & ~MV_TDMA_CNT_OWN), (reg & ~MV_TDMA_CNT_OWN)); 3373 3374 reg = MVXPSEC_READ(sc, MV_TDMA_SRC); 3375 printf("TDMA DATA SRC: 0x%08x\n", reg); 3376 3377 reg = MVXPSEC_READ(sc, MV_TDMA_DST); 3378 printf("TDMA DATA DST: 0x%08x\n", reg); 3379 3380 reg = MVXPSEC_READ(sc, MV_TDMA_NXT); 3381 printf("TDMA DATA NXT: 0x%08x\n", reg); 3382 3383 reg = MVXPSEC_READ(sc, MV_TDMA_CUR); 3384 printf("TDMA DATA CUR: 0x%08x\n", reg); 3385 3386 printf("--- ACC Command Register ---\n"); 3387 reg = MVXPSEC_READ(sc, MV_ACC_COMMAND); 3388 printf("ACC COMMAND: 0x%08x\n", reg); 3389 printf("ACC: %sACT %sSTOP\n", 3390 (reg & MV_ACC_COMMAND_ACT) ? "+" : "-", 3391 (reg & MV_ACC_COMMAND_STOP) ? "+" : "-"); 3392 3393 reg = MVXPSEC_READ(sc, MV_ACC_CONFIG); 3394 printf("ACC CONFIG: 0x%08x\n", reg); 3395 reg = MVXPSEC_READ(sc, MV_ACC_DESC); 3396 printf("ACC DESC: 0x%08x\n", reg); 3397 3398 printf("--- DES Key Register ---\n"); 3399 reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0L); 3400 printf("DES KEY0 Low: 0x%08x\n", reg); 3401 reg = MVXPSEC_READ(sc, MV_CE_DES_KEY0H); 3402 printf("DES KEY0 High: 0x%08x\n", reg); 3403 reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1L); 3404 printf("DES KEY1 Low: 0x%08x\n", reg); 3405 reg = MVXPSEC_READ(sc, MV_CE_DES_KEY1H); 3406 printf("DES KEY1 High: 0x%08x\n", reg); 3407 reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2L); 3408 printf("DES KEY2 Low: 0x%08x\n", reg); 3409 reg = MVXPSEC_READ(sc, MV_CE_DES_KEY2H); 3410 printf("DES KEY2 High: 0x%08x\n", reg); 3411 3412 printf("--- AES Key Register ---\n"); 3413 for (i = 0; i < 8; i++) { 3414 reg = MVXPSEC_READ(sc, MV_CE_AES_EKEY(i)); 3415 printf("AES ENC KEY COL%d: %08x\n", i, reg); 3416 } 3417 for (i = 0; i < 8; i++) { 3418 reg = MVXPSEC_READ(sc, MV_CE_AES_DKEY(i)); 3419 printf("AES DEC KEY COL%d: %08x\n", i, reg); 3420 } 3421 3422 return; 3423} 3424 3425STATIC void 3426mvxpsec_dump_sram(const char *name, struct mvxpsec_softc *sc, size_t len) 3427{ 3428 uint32_t reg; 3429 3430 if (sc->sc_sram_va == NULL) 3431 return; 3432 3433 if (len == 0) { 3434 printf("\n%s NO DATA(len=0)\n", name); 3435 return; 3436 } 3437 else if (len > MV_ACC_SRAM_SIZE) 3438 len = MV_ACC_SRAM_SIZE; 3439 3440 mutex_enter(&sc->sc_dma_mtx); 3441 reg = MVXPSEC_READ(sc, MV_TDMA_CONTROL); 3442 if (reg & MV_TDMA_CONTROL_ACT) { 3443 printf("TDMA is active, cannot access SRAM\n"); 3444 mutex_exit(&sc->sc_dma_mtx); 3445 return; 3446 } 3447 reg = MVXPSEC_READ(sc, MV_ACC_COMMAND); 3448 if (reg & MV_ACC_COMMAND_ACT) { 3449 printf("SA is active, cannot access SRAM\n"); 3450 mutex_exit(&sc->sc_dma_mtx); 3451 return; 3452 } 3453 3454 printf("%s: dump SRAM, %zu bytes\n", name, len); 3455 mvxpsec_dump_data(name, sc->sc_sram_va, len); 3456 mutex_exit(&sc->sc_dma_mtx); 3457 return; 3458} 3459 3460 3461_STATIC void 3462mvxpsec_dump_dmaq(struct mvxpsec_descriptor_handle *dh) 3463{ 3464 struct mvxpsec_descriptor *d = 3465 (struct mvxpsec_descriptor *)dh->_desc; 3466 3467 printf("--- DMA Command Descriptor ---\n"); 3468 printf("DESC: VA=%p PA=0x%08x\n", 3469 d, (uint32_t)dh->phys_addr); 3470 printf("DESC: WORD0 = 0x%08x\n", d->tdma_word0); 3471 printf("DESC: SRC = 0x%08x\n", d->tdma_src); 3472 printf("DESC: DST = 0x%08x\n", d->tdma_dst); 3473 printf("DESC: NXT = 0x%08x\n", d->tdma_nxt); 3474 3475 return; 3476} 3477 3478STATIC void 3479mvxpsec_dump_data(const char *name, void *p, size_t len) 3480{ 3481 uint8_t *data = p; 3482 off_t off; 3483 3484 printf("%s: dump %p, %zu bytes", name, p, len); 3485 if (p == NULL || len == 0) { 3486 printf("\n%s: NO DATA\n", name); 3487 return; 3488 } 3489 for (off = 0; off < len; off++) { 3490 if ((off % 16) == 0) { 3491 printf("\n%s: 0x%08x:", name, (uint32_t)off); 3492 } 3493 if ((off % 4) == 0) { 3494 printf(" "); 3495 } 3496 printf("%02x", data[off]); 3497 } 3498 printf("\n"); 3499 3500 return; 3501} 3502 3503_STATIC void 3504mvxpsec_dump_packet(const char *name, struct mvxpsec_packet *mv_p) 3505{ 3506 struct mvxpsec_softc *sc = mv_p->mv_s->sc; 3507 3508 printf("%s: packet_data:\n", name); 3509 mvxpsec_dump_packet_data(name, mv_p); 3510 3511 printf("%s: SRAM:\n", name); 3512 mvxpsec_dump_sram(name, sc, 2000); 3513 3514 printf("%s: packet_descriptor:\n", name); 3515 mvxpsec_dump_packet_desc(name, mv_p); 3516} 3517 3518_STATIC void 3519mvxpsec_dump_packet_data(const char *name, struct mvxpsec_packet *mv_p) 3520{ 3521 static char buf[1500]; 3522 int len; 3523 3524 if (mv_p->data_type == MVXPSEC_DATA_MBUF) { 3525 struct mbuf *m; 3526 3527 m = mv_p->data.mbuf; 3528 len = m->m_pkthdr.len; 3529 if (len > sizeof(buf)) 3530 len = sizeof(buf); 3531 m_copydata(m, 0, len, buf); 3532 } 3533 else if (mv_p->data_type == MVXPSEC_DATA_UIO) { 3534 struct uio *uio; 3535 3536 uio = mv_p->data.uio; 3537 len = uio->uio_resid; 3538 if (len > sizeof(buf)) 3539 len = sizeof(buf); 3540 cuio_copydata(uio, 0, len, buf); 3541 } 3542 else if (mv_p->data_type == MVXPSEC_DATA_RAW) { 3543 len = mv_p->data_len; 3544 if (len > sizeof(buf)) 3545 len = sizeof(buf); 3546 memcpy(buf, mv_p->data.raw, len); 3547 } 3548 else 3549 return; 3550 mvxpsec_dump_data(name, buf, len); 3551 3552 return; 3553} 3554 3555_STATIC void 3556mvxpsec_dump_packet_desc(const char *name, struct mvxpsec_packet *mv_p) 3557{ 3558 uint32_t *words; 3559 3560 if (mv_p == NULL) 3561 return; 3562 3563 words = &mv_p->pkt_header.desc.acc_desc_dword0; 3564 mvxpsec_dump_acc_config(name, words[0]); 3565 mvxpsec_dump_acc_encdata(name, words[1], words[2]); 3566 mvxpsec_dump_acc_enclen(name, words[2]); 3567 mvxpsec_dump_acc_enckey(name, words[3]); 3568 mvxpsec_dump_acc_enciv(name, words[4]); 3569 mvxpsec_dump_acc_macsrc(name, words[5]); 3570 mvxpsec_dump_acc_macdst(name, words[6]); 3571 mvxpsec_dump_acc_maciv(name, words[7]); 3572 3573 return; 3574} 3575 3576_STATIC void 3577mvxpsec_dump_acc_config(const char *name, uint32_t w) 3578{ 3579 /* SA: Dword 0 */ 3580 printf("%s: Dword0=0x%08x\n", name, w); 3581 printf("%s: OP = %s\n", name, 3582 s_xpsec_op(MV_ACC_CRYPTO_OP(w))); 3583 printf("%s: MAC = %s\n", name, 3584 s_xpsec_mac(MV_ACC_CRYPTO_MAC(w))); 3585 printf("%s: MAC_LEN = %s\n", name, 3586 w & MV_ACC_CRYPTO_MAC_96 ? "96-bit" : "full-bit"); 3587 printf("%s: ENC = %s\n", name, 3588 s_xpsec_enc(MV_ACC_CRYPTO_ENC(w))); 3589 printf("%s: DIR = %s\n", name, 3590 w & MV_ACC_CRYPTO_DECRYPT ? "decryption" : "encryption"); 3591 printf("%s: CHAIN = %s\n", name, 3592 w & MV_ACC_CRYPTO_CBC ? "CBC" : "ECB"); 3593 printf("%s: 3DES = %s\n", name, 3594 w & MV_ACC_CRYPTO_3DES_EDE ? "EDE" : "EEE"); 3595 printf("%s: FRAGMENT = %s\n", name, 3596 s_xpsec_frag(MV_ACC_CRYPTO_FRAG(w))); 3597 return; 3598} 3599 3600STATIC void 3601mvxpsec_dump_acc_encdata(const char *name, uint32_t w, uint32_t w2) 3602{ 3603 /* SA: Dword 1 */ 3604 printf("%s: Dword1=0x%08x\n", name, w); 3605 printf("%s: ENC SRC = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w)); 3606 printf("%s: ENC DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w)); 3607 printf("%s: ENC RANGE = 0x%x - 0x%x\n", name, 3608 MV_ACC_DESC_GET_VAL_1(w), 3609 MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_1(w2) - 1); 3610 return; 3611} 3612 3613STATIC void 3614mvxpsec_dump_acc_enclen(const char *name, uint32_t w) 3615{ 3616 /* SA: Dword 2 */ 3617 printf("%s: Dword2=0x%08x\n", name, w); 3618 printf("%s: ENC LEN = %d\n", name, 3619 MV_ACC_DESC_GET_VAL_1(w)); 3620 return; 3621} 3622 3623STATIC void 3624mvxpsec_dump_acc_enckey(const char *name, uint32_t w) 3625{ 3626 /* SA: Dword 3 */ 3627 printf("%s: Dword3=0x%08x\n", name, w); 3628 printf("%s: EKEY = 0x%x\n", name, 3629 MV_ACC_DESC_GET_VAL_1(w)); 3630 return; 3631} 3632 3633STATIC void 3634mvxpsec_dump_acc_enciv(const char *name, uint32_t w) 3635{ 3636 /* SA: Dword 4 */ 3637 printf("%s: Dword4=0x%08x\n", name, w); 3638 printf("%s: EIV = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w)); 3639 printf("%s: EIV_BUF = 0x%x\n", name, MV_ACC_DESC_GET_VAL_2(w)); 3640 return; 3641} 3642 3643STATIC void 3644mvxpsec_dump_acc_macsrc(const char *name, uint32_t w) 3645{ 3646 /* SA: Dword 5 */ 3647 printf("%s: Dword5=0x%08x\n", name, w); 3648 printf("%s: MAC_SRC = 0x%x\n", name, 3649 MV_ACC_DESC_GET_VAL_1(w)); 3650 printf("%s: MAC_TOTAL_LEN = %d\n", name, 3651 MV_ACC_DESC_GET_VAL_3(w)); 3652 printf("%s: MAC_RANGE = 0x%0x - 0x%0x\n", name, 3653 MV_ACC_DESC_GET_VAL_1(w), 3654 MV_ACC_DESC_GET_VAL_1(w) + MV_ACC_DESC_GET_VAL_3(w) - 1); 3655 return; 3656} 3657 3658STATIC void 3659mvxpsec_dump_acc_macdst(const char *name, uint32_t w) 3660{ 3661 /* SA: Dword 6 */ 3662 printf("%s: Dword6=0x%08x\n", name, w); 3663 printf("%s: MAC_DST = 0x%x\n", name, MV_ACC_DESC_GET_VAL_1(w)); 3664 printf("%s: MAC_BLOCK_LEN = %d\n", name, 3665 MV_ACC_DESC_GET_VAL_2(w)); 3666 return; 3667} 3668 3669STATIC void 3670mvxpsec_dump_acc_maciv(const char *name, uint32_t w) 3671{ 3672 /* SA: Dword 7 */ 3673 printf("%s: Dword7=0x%08x\n", name, w); 3674 printf("%s: MAC_INNER_IV = 0x%x\n", name, 3675 MV_ACC_DESC_GET_VAL_1(w)); 3676 printf("%s: MAC_OUTER_IV = 0x%x\n", name, 3677 MV_ACC_DESC_GET_VAL_2(w)); 3678 return; 3679} 3680#endif 3681