1/* SPDX-License-Identifier: BSD-3-Clause */ 2/* Copyright(c) 2007-2022 Intel Corporation */ 3/* System headers */ 4#include <sys/param.h> 5#include <sys/systm.h> 6#include <sys/bus.h> 7#include <sys/cpu.h> 8#include <sys/kernel.h> 9#include <sys/mbuf.h> 10#include <sys/module.h> 11#include <sys/mutex.h> 12#include <sys/sysctl.h> 13 14/* Cryptodev headers */ 15#include <opencrypto/cryptodev.h> 16#include "cryptodev_if.h" 17 18/* QAT specific headers */ 19#include "cpa.h" 20#include "cpa_cy_im.h" 21#include "cpa_cy_sym_dp.h" 22#include "adf_accel_devices.h" 23#include "adf_common_drv.h" 24#include "lac_sym_hash_defs.h" 25#include "lac_sym_qat_hash_defs_lookup.h" 26 27/* To get only IRQ instances */ 28#include "icp_accel_devices.h" 29#include "icp_adf_accel_mgr.h" 30#include "lac_sal_types.h" 31 32/* To disable AEAD HW MAC verification */ 33#include "icp_sal_user.h" 34 35/* QAT OCF specific headers */ 36#include "qat_ocf_mem_pool.h" 37#include "qat_ocf_utils.h" 38 39#define QAT_OCF_MAX_INSTANCES (256) 40#define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000) 41 42MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations"); 43 44/* QAT OCF internal structures */ 45struct qat_ocf_softc { 46 device_t sc_dev; 47 struct sysctl_oid *rc; 48 uint32_t enabled; 49 int32_t cryptodev_id; 50 struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES]; 51 int32_t numCyInstances; 52}; 53 54/* Function definitions */ 55static void qat_ocf_freesession(device_t dev, crypto_session_t cses); 56static int qat_ocf_probesession(device_t dev, 57 const struct crypto_session_params *csp); 58static int qat_ocf_newsession(device_t dev, 59 crypto_session_t cses, 60 const struct crypto_session_params *csp); 61static int qat_ocf_attach(device_t dev); 62static int qat_ocf_detach(device_t dev); 63 64static void 65symDpCallback(CpaCySymDpOpData *pOpData, 66 CpaStatus result, 67 CpaBoolean verifyResult) 68{ 69 struct qat_ocf_cookie *qat_cookie; 70 struct cryptop *crp; 71 struct qat_ocf_dsession *qat_dsession = NULL; 72 struct qat_ocf_session *qat_session = NULL; 73 struct qat_ocf_instance *qat_instance = NULL; 74 CpaStatus status; 75 int rc = 0; 76 77 qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag; 78 if (!qat_cookie) 79 return; 80 81 crp = qat_cookie->crp_op; 82 83 qat_dsession = crypto_get_driver_session(crp->crp_session); 84 qat_instance = qat_dsession->qatInstance; 85 86 status = qat_ocf_cookie_dma_post_sync(crp, pOpData); 87 if (CPA_STATUS_SUCCESS != status) { 88 rc = EIO; 89 goto exit; 90 } 91 92 status = qat_ocf_cookie_dma_unload(crp, pOpData); 93 if (CPA_STATUS_SUCCESS != status) { 94 rc = EIO; 95 goto exit; 96 } 97 98 /* Verify result */ 99 if (CPA_STATUS_SUCCESS != result) { 100 rc = EBADMSG; 101 goto exit; 102 } 103 104 /* Verify digest by FW (GCM and CCM only) */ 105 if (CPA_TRUE != verifyResult) { 106 rc = EBADMSG; 107 goto exit; 108 } 109 110 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 111 qat_session = &qat_dsession->encSession; 112 else 113 qat_session = &qat_dsession->decSession; 114 115 /* Copy back digest result if it's stored in separated buffer */ 116 if (pOpData->digestResult && qat_session->authLen > 0) { 117 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { 118 char icv[QAT_OCF_MAX_DIGEST] = { 0 }; 119 crypto_copydata(crp, 120 crp->crp_digest_start, 121 qat_session->authLen, 122 icv); 123 if (timingsafe_bcmp(icv, 124 qat_cookie->qat_ocf_digest, 125 qat_session->authLen) != 0) { 126 rc = EBADMSG; 127 goto exit; 128 } 129 } else { 130 crypto_copyback(crp, 131 crp->crp_digest_start, 132 qat_session->authLen, 133 qat_cookie->qat_ocf_digest); 134 } 135 } 136 137exit: 138 qat_ocf_cookie_free(qat_instance, qat_cookie); 139 crp->crp_etype = rc; 140 crypto_done(crp); 141 142 return; 143} 144 145static inline CpaPhysicalAddr 146qatVirtToPhys(void *virtAddr) 147{ 148 return (CpaPhysicalAddr)vtophys(virtAddr); 149} 150 151static int 152qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp) 153{ 154 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 155 0) { 156 return EINVAL; 157 } 158 159 switch (csp->csp_mode) { 160 case CSP_MODE_CIPHER: 161 switch (csp->csp_cipher_alg) { 162 case CRYPTO_AES_CBC: 163 case CRYPTO_AES_ICM: 164 if (csp->csp_ivlen != AES_BLOCK_LEN) 165 return EINVAL; 166 break; 167 case CRYPTO_AES_XTS: 168 if (csp->csp_ivlen != AES_XTS_IV_LEN) 169 return EINVAL; 170 break; 171 default: 172 return EINVAL; 173 } 174 break; 175 case CSP_MODE_DIGEST: 176 switch (csp->csp_auth_alg) { 177 case CRYPTO_SHA1: 178 case CRYPTO_SHA1_HMAC: 179 case CRYPTO_SHA2_256: 180 case CRYPTO_SHA2_256_HMAC: 181 case CRYPTO_SHA2_384: 182 case CRYPTO_SHA2_384_HMAC: 183 case CRYPTO_SHA2_512: 184 case CRYPTO_SHA2_512_HMAC: 185 break; 186 case CRYPTO_AES_NIST_GMAC: 187 if (csp->csp_ivlen != AES_GCM_IV_LEN) 188 return EINVAL; 189 break; 190 default: 191 return EINVAL; 192 } 193 break; 194 case CSP_MODE_AEAD: 195 switch (csp->csp_cipher_alg) { 196 case CRYPTO_AES_NIST_GCM_16: 197 if (csp->csp_ivlen != AES_GCM_IV_LEN) 198 return EINVAL; 199 break; 200 default: 201 return EINVAL; 202 } 203 break; 204 case CSP_MODE_ETA: 205 switch (csp->csp_auth_alg) { 206 case CRYPTO_SHA1_HMAC: 207 case CRYPTO_SHA2_256_HMAC: 208 case CRYPTO_SHA2_384_HMAC: 209 case CRYPTO_SHA2_512_HMAC: 210 switch (csp->csp_cipher_alg) { 211 case CRYPTO_AES_CBC: 212 case CRYPTO_AES_ICM: 213 if (csp->csp_ivlen != AES_BLOCK_LEN) 214 return EINVAL; 215 break; 216 case CRYPTO_AES_XTS: 217 if (csp->csp_ivlen != AES_XTS_IV_LEN) 218 return EINVAL; 219 break; 220 default: 221 return EINVAL; 222 } 223 break; 224 default: 225 return EINVAL; 226 } 227 break; 228 default: 229 return EINVAL; 230 } 231 232 return CRYPTODEV_PROBE_HARDWARE; 233} 234 235static CpaStatus 236qat_ocf_session_init(device_t dev, 237 struct cryptop *crp, 238 struct qat_ocf_instance *qat_instance, 239 struct qat_ocf_session *qat_ssession) 240{ 241 CpaStatus status = CPA_STATUS_SUCCESS; 242 /* Crytpodev structures */ 243 crypto_session_t cses; 244 const struct crypto_session_params *csp; 245 /* DP API Session configuration */ 246 CpaCySymSessionSetupData sessionSetupData = { 0 }; 247 CpaCySymSessionCtx sessionCtx = NULL; 248 Cpa32U sessionCtxSize = 0; 249 250 cses = crp->crp_session; 251 if (NULL == cses) { 252 device_printf(dev, "no crypto session in cryptodev request\n"); 253 return CPA_STATUS_FAIL; 254 } 255 256 csp = crypto_get_params(cses); 257 if (NULL == csp) { 258 device_printf(dev, "no session in cryptodev session\n"); 259 return CPA_STATUS_FAIL; 260 } 261 262 /* Common fields */ 263 sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH; 264 /* Cipher key */ 265 if (crp->crp_cipher_key) 266 sessionSetupData.cipherSetupData.pCipherKey = 267 crp->crp_cipher_key; 268 else 269 sessionSetupData.cipherSetupData.pCipherKey = 270 csp->csp_cipher_key; 271 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 272 csp->csp_cipher_klen; 273 274 /* Auth key */ 275 if (crp->crp_auth_key) 276 sessionSetupData.hashSetupData.authModeSetupData.authKey = 277 crp->crp_auth_key; 278 else 279 sessionSetupData.hashSetupData.authModeSetupData.authKey = 280 csp->csp_auth_key; 281 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = 282 csp->csp_auth_klen; 283 284 qat_ssession->aadLen = crp->crp_aad_length; 285 if (CPA_TRUE == is_sep_aad_supported(csp)) 286 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 287 crp->crp_aad_length; 288 else 289 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 290 0; 291 292 /* Just setup algorithm - regardless of mode */ 293 if (csp->csp_cipher_alg) { 294 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 295 296 switch (csp->csp_cipher_alg) { 297 case CRYPTO_AES_CBC: 298 sessionSetupData.cipherSetupData.cipherAlgorithm = 299 CPA_CY_SYM_CIPHER_AES_CBC; 300 break; 301 case CRYPTO_AES_ICM: 302 sessionSetupData.cipherSetupData.cipherAlgorithm = 303 CPA_CY_SYM_CIPHER_AES_CTR; 304 break; 305 case CRYPTO_AES_XTS: 306 sessionSetupData.cipherSetupData.cipherAlgorithm = 307 CPA_CY_SYM_CIPHER_AES_XTS; 308 break; 309 case CRYPTO_AES_NIST_GCM_16: 310 sessionSetupData.cipherSetupData.cipherAlgorithm = 311 CPA_CY_SYM_CIPHER_AES_GCM; 312 sessionSetupData.hashSetupData.hashAlgorithm = 313 CPA_CY_SYM_HASH_AES_GCM; 314 sessionSetupData.hashSetupData.hashMode = 315 CPA_CY_SYM_HASH_MODE_AUTH; 316 break; 317 default: 318 device_printf(dev, 319 "cipher_alg: %d not supported\n", 320 csp->csp_cipher_alg); 321 status = CPA_STATUS_UNSUPPORTED; 322 goto fail; 323 } 324 } 325 326 if (csp->csp_auth_alg) { 327 switch (csp->csp_auth_alg) { 328 case CRYPTO_SHA1_HMAC: 329 sessionSetupData.hashSetupData.hashAlgorithm = 330 CPA_CY_SYM_HASH_SHA1; 331 sessionSetupData.hashSetupData.hashMode = 332 CPA_CY_SYM_HASH_MODE_AUTH; 333 break; 334 case CRYPTO_SHA1: 335 sessionSetupData.hashSetupData.hashAlgorithm = 336 CPA_CY_SYM_HASH_SHA1; 337 sessionSetupData.hashSetupData.hashMode = 338 CPA_CY_SYM_HASH_MODE_PLAIN; 339 break; 340 341 case CRYPTO_SHA2_256_HMAC: 342 sessionSetupData.hashSetupData.hashAlgorithm = 343 CPA_CY_SYM_HASH_SHA256; 344 sessionSetupData.hashSetupData.hashMode = 345 CPA_CY_SYM_HASH_MODE_AUTH; 346 break; 347 case CRYPTO_SHA2_256: 348 sessionSetupData.hashSetupData.hashAlgorithm = 349 CPA_CY_SYM_HASH_SHA256; 350 sessionSetupData.hashSetupData.hashMode = 351 CPA_CY_SYM_HASH_MODE_PLAIN; 352 break; 353 354 case CRYPTO_SHA2_224_HMAC: 355 sessionSetupData.hashSetupData.hashAlgorithm = 356 CPA_CY_SYM_HASH_SHA224; 357 sessionSetupData.hashSetupData.hashMode = 358 CPA_CY_SYM_HASH_MODE_AUTH; 359 break; 360 case CRYPTO_SHA2_224: 361 sessionSetupData.hashSetupData.hashAlgorithm = 362 CPA_CY_SYM_HASH_SHA224; 363 sessionSetupData.hashSetupData.hashMode = 364 CPA_CY_SYM_HASH_MODE_PLAIN; 365 break; 366 367 case CRYPTO_SHA2_384_HMAC: 368 sessionSetupData.hashSetupData.hashAlgorithm = 369 CPA_CY_SYM_HASH_SHA384; 370 sessionSetupData.hashSetupData.hashMode = 371 CPA_CY_SYM_HASH_MODE_AUTH; 372 break; 373 case CRYPTO_SHA2_384: 374 sessionSetupData.hashSetupData.hashAlgorithm = 375 CPA_CY_SYM_HASH_SHA384; 376 sessionSetupData.hashSetupData.hashMode = 377 CPA_CY_SYM_HASH_MODE_PLAIN; 378 break; 379 380 case CRYPTO_SHA2_512_HMAC: 381 sessionSetupData.hashSetupData.hashAlgorithm = 382 CPA_CY_SYM_HASH_SHA512; 383 sessionSetupData.hashSetupData.hashMode = 384 CPA_CY_SYM_HASH_MODE_AUTH; 385 break; 386 case CRYPTO_SHA2_512: 387 sessionSetupData.hashSetupData.hashAlgorithm = 388 CPA_CY_SYM_HASH_SHA512; 389 sessionSetupData.hashSetupData.hashMode = 390 CPA_CY_SYM_HASH_MODE_PLAIN; 391 break; 392 case CRYPTO_AES_NIST_GMAC: 393 sessionSetupData.hashSetupData.hashAlgorithm = 394 CPA_CY_SYM_HASH_AES_GMAC; 395 break; 396 default: 397 status = CPA_STATUS_UNSUPPORTED; 398 goto fail; 399 } 400 } /* csp->csp_auth_alg */ 401 402 /* Setting digest-length if no cipher-only mode is set */ 403 if (csp->csp_mode != CSP_MODE_CIPHER) { 404 lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL; 405 if (csp->csp_auth_mlen) { 406 sessionSetupData.hashSetupData.digestResultLenInBytes = 407 csp->csp_auth_mlen; 408 qat_ssession->authLen = csp->csp_auth_mlen; 409 } else { 410 LacSymQat_HashDefsLookupGet( 411 qat_instance->cyInstHandle, 412 sessionSetupData.hashSetupData.hashAlgorithm, 413 &pHashDefsInfo); 414 if (NULL == pHashDefsInfo) { 415 device_printf( 416 dev, 417 "unable to find corresponding hash data\n"); 418 status = CPA_STATUS_UNSUPPORTED; 419 goto fail; 420 } 421 sessionSetupData.hashSetupData.digestResultLenInBytes = 422 pHashDefsInfo->algInfo->digestLength; 423 qat_ssession->authLen = 424 pHashDefsInfo->algInfo->digestLength; 425 } 426 sessionSetupData.verifyDigest = CPA_FALSE; 427 } 428 429 switch (csp->csp_mode) { 430 case CSP_MODE_AEAD: 431 case CSP_MODE_ETA: 432 sessionSetupData.symOperation = 433 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 434 /* Place the digest result in a buffer unrelated to srcBuffer */ 435 sessionSetupData.digestIsAppended = CPA_FALSE; 436 /* Due to FW limitation to verify only appended MACs */ 437 sessionSetupData.verifyDigest = CPA_FALSE; 438 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 439 sessionSetupData.cipherSetupData.cipherDirection = 440 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 441 sessionSetupData.algChainOrder = 442 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 443 } else { 444 sessionSetupData.cipherSetupData.cipherDirection = 445 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 446 sessionSetupData.algChainOrder = 447 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 448 } 449 break; 450 case CSP_MODE_CIPHER: 451 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 452 sessionSetupData.cipherSetupData.cipherDirection = 453 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 454 } else { 455 sessionSetupData.cipherSetupData.cipherDirection = 456 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 457 } 458 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 459 break; 460 case CSP_MODE_DIGEST: 461 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 462 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { 463 sessionSetupData.symOperation = 464 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 465 /* GMAC is always encrypt */ 466 sessionSetupData.cipherSetupData.cipherDirection = 467 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 468 sessionSetupData.algChainOrder = 469 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 470 sessionSetupData.cipherSetupData.cipherAlgorithm = 471 CPA_CY_SYM_CIPHER_AES_GCM; 472 sessionSetupData.hashSetupData.hashAlgorithm = 473 CPA_CY_SYM_HASH_AES_GMAC; 474 sessionSetupData.hashSetupData.hashMode = 475 CPA_CY_SYM_HASH_MODE_AUTH; 476 /* Same key for cipher and auth */ 477 sessionSetupData.cipherSetupData.pCipherKey = 478 csp->csp_auth_key; 479 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 480 csp->csp_auth_klen; 481 /* Generated GMAC stored in separated buffer */ 482 sessionSetupData.digestIsAppended = CPA_FALSE; 483 /* Digest verification not allowed in GMAC case */ 484 sessionSetupData.verifyDigest = CPA_FALSE; 485 /* No AAD allowed */ 486 sessionSetupData.hashSetupData.authModeSetupData 487 .aadLenInBytes = 0; 488 } else { 489 sessionSetupData.cipherSetupData.cipherDirection = 490 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 491 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 492 sessionSetupData.digestIsAppended = CPA_FALSE; 493 } 494 break; 495 default: 496 device_printf(dev, 497 "%s: unhandled crypto algorithm %d, %d\n", 498 __func__, 499 csp->csp_cipher_alg, 500 csp->csp_auth_alg); 501 status = CPA_STATUS_FAIL; 502 goto fail; 503 } 504 505 /* Extracting session size */ 506 status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle, 507 &sessionSetupData, 508 &sessionCtxSize); 509 if (CPA_STATUS_SUCCESS != status) { 510 device_printf(dev, "unable to get session size\n"); 511 goto fail; 512 } 513 514 /* Allocating contiguous memory for session */ 515 sessionCtx = contigmalloc(sessionCtxSize, 516 M_QAT_OCF, 517 M_NOWAIT, 518 0, 519 ~1UL, 520 1 << (ilog2(sessionCtxSize - 1) + 1), 521 0); 522 if (NULL == sessionCtx) { 523 device_printf(dev, "unable to allocate memory for session\n"); 524 status = CPA_STATUS_RESOURCE; 525 goto fail; 526 } 527 528 status = cpaCySymDpInitSession(qat_instance->cyInstHandle, 529 &sessionSetupData, 530 sessionCtx); 531 if (CPA_STATUS_SUCCESS != status) { 532 device_printf(dev, "session initialization failed\n"); 533 goto fail; 534 } 535 536 /* NOTE: lets keep double session (both directions) approach to overcome 537 * lack of direction update in FBSD QAT. 538 */ 539 qat_ssession->sessionCtx = sessionCtx; 540 qat_ssession->sessionCtxSize = sessionCtxSize; 541 542 return CPA_STATUS_SUCCESS; 543 544fail: 545 /* Release resources if any */ 546 if (sessionCtx) 547 contigfree(sessionCtx, sessionCtxSize, M_QAT_OCF); 548 549 return status; 550} 551 552static int 553qat_ocf_newsession(device_t dev, 554 crypto_session_t cses, 555 const struct crypto_session_params *csp) 556{ 557 /* Cryptodev QAT structures */ 558 struct qat_ocf_softc *qat_softc; 559 struct qat_ocf_dsession *qat_dsession; 560 struct qat_ocf_instance *qat_instance; 561 u_int cpu_id = PCPU_GET(cpuid); 562 563 /* Create cryptodev session */ 564 qat_softc = device_get_softc(dev); 565 if (qat_softc->numCyInstances > 0) { 566 qat_instance = 567 &qat_softc 568 ->cyInstHandles[cpu_id % qat_softc->numCyInstances]; 569 qat_dsession = crypto_get_driver_session(cses); 570 if (NULL == qat_dsession) { 571 device_printf(dev, "Unable to create new session\n"); 572 return (EINVAL); 573 } 574 575 /* Add only instance at this point remaining operations moved to 576 * lazy session init */ 577 qat_dsession->qatInstance = qat_instance; 578 } else { 579 return ENXIO; 580 } 581 582 return 0; 583} 584 585static CpaStatus 586qat_ocf_remove_session(device_t dev, 587 CpaInstanceHandle cyInstHandle, 588 struct qat_ocf_session *qat_session) 589{ 590 CpaStatus status = CPA_STATUS_SUCCESS; 591 592 if (NULL == qat_session->sessionCtx) 593 return CPA_STATUS_SUCCESS; 594 595 /* User callback is executed right before decrementing pending 596 * callback atomic counter. To avoid removing session rejection 597 * we have to wait a very short while for counter update 598 * after call back execution. */ 599 status = qat_ocf_wait_for_session(qat_session->sessionCtx, 600 QAT_OCF_SESSION_WAIT_TIMEOUT_MS); 601 if (CPA_STATUS_SUCCESS != status) { 602 device_printf(dev, "waiting for session un-busy failed\n"); 603 return CPA_STATUS_FAIL; 604 } 605 606 status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx); 607 if (CPA_STATUS_SUCCESS != status) { 608 device_printf(dev, "error while removing session\n"); 609 return CPA_STATUS_FAIL; 610 } 611 612 explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize); 613 contigfree(qat_session->sessionCtx, 614 qat_session->sessionCtxSize, 615 M_QAT_OCF); 616 qat_session->sessionCtx = NULL; 617 qat_session->sessionCtxSize = 0; 618 619 return CPA_STATUS_SUCCESS; 620} 621 622static void 623qat_ocf_freesession(device_t dev, crypto_session_t cses) 624{ 625 CpaStatus status = CPA_STATUS_SUCCESS; 626 struct qat_ocf_dsession *qat_dsession = NULL; 627 struct qat_ocf_instance *qat_instance = NULL; 628 629 qat_dsession = crypto_get_driver_session(cses); 630 qat_instance = qat_dsession->qatInstance; 631 mtx_lock(&qat_instance->cyInstMtx); 632 status = qat_ocf_remove_session(dev, 633 qat_dsession->qatInstance->cyInstHandle, 634 &qat_dsession->encSession); 635 if (CPA_STATUS_SUCCESS != status) 636 device_printf(dev, "unable to remove encrypt session\n"); 637 status = qat_ocf_remove_session(dev, 638 qat_dsession->qatInstance->cyInstHandle, 639 &qat_dsession->decSession); 640 if (CPA_STATUS_SUCCESS != status) 641 device_printf(dev, "unable to remove decrypt session\n"); 642 mtx_unlock(&qat_instance->cyInstMtx); 643} 644 645/* QAT GCM/CCM FW API are only algorithms which support separated AAD. */ 646static CpaStatus 647qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 648{ 649 CpaCySymDpOpData *pOpData; 650 651 pOpData = &qat_cookie->pOpdata; 652 653 if (NULL != crp->crp_aad) 654 memcpy(qat_cookie->qat_ocf_gcm_aad, 655 crp->crp_aad, 656 crp->crp_aad_length); 657 else 658 crypto_copydata(crp, 659 crp->crp_aad_start, 660 crp->crp_aad_length, 661 qat_cookie->qat_ocf_gcm_aad); 662 663 pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad; 664 pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr; 665 666 return CPA_STATUS_SUCCESS; 667} 668 669static CpaStatus 670qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 671{ 672 CpaStatus status = CPA_STATUS_SUCCESS; 673 const struct crypto_session_params *csp; 674 CpaCySymDpOpData *pOpData; 675 struct qat_ocf_load_cb_arg args; 676 677 pOpData = &qat_cookie->pOpdata; 678 pOpData->pAdditionalAuthData = NULL; 679 pOpData->additionalAuthData = 0UL; 680 681 if (crp->crp_aad_length == 0) 682 return CPA_STATUS_SUCCESS; 683 684 if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX) 685 return CPA_STATUS_FAIL; 686 687 csp = crypto_get_params(crp->crp_session); 688 689 /* Handle GCM/CCM case */ 690 if (CPA_TRUE == is_sep_aad_supported(csp)) 691 return qat_ocf_load_aad_gcm(crp, qat_cookie); 692 693 if (NULL == crp->crp_aad) { 694 /* AAD already embedded in source buffer */ 695 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 696 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 697 698 pOpData->messageLenToHashInBytes = 699 crp->crp_aad_length + crp->crp_payload_length; 700 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 701 702 return CPA_STATUS_SUCCESS; 703 } 704 705 /* Separated AAD not supported by QAT - lets place the content 706 * of ADD buffer at the very beginning of source SGL */ 707 args.crp_op = crp; 708 args.qat_cookie = qat_cookie; 709 args.pOpData = pOpData; 710 args.error = 0; 711 status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag, 712 qat_cookie->gcm_aad_dma_mem.dma_map, 713 crp->crp_aad, 714 crp->crp_aad_length, 715 qat_ocf_crypto_load_aadbuf_cb, 716 &args, 717 BUS_DMA_NOWAIT); 718 qat_cookie->is_sep_aad_used = CPA_TRUE; 719 720 /* Right after this step we have AAD placed in the first flat buffer 721 * in source SGL */ 722 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 723 pOpData->cryptoStartSrcOffsetInBytes = 724 crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start; 725 726 pOpData->messageLenToHashInBytes = 727 crp->crp_aad_length + crp->crp_payload_length; 728 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 729 730 return status; 731} 732 733static CpaStatus 734qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 735{ 736 CpaStatus status = CPA_STATUS_SUCCESS; 737 CpaCySymDpOpData *pOpData; 738 struct qat_ocf_load_cb_arg args; 739 /* cryptodev internals */ 740 const struct crypto_session_params *csp; 741 742 pOpData = &qat_cookie->pOpdata; 743 744 csp = crypto_get_params(crp->crp_session); 745 746 /* Load IV buffer if present */ 747 if (csp->csp_ivlen > 0) { 748 memset(qat_cookie->qat_ocf_iv_buf, 749 0, 750 sizeof(qat_cookie->qat_ocf_iv_buf)); 751 crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf); 752 pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr; 753 pOpData->pIv = qat_cookie->qat_ocf_iv_buf; 754 pOpData->ivLenInBytes = csp->csp_ivlen; 755 } 756 757 /* GCM/CCM - load AAD to separated buffer 758 * AES+SHA - load AAD to first flat in SGL */ 759 status = qat_ocf_load_aad(crp, qat_cookie); 760 if (CPA_STATUS_SUCCESS != status) 761 goto fail; 762 763 /* Load source buffer */ 764 args.crp_op = crp; 765 args.qat_cookie = qat_cookie; 766 args.pOpData = pOpData; 767 args.error = 0; 768 status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag, 769 qat_cookie->src_dma_mem.dma_map, 770 &crp->crp_buf, 771 qat_ocf_crypto_load_buf_cb, 772 &args, 773 BUS_DMA_NOWAIT); 774 if (CPA_STATUS_SUCCESS != status) 775 goto fail; 776 pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr; 777 pOpData->srcBufferLen = CPA_DP_BUFLIST; 778 779 /* Load destination buffer */ 780 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 781 status = 782 bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag, 783 qat_cookie->dst_dma_mem.dma_map, 784 &crp->crp_obuf, 785 qat_ocf_crypto_load_obuf_cb, 786 &args, 787 BUS_DMA_NOWAIT); 788 if (CPA_STATUS_SUCCESS != status) 789 goto fail; 790 pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr; 791 pOpData->dstBufferLen = CPA_DP_BUFLIST; 792 } else { 793 pOpData->dstBuffer = pOpData->srcBuffer; 794 pOpData->dstBufferLen = pOpData->srcBufferLen; 795 } 796 797 if (CPA_TRUE == is_use_sep_digest(csp)) 798 pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr; 799 else 800 pOpData->digestResult = 0UL; 801 802 /* GMAC - aka zero length buffer */ 803 if (CPA_TRUE == is_gmac_exception(csp)) 804 pOpData->messageLenToCipherInBytes = 0; 805 806fail: 807 return status; 808} 809 810static int 811qat_ocf_check_input(device_t dev, struct cryptop *crp) 812{ 813 const struct crypto_session_params *csp; 814 csp = crypto_get_params(crp->crp_session); 815 816 if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN) 817 return E2BIG; 818 819 if (CPA_TRUE == is_sep_aad_supported(csp) && 820 (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)) 821 return EBADMSG; 822 823 return 0; 824} 825 826static int 827qat_ocf_process(device_t dev, struct cryptop *crp, int hint) 828{ 829 CpaStatus status = CPA_STATUS_SUCCESS; 830 int rc = 0; 831 struct qat_ocf_dsession *qat_dsession = NULL; 832 struct qat_ocf_session *qat_session = NULL; 833 struct qat_ocf_instance *qat_instance = NULL; 834 CpaCySymDpOpData *pOpData = NULL; 835 struct qat_ocf_cookie *qat_cookie = NULL; 836 CpaBoolean memLoaded = CPA_FALSE; 837 838 rc = qat_ocf_check_input(dev, crp); 839 if (rc) 840 goto fail; 841 842 qat_dsession = crypto_get_driver_session(crp->crp_session); 843 844 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 845 qat_session = &qat_dsession->encSession; 846 else 847 qat_session = &qat_dsession->decSession; 848 qat_instance = qat_dsession->qatInstance; 849 850 status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie); 851 if (CPA_STATUS_SUCCESS != status) { 852 rc = EAGAIN; 853 goto fail; 854 } 855 856 qat_cookie->crp_op = crp; 857 858 /* Common request fields */ 859 pOpData = &qat_cookie->pOpdata; 860 pOpData->instanceHandle = qat_instance->cyInstHandle; 861 pOpData->sessionCtx = NULL; 862 863 /* Cipher fields */ 864 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 865 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 866 /* Digest fields - any exceptions from this basic rules are covered 867 * in qat_ocf_load */ 868 pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start; 869 pOpData->messageLenToHashInBytes = crp->crp_payload_length; 870 871 status = qat_ocf_load(crp, qat_cookie); 872 if (CPA_STATUS_SUCCESS != status) { 873 device_printf(dev, 874 "unable to load OCF buffers to QAT DMA " 875 "transaction\n"); 876 rc = EIO; 877 goto fail; 878 } 879 memLoaded = CPA_TRUE; 880 881 status = qat_ocf_cookie_dma_pre_sync(crp, pOpData); 882 if (CPA_STATUS_SUCCESS != status) { 883 device_printf(dev, "unable to sync DMA buffers\n"); 884 rc = EIO; 885 goto fail; 886 } 887 888 mtx_lock(&qat_instance->cyInstMtx); 889 /* Session initialization at the first request. It's done 890 * in such way to overcome missing QAT specific session data 891 * such like AAD length and limited possibility to update 892 * QAT session while handling traffic. 893 */ 894 if (NULL == qat_session->sessionCtx) { 895 status = 896 qat_ocf_session_init(dev, crp, qat_instance, qat_session); 897 if (CPA_STATUS_SUCCESS != status) { 898 mtx_unlock(&qat_instance->cyInstMtx); 899 device_printf(dev, "unable to init session\n"); 900 rc = EIO; 901 goto fail; 902 } 903 } else { 904 status = qat_ocf_handle_session_update(qat_dsession, crp); 905 if (CPA_STATUS_RESOURCE == status) { 906 mtx_unlock(&qat_instance->cyInstMtx); 907 rc = EAGAIN; 908 goto fail; 909 } else if (CPA_STATUS_SUCCESS != status) { 910 mtx_unlock(&qat_instance->cyInstMtx); 911 rc = EIO; 912 goto fail; 913 } 914 } 915 pOpData->sessionCtx = qat_session->sessionCtx; 916 status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE); 917 mtx_unlock(&qat_instance->cyInstMtx); 918 if (CPA_STATUS_SUCCESS != status) { 919 if (CPA_STATUS_RETRY == status) { 920 rc = EAGAIN; 921 goto fail; 922 } 923 device_printf(dev, 924 "unable to send request. Status: %d\n", 925 status); 926 rc = EIO; 927 goto fail; 928 } 929 930 return 0; 931fail: 932 if (qat_cookie) { 933 if (memLoaded) 934 qat_ocf_cookie_dma_unload(crp, pOpData); 935 qat_ocf_cookie_free(qat_instance, qat_cookie); 936 } 937 crp->crp_etype = rc; 938 crypto_done(crp); 939 940 return 0; 941} 942 943static void 944qat_ocf_identify(driver_t *drv, device_t parent) 945{ 946 if (device_find_child(parent, "qat_ocf", -1) == NULL && 947 BUS_ADD_CHILD(parent, 200, "qat_ocf", -1) == 0) 948 device_printf(parent, "qat_ocf: could not attach!"); 949} 950 951static int 952qat_ocf_probe(device_t dev) 953{ 954 device_set_desc(dev, "QAT engine"); 955 return (BUS_PROBE_NOWILDCARD); 956} 957 958static CpaStatus 959qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles, 960 Cpa16U cyInstHandlesSize, 961 Cpa16U *foundInstances) 962{ 963 CpaStatus status = CPA_STATUS_SUCCESS; 964 icp_accel_dev_t **pAdfInsts = NULL; 965 icp_accel_dev_t *dev_addr = NULL; 966 sal_t *baseAddr = NULL; 967 sal_list_t *listTemp = NULL; 968 CpaInstanceHandle cyInstHandle; 969 CpaInstanceInfo2 info; 970 Cpa16U numDevices; 971 Cpa32U instCtr = 0; 972 Cpa32U i; 973 974 /* Get the number of devices */ 975 status = icp_amgr_getNumInstances(&numDevices); 976 if (CPA_STATUS_SUCCESS != status) 977 return status; 978 979 /* Allocate memory to store addr of accel_devs */ 980 pAdfInsts = 981 malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK); 982 983 /* Get ADF to return all accel_devs that support either 984 * symmetric or asymmetric crypto */ 985 status = icp_amgr_getAllAccelDevByCapabilities( 986 (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices); 987 if (CPA_STATUS_SUCCESS != status) { 988 free(pAdfInsts, M_QAT_OCF); 989 return status; 990 } 991 992 for (i = 0; i < numDevices; i++) { 993 dev_addr = (icp_accel_dev_t *)pAdfInsts[i]; 994 baseAddr = dev_addr->pSalHandle; 995 if (NULL == baseAddr) 996 continue; 997 listTemp = baseAddr->sym_services; 998 if (NULL == listTemp) { 999 listTemp = baseAddr->crypto_services; 1000 } 1001 1002 while (NULL != listTemp) { 1003 cyInstHandle = SalList_getObject(listTemp); 1004 status = cpaCyInstanceGetInfo2(cyInstHandle, &info); 1005 if (CPA_STATUS_SUCCESS != status) 1006 continue; 1007 listTemp = SalList_next(listTemp); 1008 if (CPA_TRUE == info.isPolled) 1009 continue; 1010 if (instCtr >= cyInstHandlesSize) 1011 break; 1012 cyInstHandles[instCtr++] = cyInstHandle; 1013 } 1014 } 1015 free(pAdfInsts, M_QAT_OCF); 1016 *foundInstances = instCtr; 1017 1018 return CPA_STATUS_SUCCESS; 1019} 1020 1021static CpaStatus 1022qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev) 1023{ 1024 CpaStatus status = CPA_STATUS_SUCCESS; 1025 Cpa16U numInstances = 0; 1026 CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 }; 1027 CpaInstanceHandle cyInstHandle = NULL; 1028 Cpa32U startedInstances = 0; 1029 Cpa32U i; 1030 1031 qat_softc->numCyInstances = 0; 1032 status = qat_ocf_get_irq_instances(cyInstHandles, 1033 QAT_OCF_MAX_INSTANCES, 1034 &numInstances); 1035 if (CPA_STATUS_SUCCESS != status) 1036 return status; 1037 1038 for (i = 0; i < numInstances; i++) { 1039 struct qat_ocf_instance *qat_ocf_instance; 1040 1041 cyInstHandle = cyInstHandles[i]; 1042 if (!cyInstHandle) 1043 continue; 1044 1045 /* Starting instance */ 1046 status = cpaCyStartInstance(cyInstHandle); 1047 if (CPA_STATUS_SUCCESS != status) { 1048 device_printf(qat_softc->sc_dev, 1049 "unable to get start instance\n"); 1050 continue; 1051 } 1052 1053 qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances]; 1054 qat_ocf_instance->cyInstHandle = cyInstHandle; 1055 mtx_init(&qat_ocf_instance->cyInstMtx, 1056 "Instance MTX", 1057 NULL, 1058 MTX_DEF); 1059 1060 status = 1061 cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys); 1062 if (CPA_STATUS_SUCCESS != status) { 1063 device_printf(qat_softc->sc_dev, 1064 "unable to add virt to phys callback\n"); 1065 goto fail; 1066 } 1067 1068 status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback); 1069 if (CPA_STATUS_SUCCESS != status) { 1070 device_printf(qat_softc->sc_dev, 1071 "unable to add user callback\n"); 1072 goto fail; 1073 } 1074 1075 /* Initialize cookie pool */ 1076 status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev); 1077 if (CPA_STATUS_SUCCESS != status) { 1078 device_printf(qat_softc->sc_dev, 1079 "unable to create cookie pool\n"); 1080 goto fail; 1081 } 1082 1083 /* Disable forcing HW MAC validation for AEAD */ 1084 status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE); 1085 if (CPA_STATUS_SUCCESS != status) { 1086 device_printf( 1087 qat_softc->sc_dev, 1088 "unable to disable AEAD HW MAC verification\n"); 1089 goto fail; 1090 } 1091 1092 qat_ocf_instance->driver_id = qat_softc->cryptodev_id; 1093 1094 startedInstances++; 1095 continue; 1096 fail: 1097 mtx_destroy(&qat_ocf_instance->cyInstMtx); 1098 1099 /* Stop instance */ 1100 status = cpaCyStopInstance(cyInstHandle); 1101 if (CPA_STATUS_SUCCESS != status) 1102 device_printf(qat_softc->sc_dev, 1103 "unable to stop the instance\n"); 1104 } 1105 qat_softc->numCyInstances = startedInstances; 1106 1107 return CPA_STATUS_SUCCESS; 1108} 1109 1110static CpaStatus 1111qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc) 1112{ 1113 CpaStatus status = CPA_STATUS_SUCCESS; 1114 int i; 1115 1116 for (i = 0; i < qat_softc->numCyInstances; i++) { 1117 struct qat_ocf_instance *qat_instance; 1118 1119 qat_instance = &qat_softc->cyInstHandles[i]; 1120 status = cpaCyStopInstance(qat_instance->cyInstHandle); 1121 if (CPA_STATUS_SUCCESS != status) { 1122 pr_err("QAT: stopping instance id: %d failed\n", i); 1123 continue; 1124 } 1125 qat_ocf_cookie_pool_deinit(qat_instance); 1126 mtx_destroy(&qat_instance->cyInstMtx); 1127 } 1128 1129 qat_softc->numCyInstances = 0; 1130 1131 return status; 1132} 1133 1134static int 1135qat_ocf_deinit(struct qat_ocf_softc *qat_softc) 1136{ 1137 int status = 0; 1138 CpaStatus cpaStatus; 1139 1140 if (qat_softc->cryptodev_id >= 0) { 1141 crypto_unregister_all(qat_softc->cryptodev_id); 1142 qat_softc->cryptodev_id = -1; 1143 } 1144 1145 /* Stop QAT instances */ 1146 cpaStatus = qat_ocf_stop_instances(qat_softc); 1147 if (CPA_STATUS_SUCCESS != cpaStatus) { 1148 device_printf(qat_softc->sc_dev, "unable to stop instances\n"); 1149 status = EIO; 1150 } 1151 1152 return status; 1153} 1154 1155static int 1156qat_ocf_init(struct qat_ocf_softc *qat_softc) 1157{ 1158 int32_t cryptodev_id; 1159 1160 /* Starting instances for OCF */ 1161 if (qat_ocf_start_instances(qat_softc, qat_softc->sc_dev)) { 1162 device_printf(qat_softc->sc_dev, 1163 "unable to get QAT IRQ instances\n"); 1164 goto fail; 1165 } 1166 1167 /* Register only if instances available */ 1168 if (qat_softc->numCyInstances) { 1169 cryptodev_id = 1170 crypto_get_driverid(qat_softc->sc_dev, 1171 sizeof(struct qat_ocf_dsession), 1172 CRYPTOCAP_F_HARDWARE); 1173 if (cryptodev_id < 0) { 1174 device_printf(qat_softc->sc_dev, 1175 "cannot initialize!\n"); 1176 goto fail; 1177 } 1178 qat_softc->cryptodev_id = cryptodev_id; 1179 } 1180 1181 return 0; 1182fail: 1183 qat_ocf_deinit(qat_softc); 1184 1185 return ENXIO; 1186} 1187 1188static int qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS) 1189{ 1190 struct qat_ocf_softc *qat_softc = NULL; 1191 int ret = 0; 1192 device_t dev = arg1; 1193 u_int enabled; 1194 1195 qat_softc = device_get_softc(dev); 1196 enabled = qat_softc->enabled; 1197 1198 ret = sysctl_handle_int(oidp, &enabled, 0, req); 1199 if (ret || !req->newptr) 1200 return (ret); 1201 1202 if (qat_softc->enabled != enabled) { 1203 if (enabled) { 1204 ret = qat_ocf_init(qat_softc); 1205 1206 } else { 1207 ret = qat_ocf_deinit(qat_softc); 1208 } 1209 1210 if (!ret) 1211 qat_softc->enabled = enabled; 1212 } 1213 1214 return ret; 1215} 1216 1217static int 1218qat_ocf_attach(device_t dev) 1219{ 1220 int status; 1221 struct qat_ocf_softc *qat_softc; 1222 1223 qat_softc = device_get_softc(dev); 1224 qat_softc->sc_dev = dev; 1225 qat_softc->cryptodev_id = -1; 1226 qat_softc->enabled = 1; 1227 1228 qat_softc->rc = 1229 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 1230 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 1231 OID_AUTO, 1232 "enable", 1233 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 1234 dev, 1235 0, 1236 qat_ocf_sysctl_handle, 1237 "I", 1238 "QAT OCF support enablement"); 1239 1240 if (!qat_softc->rc) 1241 return ENOMEM; 1242 if (qat_softc->enabled) { 1243 status = qat_ocf_init(qat_softc); 1244 if (status) { 1245 device_printf(dev, "qat_ocf init failed\n"); 1246 goto fail; 1247 } 1248 } 1249 1250 return 0; 1251fail: 1252 qat_ocf_deinit(qat_softc); 1253 1254 return (ENXIO); 1255} 1256 1257static int 1258qat_ocf_detach(device_t dev) 1259{ 1260 struct qat_ocf_softc *qat_softc = device_get_softc(dev); 1261 1262 return qat_ocf_deinit(qat_softc); 1263} 1264 1265static device_method_t qat_ocf_methods[] = 1266 { DEVMETHOD(device_identify, qat_ocf_identify), 1267 DEVMETHOD(device_probe, qat_ocf_probe), 1268 DEVMETHOD(device_attach, qat_ocf_attach), 1269 DEVMETHOD(device_detach, qat_ocf_detach), 1270 1271 /* Cryptodev interface */ 1272 DEVMETHOD(cryptodev_probesession, qat_ocf_probesession), 1273 DEVMETHOD(cryptodev_newsession, qat_ocf_newsession), 1274 DEVMETHOD(cryptodev_freesession, qat_ocf_freesession), 1275 DEVMETHOD(cryptodev_process, qat_ocf_process), 1276 1277 DEVMETHOD_END }; 1278 1279static driver_t qat_ocf_driver = { 1280 .name = "qat_ocf", 1281 .methods = qat_ocf_methods, 1282 .size = sizeof(struct qat_ocf_softc), 1283}; 1284 1285 1286DRIVER_MODULE_ORDERED(qat, 1287 nexus, 1288 qat_ocf_driver, 1289 NULL, 1290 NULL, 1291 SI_ORDER_ANY); 1292MODULE_VERSION(qat, 1); 1293MODULE_DEPEND(qat, qat_c62x, 1, 1, 1); 1294MODULE_DEPEND(qat, qat_200xx, 1, 1, 1); 1295MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1); 1296MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1); 1297MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1); 1298MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1); 1299MODULE_DEPEND(qat, crypto, 1, 1, 1); 1300MODULE_DEPEND(qat, qat_common, 1, 1, 1); 1301MODULE_DEPEND(qat, qat_api, 1, 1, 1); 1302MODULE_DEPEND(qat, linuxkpi, 1, 1, 1); 1303