1/* SPDX-License-Identifier: BSD-3-Clause */
2/* Copyright(c) 2007-2022 Intel Corporation */
3/* System headers */
4#include <sys/param.h>
5#include <sys/systm.h>
6#include <sys/bus.h>
7#include <sys/cpu.h>
8#include <sys/kernel.h>
9#include <sys/mbuf.h>
10#include <sys/module.h>
11#include <sys/mutex.h>
12#include <sys/sysctl.h>
13
14/* Cryptodev headers */
15#include <opencrypto/cryptodev.h>
16#include "cryptodev_if.h"
17
18/* QAT specific headers */
19#include "cpa.h"
20#include "cpa_cy_im.h"
21#include "cpa_cy_sym_dp.h"
22#include "adf_accel_devices.h"
23#include "adf_common_drv.h"
24#include "lac_sym_hash_defs.h"
25#include "lac_sym_qat_hash_defs_lookup.h"
26
27/* To get only IRQ instances */
28#include "icp_accel_devices.h"
29#include "icp_adf_accel_mgr.h"
30#include "lac_sal_types.h"
31
32/* To disable AEAD HW MAC verification */
33#include "icp_sal_user.h"
34
35/* QAT OCF specific headers */
36#include "qat_ocf_mem_pool.h"
37#include "qat_ocf_utils.h"
38
39#define QAT_OCF_MAX_INSTANCES (256)
40#define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000)
41
42MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations");
43
44/* QAT OCF internal structures */
45struct qat_ocf_softc {
46	device_t sc_dev;
47	struct sysctl_oid *rc;
48	uint32_t enabled;
49	int32_t cryptodev_id;
50	struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES];
51	int32_t numCyInstances;
52};
53
54/* Function definitions */
55static void qat_ocf_freesession(device_t dev, crypto_session_t cses);
56static int qat_ocf_probesession(device_t dev,
57				const struct crypto_session_params *csp);
58static int qat_ocf_newsession(device_t dev,
59			      crypto_session_t cses,
60			      const struct crypto_session_params *csp);
61static int qat_ocf_attach(device_t dev);
62static int qat_ocf_detach(device_t dev);
63
64static void
65symDpCallback(CpaCySymDpOpData *pOpData,
66	      CpaStatus result,
67	      CpaBoolean verifyResult)
68{
69	struct qat_ocf_cookie *qat_cookie;
70	struct cryptop *crp;
71	struct qat_ocf_dsession *qat_dsession = NULL;
72	struct qat_ocf_session *qat_session = NULL;
73	struct qat_ocf_instance *qat_instance = NULL;
74	CpaStatus status;
75	int rc = 0;
76
77	qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag;
78	if (!qat_cookie)
79		return;
80
81	crp = qat_cookie->crp_op;
82
83	qat_dsession = crypto_get_driver_session(crp->crp_session);
84	qat_instance = qat_dsession->qatInstance;
85
86	status = qat_ocf_cookie_dma_post_sync(crp, pOpData);
87	if (CPA_STATUS_SUCCESS != status) {
88		rc = EIO;
89		goto exit;
90	}
91
92	status = qat_ocf_cookie_dma_unload(crp, pOpData);
93	if (CPA_STATUS_SUCCESS != status) {
94		rc = EIO;
95		goto exit;
96	}
97
98	/* Verify result */
99	if (CPA_STATUS_SUCCESS != result) {
100		rc = EBADMSG;
101		goto exit;
102	}
103
104	/* Verify digest by FW (GCM and CCM only) */
105	if (CPA_TRUE != verifyResult) {
106		rc = EBADMSG;
107		goto exit;
108	}
109
110	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
111		qat_session = &qat_dsession->encSession;
112	else
113		qat_session = &qat_dsession->decSession;
114
115	/* Copy back digest result if it's stored in separated buffer */
116	if (pOpData->digestResult && qat_session->authLen > 0) {
117		if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
118			char icv[QAT_OCF_MAX_DIGEST] = { 0 };
119			crypto_copydata(crp,
120					crp->crp_digest_start,
121					qat_session->authLen,
122					icv);
123			if (timingsafe_bcmp(icv,
124					    qat_cookie->qat_ocf_digest,
125					    qat_session->authLen) != 0) {
126				rc = EBADMSG;
127				goto exit;
128			}
129		} else {
130			crypto_copyback(crp,
131					crp->crp_digest_start,
132					qat_session->authLen,
133					qat_cookie->qat_ocf_digest);
134		}
135	}
136
137exit:
138	qat_ocf_cookie_free(qat_instance, qat_cookie);
139	crp->crp_etype = rc;
140	crypto_done(crp);
141
142	return;
143}
144
145static inline CpaPhysicalAddr
146qatVirtToPhys(void *virtAddr)
147{
148	return (CpaPhysicalAddr)vtophys(virtAddr);
149}
150
151static int
152qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp)
153{
154	if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
155	    0) {
156		return EINVAL;
157	}
158
159	switch (csp->csp_mode) {
160	case CSP_MODE_CIPHER:
161		switch (csp->csp_cipher_alg) {
162		case CRYPTO_AES_CBC:
163		case CRYPTO_AES_ICM:
164			if (csp->csp_ivlen != AES_BLOCK_LEN)
165				return EINVAL;
166			break;
167		case CRYPTO_AES_XTS:
168			if (csp->csp_ivlen != AES_XTS_IV_LEN)
169				return EINVAL;
170			break;
171		default:
172			return EINVAL;
173		}
174		break;
175	case CSP_MODE_DIGEST:
176		switch (csp->csp_auth_alg) {
177		case CRYPTO_SHA1:
178		case CRYPTO_SHA1_HMAC:
179		case CRYPTO_SHA2_256:
180		case CRYPTO_SHA2_256_HMAC:
181		case CRYPTO_SHA2_384:
182		case CRYPTO_SHA2_384_HMAC:
183		case CRYPTO_SHA2_512:
184		case CRYPTO_SHA2_512_HMAC:
185			break;
186		case CRYPTO_AES_NIST_GMAC:
187			if (csp->csp_ivlen != AES_GCM_IV_LEN)
188				return EINVAL;
189			break;
190		default:
191			return EINVAL;
192		}
193		break;
194	case CSP_MODE_AEAD:
195		switch (csp->csp_cipher_alg) {
196		case CRYPTO_AES_NIST_GCM_16:
197			if (csp->csp_ivlen != AES_GCM_IV_LEN)
198				return EINVAL;
199			break;
200		default:
201			return EINVAL;
202		}
203		break;
204	case CSP_MODE_ETA:
205		switch (csp->csp_auth_alg) {
206		case CRYPTO_SHA1_HMAC:
207		case CRYPTO_SHA2_256_HMAC:
208		case CRYPTO_SHA2_384_HMAC:
209		case CRYPTO_SHA2_512_HMAC:
210			switch (csp->csp_cipher_alg) {
211			case CRYPTO_AES_CBC:
212			case CRYPTO_AES_ICM:
213				if (csp->csp_ivlen != AES_BLOCK_LEN)
214					return EINVAL;
215				break;
216			case CRYPTO_AES_XTS:
217				if (csp->csp_ivlen != AES_XTS_IV_LEN)
218					return EINVAL;
219				break;
220			default:
221				return EINVAL;
222			}
223			break;
224		default:
225			return EINVAL;
226		}
227		break;
228	default:
229		return EINVAL;
230	}
231
232	return CRYPTODEV_PROBE_HARDWARE;
233}
234
235static CpaStatus
236qat_ocf_session_init(device_t dev,
237		     struct cryptop *crp,
238		     struct qat_ocf_instance *qat_instance,
239		     struct qat_ocf_session *qat_ssession)
240{
241	CpaStatus status = CPA_STATUS_SUCCESS;
242	/* Crytpodev structures */
243	crypto_session_t cses;
244	const struct crypto_session_params *csp;
245	/* DP API Session configuration */
246	CpaCySymSessionSetupData sessionSetupData = { 0 };
247	CpaCySymSessionCtx sessionCtx = NULL;
248	Cpa32U sessionCtxSize = 0;
249
250	cses = crp->crp_session;
251	if (NULL == cses) {
252		device_printf(dev, "no crypto session in cryptodev request\n");
253		return CPA_STATUS_FAIL;
254	}
255
256	csp = crypto_get_params(cses);
257	if (NULL == csp) {
258		device_printf(dev, "no session in cryptodev session\n");
259		return CPA_STATUS_FAIL;
260	}
261
262	/* Common fields */
263	sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
264	/* Cipher key */
265	if (crp->crp_cipher_key)
266		sessionSetupData.cipherSetupData.pCipherKey =
267		    crp->crp_cipher_key;
268	else
269		sessionSetupData.cipherSetupData.pCipherKey =
270		    csp->csp_cipher_key;
271	sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
272	    csp->csp_cipher_klen;
273
274	/* Auth key */
275	if (crp->crp_auth_key)
276		sessionSetupData.hashSetupData.authModeSetupData.authKey =
277		    crp->crp_auth_key;
278	else
279		sessionSetupData.hashSetupData.authModeSetupData.authKey =
280		    csp->csp_auth_key;
281	sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
282	    csp->csp_auth_klen;
283
284	qat_ssession->aadLen = crp->crp_aad_length;
285	if (CPA_TRUE == is_sep_aad_supported(csp))
286		sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
287		    crp->crp_aad_length;
288	else
289		sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
290		    0;
291
292	/* Just setup algorithm - regardless of mode */
293	if (csp->csp_cipher_alg) {
294		sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
295
296		switch (csp->csp_cipher_alg) {
297		case CRYPTO_AES_CBC:
298			sessionSetupData.cipherSetupData.cipherAlgorithm =
299			    CPA_CY_SYM_CIPHER_AES_CBC;
300			break;
301		case CRYPTO_AES_ICM:
302			sessionSetupData.cipherSetupData.cipherAlgorithm =
303			    CPA_CY_SYM_CIPHER_AES_CTR;
304			break;
305		case CRYPTO_AES_XTS:
306			sessionSetupData.cipherSetupData.cipherAlgorithm =
307			    CPA_CY_SYM_CIPHER_AES_XTS;
308			break;
309		case CRYPTO_AES_NIST_GCM_16:
310			sessionSetupData.cipherSetupData.cipherAlgorithm =
311			    CPA_CY_SYM_CIPHER_AES_GCM;
312			sessionSetupData.hashSetupData.hashAlgorithm =
313			    CPA_CY_SYM_HASH_AES_GCM;
314			sessionSetupData.hashSetupData.hashMode =
315			    CPA_CY_SYM_HASH_MODE_AUTH;
316			break;
317		default:
318			device_printf(dev,
319				      "cipher_alg: %d not supported\n",
320				      csp->csp_cipher_alg);
321			status = CPA_STATUS_UNSUPPORTED;
322			goto fail;
323		}
324	}
325
326	if (csp->csp_auth_alg) {
327		switch (csp->csp_auth_alg) {
328		case CRYPTO_SHA1_HMAC:
329			sessionSetupData.hashSetupData.hashAlgorithm =
330			    CPA_CY_SYM_HASH_SHA1;
331			sessionSetupData.hashSetupData.hashMode =
332			    CPA_CY_SYM_HASH_MODE_AUTH;
333			break;
334		case CRYPTO_SHA1:
335			sessionSetupData.hashSetupData.hashAlgorithm =
336			    CPA_CY_SYM_HASH_SHA1;
337			sessionSetupData.hashSetupData.hashMode =
338			    CPA_CY_SYM_HASH_MODE_PLAIN;
339			break;
340
341		case CRYPTO_SHA2_256_HMAC:
342			sessionSetupData.hashSetupData.hashAlgorithm =
343			    CPA_CY_SYM_HASH_SHA256;
344			sessionSetupData.hashSetupData.hashMode =
345			    CPA_CY_SYM_HASH_MODE_AUTH;
346			break;
347		case CRYPTO_SHA2_256:
348			sessionSetupData.hashSetupData.hashAlgorithm =
349			    CPA_CY_SYM_HASH_SHA256;
350			sessionSetupData.hashSetupData.hashMode =
351			    CPA_CY_SYM_HASH_MODE_PLAIN;
352			break;
353
354		case CRYPTO_SHA2_224_HMAC:
355			sessionSetupData.hashSetupData.hashAlgorithm =
356			    CPA_CY_SYM_HASH_SHA224;
357			sessionSetupData.hashSetupData.hashMode =
358			    CPA_CY_SYM_HASH_MODE_AUTH;
359			break;
360		case CRYPTO_SHA2_224:
361			sessionSetupData.hashSetupData.hashAlgorithm =
362			    CPA_CY_SYM_HASH_SHA224;
363			sessionSetupData.hashSetupData.hashMode =
364			    CPA_CY_SYM_HASH_MODE_PLAIN;
365			break;
366
367		case CRYPTO_SHA2_384_HMAC:
368			sessionSetupData.hashSetupData.hashAlgorithm =
369			    CPA_CY_SYM_HASH_SHA384;
370			sessionSetupData.hashSetupData.hashMode =
371			    CPA_CY_SYM_HASH_MODE_AUTH;
372			break;
373		case CRYPTO_SHA2_384:
374			sessionSetupData.hashSetupData.hashAlgorithm =
375			    CPA_CY_SYM_HASH_SHA384;
376			sessionSetupData.hashSetupData.hashMode =
377			    CPA_CY_SYM_HASH_MODE_PLAIN;
378			break;
379
380		case CRYPTO_SHA2_512_HMAC:
381			sessionSetupData.hashSetupData.hashAlgorithm =
382			    CPA_CY_SYM_HASH_SHA512;
383			sessionSetupData.hashSetupData.hashMode =
384			    CPA_CY_SYM_HASH_MODE_AUTH;
385			break;
386		case CRYPTO_SHA2_512:
387			sessionSetupData.hashSetupData.hashAlgorithm =
388			    CPA_CY_SYM_HASH_SHA512;
389			sessionSetupData.hashSetupData.hashMode =
390			    CPA_CY_SYM_HASH_MODE_PLAIN;
391			break;
392		case CRYPTO_AES_NIST_GMAC:
393			sessionSetupData.hashSetupData.hashAlgorithm =
394			    CPA_CY_SYM_HASH_AES_GMAC;
395			break;
396		default:
397			status = CPA_STATUS_UNSUPPORTED;
398			goto fail;
399		}
400	} /* csp->csp_auth_alg */
401
402	/* Setting digest-length if no cipher-only mode is set */
403	if (csp->csp_mode != CSP_MODE_CIPHER) {
404		lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL;
405		if (csp->csp_auth_mlen) {
406			sessionSetupData.hashSetupData.digestResultLenInBytes =
407			    csp->csp_auth_mlen;
408			qat_ssession->authLen = csp->csp_auth_mlen;
409		} else {
410			LacSymQat_HashDefsLookupGet(
411			    qat_instance->cyInstHandle,
412			    sessionSetupData.hashSetupData.hashAlgorithm,
413			    &pHashDefsInfo);
414			if (NULL == pHashDefsInfo) {
415				device_printf(
416				    dev,
417				    "unable to find corresponding hash data\n");
418				status = CPA_STATUS_UNSUPPORTED;
419				goto fail;
420			}
421			sessionSetupData.hashSetupData.digestResultLenInBytes =
422			    pHashDefsInfo->algInfo->digestLength;
423			qat_ssession->authLen =
424			    pHashDefsInfo->algInfo->digestLength;
425		}
426		sessionSetupData.verifyDigest = CPA_FALSE;
427	}
428
429	switch (csp->csp_mode) {
430	case CSP_MODE_AEAD:
431	case CSP_MODE_ETA:
432		sessionSetupData.symOperation =
433		    CPA_CY_SYM_OP_ALGORITHM_CHAINING;
434		/* Place the digest result in a buffer unrelated to srcBuffer */
435		sessionSetupData.digestIsAppended = CPA_FALSE;
436		/* Due to FW limitation to verify only appended MACs */
437		sessionSetupData.verifyDigest = CPA_FALSE;
438		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
439			sessionSetupData.cipherSetupData.cipherDirection =
440			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
441			sessionSetupData.algChainOrder =
442			    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
443		} else {
444			sessionSetupData.cipherSetupData.cipherDirection =
445			    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
446			sessionSetupData.algChainOrder =
447			    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
448		}
449		break;
450	case CSP_MODE_CIPHER:
451		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
452			sessionSetupData.cipherSetupData.cipherDirection =
453			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
454		} else {
455			sessionSetupData.cipherSetupData.cipherDirection =
456			    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
457		}
458		sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
459		break;
460	case CSP_MODE_DIGEST:
461		sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
462		if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
463			sessionSetupData.symOperation =
464			    CPA_CY_SYM_OP_ALGORITHM_CHAINING;
465			/* GMAC is always encrypt */
466			sessionSetupData.cipherSetupData.cipherDirection =
467			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
468			sessionSetupData.algChainOrder =
469			    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
470			sessionSetupData.cipherSetupData.cipherAlgorithm =
471			    CPA_CY_SYM_CIPHER_AES_GCM;
472			sessionSetupData.hashSetupData.hashAlgorithm =
473			    CPA_CY_SYM_HASH_AES_GMAC;
474			sessionSetupData.hashSetupData.hashMode =
475			    CPA_CY_SYM_HASH_MODE_AUTH;
476			/* Same key for cipher and auth */
477			sessionSetupData.cipherSetupData.pCipherKey =
478			    csp->csp_auth_key;
479			sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
480			    csp->csp_auth_klen;
481			/* Generated GMAC stored in separated buffer */
482			sessionSetupData.digestIsAppended = CPA_FALSE;
483			/* Digest verification not allowed in GMAC case */
484			sessionSetupData.verifyDigest = CPA_FALSE;
485			/* No AAD allowed */
486			sessionSetupData.hashSetupData.authModeSetupData
487			    .aadLenInBytes = 0;
488		} else {
489			sessionSetupData.cipherSetupData.cipherDirection =
490			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
491			sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
492			sessionSetupData.digestIsAppended = CPA_FALSE;
493		}
494		break;
495	default:
496		device_printf(dev,
497			      "%s: unhandled crypto algorithm %d, %d\n",
498			      __func__,
499			      csp->csp_cipher_alg,
500			      csp->csp_auth_alg);
501		status = CPA_STATUS_FAIL;
502		goto fail;
503	}
504
505	/* Extracting session size */
506	status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle,
507					   &sessionSetupData,
508					   &sessionCtxSize);
509	if (CPA_STATUS_SUCCESS != status) {
510		device_printf(dev, "unable to get session size\n");
511		goto fail;
512	}
513
514	/* Allocating contiguous memory for session */
515	sessionCtx = contigmalloc(sessionCtxSize,
516				  M_QAT_OCF,
517				  M_NOWAIT,
518				  0,
519				  ~1UL,
520				  1 << (ilog2(sessionCtxSize - 1) + 1),
521				  0);
522	if (NULL == sessionCtx) {
523		device_printf(dev, "unable to allocate memory for session\n");
524		status = CPA_STATUS_RESOURCE;
525		goto fail;
526	}
527
528	status = cpaCySymDpInitSession(qat_instance->cyInstHandle,
529				       &sessionSetupData,
530				       sessionCtx);
531	if (CPA_STATUS_SUCCESS != status) {
532		device_printf(dev, "session initialization failed\n");
533		goto fail;
534	}
535
536	/* NOTE: lets keep double session (both directions) approach to overcome
537	 * lack of direction update in FBSD QAT.
538	 */
539	qat_ssession->sessionCtx = sessionCtx;
540	qat_ssession->sessionCtxSize = sessionCtxSize;
541
542	return CPA_STATUS_SUCCESS;
543
544fail:
545	/* Release resources if any */
546	if (sessionCtx)
547		contigfree(sessionCtx, sessionCtxSize, M_QAT_OCF);
548
549	return status;
550}
551
552static int
553qat_ocf_newsession(device_t dev,
554		   crypto_session_t cses,
555		   const struct crypto_session_params *csp)
556{
557	/* Cryptodev QAT structures */
558	struct qat_ocf_softc *qat_softc;
559	struct qat_ocf_dsession *qat_dsession;
560	struct qat_ocf_instance *qat_instance;
561	u_int cpu_id = PCPU_GET(cpuid);
562
563	/* Create cryptodev session */
564	qat_softc = device_get_softc(dev);
565	if (qat_softc->numCyInstances > 0) {
566		qat_instance =
567		    &qat_softc
568			 ->cyInstHandles[cpu_id % qat_softc->numCyInstances];
569		qat_dsession = crypto_get_driver_session(cses);
570		if (NULL == qat_dsession) {
571			device_printf(dev, "Unable to create new session\n");
572			return (EINVAL);
573		}
574
575		/* Add only instance at this point remaining operations moved to
576		 * lazy session init */
577		qat_dsession->qatInstance = qat_instance;
578	} else {
579		return ENXIO;
580	}
581
582	return 0;
583}
584
585static CpaStatus
586qat_ocf_remove_session(device_t dev,
587		       CpaInstanceHandle cyInstHandle,
588		       struct qat_ocf_session *qat_session)
589{
590	CpaStatus status = CPA_STATUS_SUCCESS;
591
592	if (NULL == qat_session->sessionCtx)
593		return CPA_STATUS_SUCCESS;
594
595	/* User callback is executed right before decrementing pending
596	 * callback atomic counter. To avoid removing session rejection
597	 * we have to wait a very short while for counter update
598	 * after call back execution. */
599	status = qat_ocf_wait_for_session(qat_session->sessionCtx,
600					  QAT_OCF_SESSION_WAIT_TIMEOUT_MS);
601	if (CPA_STATUS_SUCCESS != status) {
602		device_printf(dev, "waiting for session un-busy failed\n");
603		return CPA_STATUS_FAIL;
604	}
605
606	status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx);
607	if (CPA_STATUS_SUCCESS != status) {
608		device_printf(dev, "error while removing session\n");
609		return CPA_STATUS_FAIL;
610	}
611
612	explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize);
613	contigfree(qat_session->sessionCtx,
614		   qat_session->sessionCtxSize,
615		   M_QAT_OCF);
616	qat_session->sessionCtx = NULL;
617	qat_session->sessionCtxSize = 0;
618
619	return CPA_STATUS_SUCCESS;
620}
621
622static void
623qat_ocf_freesession(device_t dev, crypto_session_t cses)
624{
625	CpaStatus status = CPA_STATUS_SUCCESS;
626	struct qat_ocf_dsession *qat_dsession = NULL;
627	struct qat_ocf_instance *qat_instance = NULL;
628
629	qat_dsession = crypto_get_driver_session(cses);
630	qat_instance = qat_dsession->qatInstance;
631	mtx_lock(&qat_instance->cyInstMtx);
632	status = qat_ocf_remove_session(dev,
633					qat_dsession->qatInstance->cyInstHandle,
634					&qat_dsession->encSession);
635	if (CPA_STATUS_SUCCESS != status)
636		device_printf(dev, "unable to remove encrypt session\n");
637	status = qat_ocf_remove_session(dev,
638					qat_dsession->qatInstance->cyInstHandle,
639					&qat_dsession->decSession);
640	if (CPA_STATUS_SUCCESS != status)
641		device_printf(dev, "unable to remove decrypt session\n");
642	mtx_unlock(&qat_instance->cyInstMtx);
643}
644
645/* QAT GCM/CCM FW API are only algorithms which support separated AAD. */
646static CpaStatus
647qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
648{
649	CpaCySymDpOpData *pOpData;
650
651	pOpData = &qat_cookie->pOpdata;
652
653	if (NULL != crp->crp_aad)
654		memcpy(qat_cookie->qat_ocf_gcm_aad,
655		       crp->crp_aad,
656		       crp->crp_aad_length);
657	else
658		crypto_copydata(crp,
659				crp->crp_aad_start,
660				crp->crp_aad_length,
661				qat_cookie->qat_ocf_gcm_aad);
662
663	pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad;
664	pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr;
665
666	return CPA_STATUS_SUCCESS;
667}
668
669static CpaStatus
670qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
671{
672	CpaStatus status = CPA_STATUS_SUCCESS;
673	const struct crypto_session_params *csp;
674	CpaCySymDpOpData *pOpData;
675	struct qat_ocf_load_cb_arg args;
676
677	pOpData = &qat_cookie->pOpdata;
678	pOpData->pAdditionalAuthData = NULL;
679	pOpData->additionalAuthData = 0UL;
680
681	if (crp->crp_aad_length == 0)
682		return CPA_STATUS_SUCCESS;
683
684	if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)
685		return CPA_STATUS_FAIL;
686
687	csp = crypto_get_params(crp->crp_session);
688
689	/* Handle GCM/CCM case */
690	if (CPA_TRUE == is_sep_aad_supported(csp))
691		return qat_ocf_load_aad_gcm(crp, qat_cookie);
692
693	if (NULL == crp->crp_aad) {
694		/* AAD already embedded in source buffer */
695		pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
696		pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
697
698		pOpData->messageLenToHashInBytes =
699		    crp->crp_aad_length + crp->crp_payload_length;
700		pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
701
702		return CPA_STATUS_SUCCESS;
703	}
704
705	/* Separated AAD not supported by QAT - lets place the content
706	 * of ADD buffer at the very beginning of source SGL */
707	args.crp_op = crp;
708	args.qat_cookie = qat_cookie;
709	args.pOpData = pOpData;
710	args.error = 0;
711	status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag,
712				 qat_cookie->gcm_aad_dma_mem.dma_map,
713				 crp->crp_aad,
714				 crp->crp_aad_length,
715				 qat_ocf_crypto_load_aadbuf_cb,
716				 &args,
717				 BUS_DMA_NOWAIT);
718	qat_cookie->is_sep_aad_used = CPA_TRUE;
719
720	/* Right after this step we have AAD placed in the first flat buffer
721	 * in source SGL */
722	pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
723	pOpData->cryptoStartSrcOffsetInBytes =
724	    crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start;
725
726	pOpData->messageLenToHashInBytes =
727	    crp->crp_aad_length + crp->crp_payload_length;
728	pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
729
730	return status;
731}
732
733static CpaStatus
734qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
735{
736	CpaStatus status = CPA_STATUS_SUCCESS;
737	CpaCySymDpOpData *pOpData;
738	struct qat_ocf_load_cb_arg args;
739	/* cryptodev internals */
740	const struct crypto_session_params *csp;
741
742	pOpData = &qat_cookie->pOpdata;
743
744	csp = crypto_get_params(crp->crp_session);
745
746	/* Load IV buffer if present */
747	if (csp->csp_ivlen > 0) {
748		memset(qat_cookie->qat_ocf_iv_buf,
749		       0,
750		       sizeof(qat_cookie->qat_ocf_iv_buf));
751		crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf);
752		pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr;
753		pOpData->pIv = qat_cookie->qat_ocf_iv_buf;
754		pOpData->ivLenInBytes = csp->csp_ivlen;
755	}
756
757	/* GCM/CCM - load AAD to separated buffer
758	 * AES+SHA - load AAD to first flat in SGL */
759	status = qat_ocf_load_aad(crp, qat_cookie);
760	if (CPA_STATUS_SUCCESS != status)
761		goto fail;
762
763	/* Load source buffer */
764	args.crp_op = crp;
765	args.qat_cookie = qat_cookie;
766	args.pOpData = pOpData;
767	args.error = 0;
768	status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag,
769					    qat_cookie->src_dma_mem.dma_map,
770					    &crp->crp_buf,
771					    qat_ocf_crypto_load_buf_cb,
772					    &args,
773					    BUS_DMA_NOWAIT);
774	if (CPA_STATUS_SUCCESS != status)
775		goto fail;
776	pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr;
777	pOpData->srcBufferLen = CPA_DP_BUFLIST;
778
779	/* Load destination buffer */
780	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
781		status =
782		    bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag,
783					       qat_cookie->dst_dma_mem.dma_map,
784					       &crp->crp_obuf,
785					       qat_ocf_crypto_load_obuf_cb,
786					       &args,
787					       BUS_DMA_NOWAIT);
788		if (CPA_STATUS_SUCCESS != status)
789			goto fail;
790		pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr;
791		pOpData->dstBufferLen = CPA_DP_BUFLIST;
792	} else {
793		pOpData->dstBuffer = pOpData->srcBuffer;
794		pOpData->dstBufferLen = pOpData->srcBufferLen;
795	}
796
797	if (CPA_TRUE == is_use_sep_digest(csp))
798		pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr;
799	else
800		pOpData->digestResult = 0UL;
801
802	/* GMAC - aka zero length buffer */
803	if (CPA_TRUE == is_gmac_exception(csp))
804		pOpData->messageLenToCipherInBytes = 0;
805
806fail:
807	return status;
808}
809
810static int
811qat_ocf_check_input(device_t dev, struct cryptop *crp)
812{
813	const struct crypto_session_params *csp;
814	csp = crypto_get_params(crp->crp_session);
815
816	if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN)
817		return E2BIG;
818
819	if (CPA_TRUE == is_sep_aad_supported(csp) &&
820	    (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX))
821		return EBADMSG;
822
823	return 0;
824}
825
826static int
827qat_ocf_process(device_t dev, struct cryptop *crp, int hint)
828{
829	CpaStatus status = CPA_STATUS_SUCCESS;
830	int rc = 0;
831	struct qat_ocf_dsession *qat_dsession = NULL;
832	struct qat_ocf_session *qat_session = NULL;
833	struct qat_ocf_instance *qat_instance = NULL;
834	CpaCySymDpOpData *pOpData = NULL;
835	struct qat_ocf_cookie *qat_cookie = NULL;
836	CpaBoolean memLoaded = CPA_FALSE;
837
838	rc = qat_ocf_check_input(dev, crp);
839	if (rc)
840		goto fail;
841
842	qat_dsession = crypto_get_driver_session(crp->crp_session);
843
844	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
845		qat_session = &qat_dsession->encSession;
846	else
847		qat_session = &qat_dsession->decSession;
848	qat_instance = qat_dsession->qatInstance;
849
850	status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie);
851	if (CPA_STATUS_SUCCESS != status) {
852		rc = EAGAIN;
853		goto fail;
854	}
855
856	qat_cookie->crp_op = crp;
857
858	/* Common request fields */
859	pOpData = &qat_cookie->pOpdata;
860	pOpData->instanceHandle = qat_instance->cyInstHandle;
861	pOpData->sessionCtx = NULL;
862
863	/* Cipher fields */
864	pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
865	pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
866	/* Digest fields - any exceptions from this basic rules are covered
867	 * in qat_ocf_load */
868	pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start;
869	pOpData->messageLenToHashInBytes = crp->crp_payload_length;
870
871	status = qat_ocf_load(crp, qat_cookie);
872	if (CPA_STATUS_SUCCESS != status) {
873		device_printf(dev,
874			      "unable to load OCF buffers to QAT DMA "
875			      "transaction\n");
876		rc = EIO;
877		goto fail;
878	}
879	memLoaded = CPA_TRUE;
880
881	status = qat_ocf_cookie_dma_pre_sync(crp, pOpData);
882	if (CPA_STATUS_SUCCESS != status) {
883		device_printf(dev, "unable to sync DMA buffers\n");
884		rc = EIO;
885		goto fail;
886	}
887
888	mtx_lock(&qat_instance->cyInstMtx);
889	/* Session initialization at the first request. It's done
890	 * in such way to overcome missing QAT specific session data
891	 * such like AAD length and limited possibility to update
892	 * QAT session while handling traffic.
893	 */
894	if (NULL == qat_session->sessionCtx) {
895		status =
896		    qat_ocf_session_init(dev, crp, qat_instance, qat_session);
897		if (CPA_STATUS_SUCCESS != status) {
898			mtx_unlock(&qat_instance->cyInstMtx);
899			device_printf(dev, "unable to init session\n");
900			rc = EIO;
901			goto fail;
902		}
903	} else {
904		status = qat_ocf_handle_session_update(qat_dsession, crp);
905		if (CPA_STATUS_RESOURCE == status) {
906			mtx_unlock(&qat_instance->cyInstMtx);
907			rc = EAGAIN;
908			goto fail;
909		} else if (CPA_STATUS_SUCCESS != status) {
910			mtx_unlock(&qat_instance->cyInstMtx);
911			rc = EIO;
912			goto fail;
913		}
914	}
915	pOpData->sessionCtx = qat_session->sessionCtx;
916	status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE);
917	mtx_unlock(&qat_instance->cyInstMtx);
918	if (CPA_STATUS_SUCCESS != status) {
919		if (CPA_STATUS_RETRY == status) {
920			rc = EAGAIN;
921			goto fail;
922		}
923		device_printf(dev,
924			      "unable to send request. Status: %d\n",
925			      status);
926		rc = EIO;
927		goto fail;
928	}
929
930	return 0;
931fail:
932	if (qat_cookie) {
933		if (memLoaded)
934			qat_ocf_cookie_dma_unload(crp, pOpData);
935		qat_ocf_cookie_free(qat_instance, qat_cookie);
936	}
937	crp->crp_etype = rc;
938	crypto_done(crp);
939
940	return 0;
941}
942
943static void
944qat_ocf_identify(driver_t *drv, device_t parent)
945{
946	if (device_find_child(parent, "qat_ocf", -1) == NULL &&
947	    BUS_ADD_CHILD(parent, 200, "qat_ocf", -1) == 0)
948		device_printf(parent, "qat_ocf: could not attach!");
949}
950
951static int
952qat_ocf_probe(device_t dev)
953{
954	device_set_desc(dev, "QAT engine");
955	return (BUS_PROBE_NOWILDCARD);
956}
957
958static CpaStatus
959qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles,
960			  Cpa16U cyInstHandlesSize,
961			  Cpa16U *foundInstances)
962{
963	CpaStatus status = CPA_STATUS_SUCCESS;
964	icp_accel_dev_t **pAdfInsts = NULL;
965	icp_accel_dev_t *dev_addr = NULL;
966	sal_t *baseAddr = NULL;
967	sal_list_t *listTemp = NULL;
968	CpaInstanceHandle cyInstHandle;
969	CpaInstanceInfo2 info;
970	Cpa16U numDevices;
971	Cpa32U instCtr = 0;
972	Cpa32U i;
973
974	/* Get the number of devices */
975	status = icp_amgr_getNumInstances(&numDevices);
976	if (CPA_STATUS_SUCCESS != status)
977		return status;
978
979	/* Allocate memory to store addr of accel_devs */
980	pAdfInsts =
981	    malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK);
982
983	/* Get ADF to return all accel_devs that support either
984	 * symmetric or asymmetric crypto */
985	status = icp_amgr_getAllAccelDevByCapabilities(
986	    (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices);
987	if (CPA_STATUS_SUCCESS != status) {
988		free(pAdfInsts, M_QAT_OCF);
989		return status;
990	}
991
992	for (i = 0; i < numDevices; i++) {
993		dev_addr = (icp_accel_dev_t *)pAdfInsts[i];
994		baseAddr = dev_addr->pSalHandle;
995		if (NULL == baseAddr)
996			continue;
997		listTemp = baseAddr->sym_services;
998		if (NULL == listTemp) {
999			listTemp = baseAddr->crypto_services;
1000		}
1001
1002		while (NULL != listTemp) {
1003			cyInstHandle = SalList_getObject(listTemp);
1004			status = cpaCyInstanceGetInfo2(cyInstHandle, &info);
1005			if (CPA_STATUS_SUCCESS != status)
1006				continue;
1007			listTemp = SalList_next(listTemp);
1008			if (CPA_TRUE == info.isPolled)
1009				continue;
1010			if (instCtr >= cyInstHandlesSize)
1011				break;
1012			cyInstHandles[instCtr++] = cyInstHandle;
1013		}
1014	}
1015	free(pAdfInsts, M_QAT_OCF);
1016	*foundInstances = instCtr;
1017
1018	return CPA_STATUS_SUCCESS;
1019}
1020
1021static CpaStatus
1022qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev)
1023{
1024	CpaStatus status = CPA_STATUS_SUCCESS;
1025	Cpa16U numInstances = 0;
1026	CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 };
1027	CpaInstanceHandle cyInstHandle = NULL;
1028	Cpa32U startedInstances = 0;
1029	Cpa32U i;
1030
1031	qat_softc->numCyInstances = 0;
1032	status = qat_ocf_get_irq_instances(cyInstHandles,
1033					   QAT_OCF_MAX_INSTANCES,
1034					   &numInstances);
1035	if (CPA_STATUS_SUCCESS != status)
1036		return status;
1037
1038	for (i = 0; i < numInstances; i++) {
1039		struct qat_ocf_instance *qat_ocf_instance;
1040
1041		cyInstHandle = cyInstHandles[i];
1042		if (!cyInstHandle)
1043			continue;
1044
1045		/* Starting instance */
1046		status = cpaCyStartInstance(cyInstHandle);
1047		if (CPA_STATUS_SUCCESS != status) {
1048			device_printf(qat_softc->sc_dev,
1049				      "unable to get start instance\n");
1050			continue;
1051		}
1052
1053		qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances];
1054		qat_ocf_instance->cyInstHandle = cyInstHandle;
1055		mtx_init(&qat_ocf_instance->cyInstMtx,
1056			 "Instance MTX",
1057			 NULL,
1058			 MTX_DEF);
1059
1060		status =
1061		    cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys);
1062		if (CPA_STATUS_SUCCESS != status) {
1063			device_printf(qat_softc->sc_dev,
1064				      "unable to add virt to phys callback\n");
1065			goto fail;
1066		}
1067
1068		status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback);
1069		if (CPA_STATUS_SUCCESS != status) {
1070			device_printf(qat_softc->sc_dev,
1071				      "unable to add user callback\n");
1072			goto fail;
1073		}
1074
1075		/* Initialize cookie pool */
1076		status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev);
1077		if (CPA_STATUS_SUCCESS != status) {
1078			device_printf(qat_softc->sc_dev,
1079				      "unable to create cookie pool\n");
1080			goto fail;
1081		}
1082
1083		/* Disable forcing HW MAC validation for AEAD */
1084		status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE);
1085		if (CPA_STATUS_SUCCESS != status) {
1086			device_printf(
1087			    qat_softc->sc_dev,
1088			    "unable to disable AEAD HW MAC verification\n");
1089			goto fail;
1090		}
1091
1092		qat_ocf_instance->driver_id = qat_softc->cryptodev_id;
1093
1094		startedInstances++;
1095		continue;
1096	fail:
1097		mtx_destroy(&qat_ocf_instance->cyInstMtx);
1098
1099		/* Stop instance */
1100		status = cpaCyStopInstance(cyInstHandle);
1101		if (CPA_STATUS_SUCCESS != status)
1102			device_printf(qat_softc->sc_dev,
1103				      "unable to stop the instance\n");
1104	}
1105	qat_softc->numCyInstances = startedInstances;
1106
1107	return CPA_STATUS_SUCCESS;
1108}
1109
1110static CpaStatus
1111qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc)
1112{
1113	CpaStatus status = CPA_STATUS_SUCCESS;
1114	int i;
1115
1116	for (i = 0; i < qat_softc->numCyInstances; i++) {
1117		struct qat_ocf_instance *qat_instance;
1118
1119		qat_instance = &qat_softc->cyInstHandles[i];
1120		status = cpaCyStopInstance(qat_instance->cyInstHandle);
1121		if (CPA_STATUS_SUCCESS != status) {
1122			pr_err("QAT: stopping instance id: %d failed\n", i);
1123			continue;
1124		}
1125		qat_ocf_cookie_pool_deinit(qat_instance);
1126		mtx_destroy(&qat_instance->cyInstMtx);
1127	}
1128
1129	qat_softc->numCyInstances = 0;
1130
1131	return status;
1132}
1133
1134static int
1135qat_ocf_deinit(struct qat_ocf_softc *qat_softc)
1136{
1137	int status = 0;
1138	CpaStatus cpaStatus;
1139
1140	if (qat_softc->cryptodev_id >= 0) {
1141		crypto_unregister_all(qat_softc->cryptodev_id);
1142		qat_softc->cryptodev_id = -1;
1143	}
1144
1145	/* Stop QAT instances */
1146	cpaStatus = qat_ocf_stop_instances(qat_softc);
1147	if (CPA_STATUS_SUCCESS != cpaStatus) {
1148		device_printf(qat_softc->sc_dev, "unable to stop instances\n");
1149		status = EIO;
1150	}
1151
1152	return status;
1153}
1154
1155static int
1156qat_ocf_init(struct qat_ocf_softc *qat_softc)
1157{
1158	int32_t cryptodev_id;
1159
1160	/* Starting instances for OCF */
1161	if (qat_ocf_start_instances(qat_softc, qat_softc->sc_dev)) {
1162		device_printf(qat_softc->sc_dev,
1163			      "unable to get QAT IRQ instances\n");
1164		goto fail;
1165	}
1166
1167	/* Register only if instances available */
1168	if (qat_softc->numCyInstances) {
1169		cryptodev_id =
1170		    crypto_get_driverid(qat_softc->sc_dev,
1171					sizeof(struct qat_ocf_dsession),
1172					CRYPTOCAP_F_HARDWARE);
1173		if (cryptodev_id < 0) {
1174			device_printf(qat_softc->sc_dev,
1175				      "cannot initialize!\n");
1176			goto fail;
1177		}
1178		qat_softc->cryptodev_id = cryptodev_id;
1179	}
1180
1181	return 0;
1182fail:
1183	qat_ocf_deinit(qat_softc);
1184
1185	return ENXIO;
1186}
1187
1188static int qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS)
1189{
1190	struct qat_ocf_softc *qat_softc = NULL;
1191	int ret = 0;
1192	device_t dev = arg1;
1193	u_int enabled;
1194
1195	qat_softc = device_get_softc(dev);
1196	enabled = qat_softc->enabled;
1197
1198	ret = sysctl_handle_int(oidp, &enabled, 0, req);
1199	if (ret || !req->newptr)
1200		return (ret);
1201
1202	if (qat_softc->enabled != enabled) {
1203		if (enabled) {
1204			ret = qat_ocf_init(qat_softc);
1205
1206		} else {
1207			ret = qat_ocf_deinit(qat_softc);
1208		}
1209
1210		if (!ret)
1211			qat_softc->enabled = enabled;
1212	}
1213
1214	return ret;
1215}
1216
1217static int
1218qat_ocf_attach(device_t dev)
1219{
1220	int status;
1221	struct qat_ocf_softc *qat_softc;
1222
1223	qat_softc = device_get_softc(dev);
1224	qat_softc->sc_dev = dev;
1225	qat_softc->cryptodev_id = -1;
1226	qat_softc->enabled = 1;
1227
1228	qat_softc->rc =
1229	    SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
1230			    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
1231			    OID_AUTO,
1232			    "enable",
1233			    CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
1234			    dev,
1235			    0,
1236			    qat_ocf_sysctl_handle,
1237			    "I",
1238			    "QAT OCF support enablement");
1239
1240	if (!qat_softc->rc)
1241		return ENOMEM;
1242	if (qat_softc->enabled) {
1243		status = qat_ocf_init(qat_softc);
1244		if (status) {
1245			device_printf(dev, "qat_ocf init failed\n");
1246			goto fail;
1247		}
1248	}
1249
1250	return 0;
1251fail:
1252	qat_ocf_deinit(qat_softc);
1253
1254	return (ENXIO);
1255}
1256
1257static int
1258qat_ocf_detach(device_t dev)
1259{
1260	struct qat_ocf_softc *qat_softc = device_get_softc(dev);
1261
1262	return qat_ocf_deinit(qat_softc);
1263}
1264
1265static device_method_t qat_ocf_methods[] =
1266    { DEVMETHOD(device_identify, qat_ocf_identify),
1267      DEVMETHOD(device_probe, qat_ocf_probe),
1268      DEVMETHOD(device_attach, qat_ocf_attach),
1269      DEVMETHOD(device_detach, qat_ocf_detach),
1270
1271      /* Cryptodev interface */
1272      DEVMETHOD(cryptodev_probesession, qat_ocf_probesession),
1273      DEVMETHOD(cryptodev_newsession, qat_ocf_newsession),
1274      DEVMETHOD(cryptodev_freesession, qat_ocf_freesession),
1275      DEVMETHOD(cryptodev_process, qat_ocf_process),
1276
1277      DEVMETHOD_END };
1278
1279static driver_t qat_ocf_driver = {
1280	.name = "qat_ocf",
1281	.methods = qat_ocf_methods,
1282	.size = sizeof(struct qat_ocf_softc),
1283};
1284
1285
1286DRIVER_MODULE_ORDERED(qat,
1287		      nexus,
1288		      qat_ocf_driver,
1289		      NULL,
1290		      NULL,
1291		      SI_ORDER_ANY);
1292MODULE_VERSION(qat, 1);
1293MODULE_DEPEND(qat, qat_c62x, 1, 1, 1);
1294MODULE_DEPEND(qat, qat_200xx, 1, 1, 1);
1295MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1);
1296MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1);
1297MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1);
1298MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1);
1299MODULE_DEPEND(qat, crypto, 1, 1, 1);
1300MODULE_DEPEND(qat, qat_common, 1, 1, 1);
1301MODULE_DEPEND(qat, qat_api, 1, 1, 1);
1302MODULE_DEPEND(qat, linuxkpi, 1, 1, 1);
1303