1/* SPDX-License-Identifier: BSD-3-Clause */
2/* Copyright(c) 2007-2022 Intel Corporation */
3
4/**
5 ***************************************************************************
6 * @file lac_sym_cb.c      Callback handler functions for symmetric components
7 *
8 * @ingroup LacSym
9 *
10 ***************************************************************************/
11
12/*
13*******************************************************************************
14* Include public/global header files
15*******************************************************************************
16*/
17
18#include "cpa.h"
19#include "cpa_cy_sym.h"
20
21#include "icp_accel_devices.h"
22#include "icp_adf_init.h"
23#include "icp_qat_fw_la.h"
24#include "icp_adf_transport.h"
25#include "icp_adf_debug.h"
26
27#include "lac_sym.h"
28#include "lac_sym_cipher.h"
29#include "lac_common.h"
30#include "lac_list.h"
31#include "lac_sal_types_crypto.h"
32#include "lac_sal.h"
33#include "lac_sal_ctrl.h"
34#include "lac_session.h"
35#include "lac_sym_stats.h"
36#include "lac_log.h"
37#include "lac_sym_cb.h"
38#include "lac_sym_hash.h"
39#include "lac_sym_qat_cipher.h"
40#include "lac_sym_qat.h"
41
42#define DEQUEUE_MSGPUT_MAX_RETRIES 10000
43
44/*
45*******************************************************************************
46* Define static function definitions
47*******************************************************************************
48*/
49
50/**
51 *****************************************************************************
52 * @ingroup LacSymCb
53 *      Function to clean computed data.
54 *
55 * @description
56 *      This function cleans GCM or CCM data in the case of a failure.
57 *
58 * @param[in]  pSessionDesc pointer to the session descriptor
59 * @param[out] pBufferList  pointer to the bufferlist to clean
60 * @param[in]  pOpData      pointer to operation data
61 * @param[in]  isCCM        is it a CCM operation boolean
62 *
63 * @return  None
64 *****************************************************************************/
65static void
66LacSymCb_CleanUserData(const lac_session_desc_t *pSessionDesc,
67		       CpaBufferList *pBufferList,
68		       const CpaCySymOpData *pOpData,
69		       CpaBoolean isCCM)
70{
71	Cpa32U authTagLen = 0;
72
73	/* Retrieve authTagLen */
74	authTagLen = pSessionDesc->hashResultSize;
75
76	/* Cleaning */
77	if (isCCM) {
78		/* for CCM the digest is inside the buffer list */
79		LacBuffDesc_BufferListZeroFromOffset(
80		    pBufferList,
81		    pOpData->cryptoStartSrcOffsetInBytes,
82		    pOpData->messageLenToCipherInBytes + authTagLen);
83	} else {
84		/* clean buffer list */
85		LacBuffDesc_BufferListZeroFromOffset(
86		    pBufferList,
87		    pOpData->cryptoStartSrcOffsetInBytes,
88		    pOpData->messageLenToCipherInBytes);
89	}
90	if ((CPA_TRUE != pSessionDesc->digestIsAppended) &&
91	    (NULL != pOpData->pDigestResult)) {
92		/* clean digest */
93		memset(pOpData->pDigestResult, 0, authTagLen);
94	}
95}
96
97/**
98 *****************************************************************************
99 * @ingroup LacSymCb
100 *      Definition of callback function for processing symmetric responses
101 *
102 * @description
103 *      This callback is invoked to process symmetric response messages from
104 *      the QAT.  It will extract some details from the message and invoke
105 *      the user's callback to complete a symmetric operation.
106 *
107 * @param[in] pCookie             Pointer to cookie associated with this request
108 * @param[in] qatRespStatusOkFlag Boolean indicating ok/fail status from QAT
109 * @param[in] status              Status variable indicating an error occurred
110 *                                in sending the message (e.g. when dequeueing)
111 * @param[in] pSessionDesc        Session descriptor
112 *
113 * @return  None
114 *****************************************************************************/
115static void
116LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie,
117				 CpaBoolean qatRespStatusOkFlag,
118				 CpaStatus status,
119				 lac_session_desc_t *pSessionDesc)
120{
121	CpaCySymCbFunc pSymCb = NULL;
122	void *pCallbackTag = NULL;
123	CpaCySymOpData *pOpData = NULL;
124	CpaBufferList *pDstBuffer = NULL;
125	CpaCySymOp operationType = CPA_CY_SYM_OP_NONE;
126	CpaStatus dequeueStatus = CPA_STATUS_SUCCESS;
127
128	CpaInstanceHandle instanceHandle = CPA_INSTANCE_HANDLE_SINGLE;
129	/* NOTE: cookie pointer validated in previous function */
130	instanceHandle = pCookie->instanceHandle;
131
132	pOpData = (CpaCySymOpData *)LAC_CONST_PTR_CAST(pCookie->pOpData);
133	operationType = pSessionDesc->symOperation;
134
135	/* Set the destination pointer to the one supplied in the cookie. */
136	pDstBuffer = pCookie->pDstBuffer;
137
138	/* For a digest verify operation - for full packet and final partial
139	 * only, perform a comparison with the digest generated and with the one
140	 * supplied in the packet. In case of AES_GCM in SPC mode, destination
141	 * buffer needs to be cleared if digest verify operation fails */
142
143	if (((SPC == pSessionDesc->singlePassState) ||
144	     (CPA_CY_SYM_OP_CIPHER != operationType)) &&
145	    (CPA_TRUE == pSessionDesc->digestVerify) &&
146	    ((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) ||
147	     (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType))) {
148		if (CPA_FALSE == qatRespStatusOkFlag) {
149			LAC_SYM_STAT_INC(numSymOpVerifyFailures,
150					 instanceHandle);
151
152			/* The comparison has failed at this point (status is
153			 * fail), need to clean any sensitive calculated data up
154			 * to this point. The data calculated is no longer
155			 * useful to the end result and does not need to be
156			 * returned to the user so setting buffers to zero.
157			 */
158			if (pSessionDesc->cipherAlgorithm ==
159			    CPA_CY_SYM_CIPHER_AES_CCM) {
160				LacSymCb_CleanUserData(pSessionDesc,
161						       pDstBuffer,
162						       pOpData,
163						       CPA_TRUE);
164			} else if (pSessionDesc->cipherAlgorithm ==
165				   CPA_CY_SYM_CIPHER_AES_GCM) {
166				LacSymCb_CleanUserData(pSessionDesc,
167						       pDstBuffer,
168						       pOpData,
169						       CPA_FALSE);
170			}
171		}
172	} else {
173		/* Most commands have no point of failure and always return
174		 * success. This is the default response from the QAT.
175		 * If status is already set to an error value, don't overwrite
176		 * it
177		 */
178		if ((CPA_STATUS_SUCCESS == status) &&
179		    (CPA_TRUE != qatRespStatusOkFlag)) {
180			LAC_LOG_ERROR("Response status value not as expected");
181			status = CPA_STATUS_FAIL;
182		}
183	}
184
185	pSymCb = pSessionDesc->pSymCb;
186	pCallbackTag = pCookie->pCallbackTag;
187
188	/* State returned to the client for intermediate partials packets
189	 * for hash only and cipher only partial packets. Cipher update
190	 * allow next partial through */
191	if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) {
192		if ((CPA_CY_SYM_OP_CIPHER == operationType) ||
193		    (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) {
194			if (CPA_TRUE == pCookie->updateUserIvOnRecieve) {
195				/* Update the user's IV buffer
196				 * Very important to do this BEFORE dequeuing
197				 * subsequent partial requests, as the state
198				 * buffer may get overwritten
199				 */
200				memcpy(pCookie->pOpData->pIv,
201				       pSessionDesc->cipherPartialOpState,
202				       pCookie->pOpData->ivLenInBytes);
203			}
204			if (CPA_TRUE == pCookie->updateKeySizeOnRecieve &&
205			    LAC_CIPHER_IS_XTS_MODE(
206				pSessionDesc->cipherAlgorithm)) {
207				LacSymQat_CipherXTSModeUpdateKeyLen(
208				    pSessionDesc,
209				    pSessionDesc->cipherKeyLenInBytes / 2);
210			}
211		}
212	} else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) {
213		if ((CPA_CY_SYM_OP_CIPHER == operationType) ||
214		    (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) {
215			if (CPA_TRUE ==
216			    LAC_CIPHER_IS_XTS_MODE(
217				pSessionDesc->cipherAlgorithm)) {
218				/*
219				 * For XTS mode, we replace the updated key with
220				 * the original key - for subsequent partial
221				 * requests
222				 *
223				 */
224				LacSymQat_CipherXTSModeUpdateKeyLen(
225				    pSessionDesc,
226				    pSessionDesc->cipherKeyLenInBytes);
227			}
228		}
229	}
230
231	if ((CPA_CY_SYM_PACKET_TYPE_FULL != pOpData->packetType) &&
232	    (qatRespStatusOkFlag != CPA_FALSE)) {
233		/* There may be requests blocked pending the completion of this
234		 * operation
235		 */
236
237		dequeueStatus = LacSymCb_PendingReqsDequeue(pSessionDesc);
238		if (CPA_STATUS_SUCCESS != dequeueStatus) {
239			LAC_SYM_STAT_INC(numSymOpCompletedErrors,
240					 instanceHandle);
241			qatRespStatusOkFlag = CPA_FALSE;
242			if (CPA_STATUS_SUCCESS == status) {
243				status = dequeueStatus;
244			}
245		}
246	}
247
248	if (CPA_STATUS_SUCCESS == status) {
249		/* update stats */
250		if (pSessionDesc->internalSession == CPA_FALSE) {
251			LAC_SYM_STAT_INC(numSymOpCompleted, instanceHandle);
252			if (CPA_STATUS_SUCCESS != status) {
253				LAC_SYM_STAT_INC(numSymOpCompletedErrors,
254						 instanceHandle);
255			}
256		}
257	}
258
259	qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount));
260
261	/* deallocate the memory for the internal callback cookie */
262	Lac_MemPoolEntryFree(pCookie);
263
264	/* user callback function is the last thing to be called */
265	pSymCb(pCallbackTag,
266	       status,
267	       operationType,
268	       pOpData,
269	       pDstBuffer,
270	       qatRespStatusOkFlag);
271}
272
273/**
274 ******************************************************************************
275 * @ingroup LacSymCb
276 *      Definition of callback function for processing symmetric Data Plane
277 *       responses
278 *
279 * @description
280 *      This callback checks the status, decrements the number of operations
281 *      pending and calls the user callback
282 *
283 * @param[in/out] pResponse          pointer to the response structure
284 * @param[in] qatRespStatusOkFlag    status
285 * @param[in] pSessionDesc           pointer to the session descriptor
286 *
287 * @return  None
288 ******************************************************************************/
289static void
290LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse,
291			   CpaBoolean qatRespStatusOkFlag,
292			   CpaStatus status,
293			   lac_session_desc_t *pSessionDesc)
294{
295	CpaCySymDpCbFunc pSymDpCb = NULL;
296
297	/* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be
298	 * cleaned as stated in RFC 3610; in DP mode, it is the user
299	 * responsability to do so */
300
301	if (((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) &&
302	     SPC != pSessionDesc->singlePassState) ||
303	    (CPA_FALSE == pSessionDesc->digestVerify)) {
304		/* If not doing digest compare and qatRespStatusOkFlag !=
305		   CPA_TRUE then there is something very wrong */
306		if ((CPA_FALSE == qatRespStatusOkFlag) &&
307		    (status != CPA_STATUS_UNSUPPORTED)) {
308			LAC_LOG_ERROR("Response status value not as expected");
309			status = CPA_STATUS_FAIL;
310		}
311	}
312
313	pSymDpCb =
314	    ((sal_crypto_service_t *)pResponse->instanceHandle)->pSymDpCb;
315
316	pSymDpCb(pResponse, status, qatRespStatusOkFlag);
317
318	/*
319	 * Decrement the number of pending CB.
320	 *
321	 * If the @pendingDpCbCount becomes zero, we may remove the session,
322	 * please read more information in the cpaCySymRemoveSession().
323	 *
324	 * But there is a field in the @pResponse to store the session,
325	 * the "sessionCtx". In another word, in the above @->pSymDpCb()
326	 * callback, it may use the session again. If we decrease the
327	 * @pendingDpCbCount before the @->pSymDpCb(), there is a _risk_ the
328	 * @->pSymDpCb() may reference to a deleted session.
329	 *
330	 * So in order to avoid the risk, we decrease the @pendingDpCbCount
331	 * after the @->pSymDpCb() callback.
332	 */
333	qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount);
334}
335
336/**
337 ******************************************************************************
338 * @ingroup LacSymCb
339 *      Definition of callback function for processing symmetric responses
340 *
341 * @description
342 *      This callback, which is registered with the common symmetric response
343 *      message handler,  is invoked to process symmetric response messages from
344 *      the QAT.  It will extract the response status from the cmnRespFlags set
345 *      by the QAT, and then will pass it to @ref
346 *      LacSymCb_ProcessCallbackInternal to complete the response processing.
347 *
348 * @param[in] lacCmdId          ID of the symmetric QAT command of the request
349 *                              message
350 * @param[in] pOpaqueData       pointer to opaque data in the request message
351 * @param[in] cmnRespFlags      Flags set by QAT to indicate response status
352 *
353 * @return  None
354 ******************************************************************************/
355static void
356LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId,
357			 void *pOpaqueData,
358			 icp_qat_fw_comn_flags cmnRespFlags)
359{
360	CpaStatus status = CPA_STATUS_SUCCESS;
361	CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData;
362	lac_session_desc_t *pSessionDesc =
363	    LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx);
364	CpaBoolean qatRespStatusOkFlag =
365	    (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK ==
366			 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(cmnRespFlags));
367
368	if (CPA_TRUE == pSessionDesc->isDPSession) {
369		/* DP session */
370		if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(
371			cmnRespFlags)) {
372			status = CPA_STATUS_UNSUPPORTED;
373		}
374		LacSymCb_ProcessDpCallback(pDpOpData,
375					   qatRespStatusOkFlag,
376					   status,
377					   pSessionDesc);
378	} else {
379		/* Trad session */
380		LacSymCb_ProcessCallbackInternal((lac_sym_bulk_cookie_t *)
381						     pOpaqueData,
382						 qatRespStatusOkFlag,
383						 CPA_STATUS_SUCCESS,
384						 pSessionDesc);
385	}
386}
387
388/*
389*******************************************************************************
390* Define public/global function definitions
391*******************************************************************************
392*/
393
394/**
395 * @ingroup LacSymCb
396 *
397 * @return CpaStatus
398 *      value returned will be the result of icp_adf_transPutMsg
399 */
400CpaStatus
401LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc)
402{
403	CpaStatus status = CPA_STATUS_SUCCESS;
404	sal_crypto_service_t *pService = NULL;
405	Cpa32U retries = 0;
406
407	pService = (sal_crypto_service_t *)pSessionDesc->pInstance;
408
409	/* Need to protect access to queue head and tail pointers, which may
410	 * be accessed by multiple contexts simultaneously for enqueue and
411	 * dequeue operations
412	 */
413	LAC_SPINLOCK(&pSessionDesc->requestQueueLock);
414
415	/* Clear the blocking flag in the session descriptor */
416	pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE;
417
418	while ((NULL != pSessionDesc->pRequestQueueHead) &&
419	       (CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) {
420
421		/* If we send a partial packet request, set the
422		 * blockingOpsInProgress flag for the session to indicate that
423		 * subsequent requests must be queued up until this request
424		 * completes
425		 */
426		if (CPA_CY_SYM_PACKET_TYPE_FULL !=
427		    pSessionDesc->pRequestQueueHead->pOpData->packetType) {
428			pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE;
429		}
430
431		/* At this point, we're clear to send the request.  For cipher
432		 * requests, we need to check if the session IV needs to be
433		 * updated.  This can only be done when no other partials are in
434		 * flight for this session, to ensure the cipherPartialOpState
435		 * buffer in the session descriptor is not currently in use
436		 */
437		if (CPA_TRUE ==
438		    pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) {
439			if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
440				memcpy(pSessionDesc->cipherPartialOpState,
441				       pSessionDesc->cipherARC4InitialState,
442				       LAC_CIPHER_ARC4_STATE_LEN_BYTES);
443			} else {
444				memcpy(pSessionDesc->cipherPartialOpState,
445				       pSessionDesc->pRequestQueueHead->pOpData
446					   ->pIv,
447				       pSessionDesc->pRequestQueueHead->pOpData
448					   ->ivLenInBytes);
449			}
450		}
451
452		/*
453		 * Now we'll attempt to send the message directly to QAT. We'll
454		 * keep looing until it succeeds (or at least a very high number
455		 * of retries), as the failure only happens when the ring is
456		 * full, and this is only a temporary situation. After a few
457		 * retries, space will become availble, allowing the putMsg to
458		 * succeed.
459		 */
460		retries = 0;
461		do {
462			/* Send to QAT */
463			status = icp_adf_transPutMsg(
464			    pService->trans_handle_sym_tx,
465			    (void *)&(pSessionDesc->pRequestQueueHead->qatMsg),
466			    LAC_QAT_SYM_REQ_SZ_LW);
467
468			retries++;
469			/*
470			 * Yield to allow other threads that may be on this
471			 * session to poll and make some space on the ring
472			 */
473			if (CPA_STATUS_SUCCESS != status) {
474				qatUtilsYield();
475			}
476		} while ((CPA_STATUS_SUCCESS != status) &&
477			 (retries < DEQUEUE_MSGPUT_MAX_RETRIES));
478
479		if ((CPA_STATUS_SUCCESS != status) ||
480		    (retries >= DEQUEUE_MSGPUT_MAX_RETRIES)) {
481			LAC_LOG_ERROR(
482			    "Failed to SalQatMsg_transPutMsg, maximum retries exceeded.");
483			goto cleanup;
484		}
485
486		pSessionDesc->pRequestQueueHead =
487		    pSessionDesc->pRequestQueueHead->pNext;
488	}
489
490	/* If we've drained the queue, ensure the tail pointer is set to NULL */
491	if (NULL == pSessionDesc->pRequestQueueHead) {
492		pSessionDesc->pRequestQueueTail = NULL;
493	}
494
495cleanup:
496	LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock);
497	return status;
498}
499
500/**
501 * @ingroup LacSymCb
502 */
503void
504LacSymCb_CallbacksRegister(void)
505{
506	/*** HASH ***/
507	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_AUTH,
508				      LacSymCb_ProcessCallback);
509
510	/*** ALGORITHM-CHAINING CIPHER_HASH***/
511	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER_HASH,
512				      LacSymCb_ProcessCallback);
513
514	/*** ALGORITHM-CHAINING HASH_CIPHER***/
515	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_HASH_CIPHER,
516				      LacSymCb_ProcessCallback);
517
518	/*** CIPHER ***/
519	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER,
520				      LacSymCb_ProcessCallback);
521
522	/* Call compile time param check function to ensure it is included
523	   in the build by the compiler - this compile time check
524	   ensures callbacks run as expected */
525	LacSym_CompileTimeAssertions();
526}
527