1/* SPDX-License-Identifier: BSD-3-Clause */
2/* Copyright(c) 2007-2022 Intel Corporation */
3/**
4 *****************************************************************************
5 * @file lac_buffer_desc.c  Utility functions for setting buffer descriptors
6 *
7 * @ingroup LacBufferDesc
8 *
9 *****************************************************************************/
10
11/*
12*******************************************************************************
13* Include header files
14*******************************************************************************
15*/
16#include "qat_utils.h"
17#include "icp_accel_devices.h"
18#include "icp_adf_debug.h"
19#include "icp_adf_init.h"
20#include "lac_list.h"
21#include "lac_sal_types.h"
22#include "lac_buffer_desc.h"
23#include "lac_mem.h"
24#include "cpa_cy_common.h"
25
26/*
27*******************************************************************************
28* Define public/global function definitions
29*******************************************************************************
30*/
31/* Invalid physical address value */
32#define INVALID_PHYSICAL_ADDRESS 0
33
34/* Indicates what type of buffer writes need to be perfomed */
35typedef enum lac_buff_write_op_e {
36	WRITE_NORMAL = 0,
37	WRITE_AND_GET_SIZE,
38	WRITE_AND_ALLOW_ZERO_BUFFER,
39} lac_buff_write_op_t;
40
41/* This function implements the buffer description writes for the traditional
42 * APIs */
43static CpaStatus
44LacBuffDesc_CommonBufferListDescWrite(const CpaBufferList *pUserBufferList,
45				      Cpa64U *pBufListAlignedPhyAddr,
46				      CpaBoolean isPhysicalAddress,
47				      Cpa64U *totalDataLenInBytes,
48				      sal_service_t *pService,
49				      lac_buff_write_op_t operationType)
50{
51	Cpa32U numBuffers = 0;
52	icp_qat_addr_width_t bufListDescPhyAddr = 0;
53	icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
54	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
55	icp_buffer_list_desc_t *pBufferListDesc = NULL;
56	icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
57
58	if (WRITE_AND_GET_SIZE == operationType) {
59		*totalDataLenInBytes = 0;
60	}
61
62	numBuffers = pUserBufferList->numBuffers;
63	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
64
65	/*
66	 * Get the physical address of this descriptor - need to offset by the
67	 * alignment restrictions on the buffer descriptors
68	 */
69	bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
70	    (*pService), pUserBufferList->pPrivateMetaData);
71
72	if (bufListDescPhyAddr == 0) {
73		QAT_UTILS_LOG(
74		    "Unable to get the physical address of the metadata.\n");
75		return CPA_STATUS_FAIL;
76	}
77
78	bufListAlignedPhyAddr =
79	    LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
80				   ICP_DESCRIPTOR_ALIGNMENT_BYTES);
81
82	pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
83	    (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
84	    ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
85	     (LAC_ARCH_UINT)bufListDescPhyAddr));
86
87	/* Go past the Buffer List descriptor to the list of buffer descriptors
88	 */
89	pCurrFlatBufDesc =
90	    (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
91
92	pBufferListDesc->numBuffers = numBuffers;
93
94	if (WRITE_AND_GET_SIZE != operationType) {
95		/* Defining zero buffers is useful for example if running zero
96		 * length
97		 * hash */
98		if (0 == numBuffers) {
99			/* In the case where there are zero buffers within the
100			 * BufList
101			 * it is required by firmware that the number is set to
102			 * 1
103			 * but the phyBuffer and dataLenInBytes are set to
104			 * NULL.*/
105			pBufferListDesc->numBuffers = 1;
106			pCurrFlatBufDesc->dataLenInBytes = 0;
107			pCurrFlatBufDesc->phyBuffer = 0;
108		}
109	}
110
111	while (0 != numBuffers) {
112		pCurrFlatBufDesc->dataLenInBytes =
113		    pCurrClientFlatBuffer->dataLenInBytes;
114
115		if (WRITE_AND_GET_SIZE == operationType) {
116			/* Calculate the total data length in bytes */
117			*totalDataLenInBytes +=
118			    pCurrClientFlatBuffer->dataLenInBytes;
119		}
120
121		/* Check if providing a physical address in the function. If not
122		 * we
123		 * need to convert it to a physical one */
124		if (CPA_TRUE == isPhysicalAddress) {
125			pCurrFlatBufDesc->phyBuffer =
126			    LAC_MEM_CAST_PTR_TO_UINT64(
127				(LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
128		} else {
129			pCurrFlatBufDesc->phyBuffer =
130			    LAC_MEM_CAST_PTR_TO_UINT64(
131				LAC_OS_VIRT_TO_PHYS_EXTERNAL(
132				    (*pService), pCurrClientFlatBuffer->pData));
133
134			if (WRITE_AND_ALLOW_ZERO_BUFFER != operationType) {
135				if (INVALID_PHYSICAL_ADDRESS ==
136				    pCurrFlatBufDesc->phyBuffer) {
137					QAT_UTILS_LOG(
138					    "Unable to get the physical address of the client buffer.\n");
139					return CPA_STATUS_FAIL;
140				}
141			}
142		}
143
144		pCurrFlatBufDesc++;
145		pCurrClientFlatBuffer++;
146
147		numBuffers--;
148	}
149
150	*pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
151	return CPA_STATUS_SUCCESS;
152}
153
154/* This function implements the buffer description writes for the traditional
155 * APIs Zero length buffers are allowed, should be used for CHA-CHA-POLY and
156 * GCM aglorithms */
157CpaStatus
158LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
159    const CpaBufferList *pUserBufferList,
160    Cpa64U *pBufListAlignedPhyAddr,
161    CpaBoolean isPhysicalAddress,
162    sal_service_t *pService)
163{
164	return LacBuffDesc_CommonBufferListDescWrite(
165	    pUserBufferList,
166	    pBufListAlignedPhyAddr,
167	    isPhysicalAddress,
168	    NULL,
169	    pService,
170	    WRITE_AND_ALLOW_ZERO_BUFFER);
171}
172
173/* This function implements the buffer description writes for the traditional
174 * APIs */
175CpaStatus
176LacBuffDesc_BufferListDescWrite(const CpaBufferList *pUserBufferList,
177				Cpa64U *pBufListAlignedPhyAddr,
178				CpaBoolean isPhysicalAddress,
179				sal_service_t *pService)
180{
181	return LacBuffDesc_CommonBufferListDescWrite(pUserBufferList,
182						     pBufListAlignedPhyAddr,
183						     isPhysicalAddress,
184						     NULL,
185						     pService,
186						     WRITE_NORMAL);
187}
188
189/* This function does the same processing as LacBuffDesc_BufferListDescWrite
190 * but calculate as well the total length in bytes of the buffer list. */
191CpaStatus
192LacBuffDesc_BufferListDescWriteAndGetSize(const CpaBufferList *pUserBufferList,
193					  Cpa64U *pBufListAlignedPhyAddr,
194					  CpaBoolean isPhysicalAddress,
195					  Cpa64U *totalDataLenInBytes,
196					  sal_service_t *pService)
197{
198	Cpa32U numBuffers = 0;
199	icp_qat_addr_width_t bufListDescPhyAddr = 0;
200	icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
201	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
202	icp_buffer_list_desc_t *pBufferListDesc = NULL;
203	icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
204	*totalDataLenInBytes = 0;
205
206	numBuffers = pUserBufferList->numBuffers;
207	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
208
209	/*
210	 * Get the physical address of this descriptor - need to offset by the
211	 * alignment restrictions on the buffer descriptors
212	 */
213	bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
214	    (*pService), pUserBufferList->pPrivateMetaData);
215
216	if (INVALID_PHYSICAL_ADDRESS == bufListDescPhyAddr) {
217		QAT_UTILS_LOG(
218		    "Unable to get the physical address of the metadata.\n");
219		return CPA_STATUS_FAIL;
220	}
221
222	bufListAlignedPhyAddr =
223	    LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
224				   ICP_DESCRIPTOR_ALIGNMENT_BYTES);
225
226	pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
227	    (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
228	    ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
229	     (LAC_ARCH_UINT)bufListDescPhyAddr));
230
231	/* Go past the Buffer List descriptor to the list of buffer descriptors
232	 */
233	pCurrFlatBufDesc =
234	    (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
235
236	pBufferListDesc->numBuffers = numBuffers;
237
238	while (0 != numBuffers) {
239		pCurrFlatBufDesc->dataLenInBytes =
240		    pCurrClientFlatBuffer->dataLenInBytes;
241
242		/* Calculate the total data length in bytes */
243		*totalDataLenInBytes += pCurrClientFlatBuffer->dataLenInBytes;
244
245		if (isPhysicalAddress == CPA_TRUE) {
246			pCurrFlatBufDesc->phyBuffer =
247			    LAC_MEM_CAST_PTR_TO_UINT64(
248				(LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
249		} else {
250			pCurrFlatBufDesc->phyBuffer =
251			    LAC_MEM_CAST_PTR_TO_UINT64(
252				LAC_OS_VIRT_TO_PHYS_EXTERNAL(
253				    (*pService), pCurrClientFlatBuffer->pData));
254
255			if (pCurrFlatBufDesc->phyBuffer == 0) {
256				QAT_UTILS_LOG(
257				    "Unable to get the physical address of the client buffer.\n");
258				return CPA_STATUS_FAIL;
259			}
260		}
261
262		pCurrFlatBufDesc++;
263		pCurrClientFlatBuffer++;
264
265		numBuffers--;
266	}
267
268	*pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
269	return CPA_STATUS_SUCCESS;
270}
271
272CpaStatus
273LacBuffDesc_FlatBufferVerify(const CpaFlatBuffer *pUserFlatBuffer,
274			     Cpa64U *pPktSize,
275			     lac_aligment_shift_t alignmentShiftExpected)
276{
277	LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
278	LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
279
280	if (0 == pUserFlatBuffer->dataLenInBytes) {
281		QAT_UTILS_LOG("FlatBuffer empty\n");
282		return CPA_STATUS_INVALID_PARAM;
283	}
284
285	/* Expected alignment */
286	if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
287		if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
288					 alignmentShiftExpected)) {
289			QAT_UTILS_LOG(
290			    "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
291			    1 << alignmentShiftExpected);
292			return CPA_STATUS_INVALID_PARAM;
293		}
294	}
295
296	/* Update the total size of the packet. This function being called in a
297	 * loop
298	 * for an entire buffer list we need to increment the value */
299	*pPktSize += pUserFlatBuffer->dataLenInBytes;
300
301	return CPA_STATUS_SUCCESS;
302}
303
304CpaStatus
305LacBuffDesc_FlatBufferVerifyNull(const CpaFlatBuffer *pUserFlatBuffer,
306				 Cpa64U *pPktSize,
307				 lac_aligment_shift_t alignmentShiftExpected)
308{
309	LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
310
311	if (0 != pUserFlatBuffer->dataLenInBytes) {
312		LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
313	}
314
315	/* Expected alignment */
316	if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
317		if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
318					 alignmentShiftExpected)) {
319			QAT_UTILS_LOG(
320			    "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
321			    1 << alignmentShiftExpected);
322			return CPA_STATUS_INVALID_PARAM;
323		}
324	}
325
326	/* Update the total size of the packet. This function being called in a
327	 * loop
328	 * for an entire buffer list we need to increment the value */
329	*pPktSize += pUserFlatBuffer->dataLenInBytes;
330
331	return CPA_STATUS_SUCCESS;
332}
333
334CpaStatus
335LacBuffDesc_BufferListVerify(const CpaBufferList *pUserBufferList,
336			     Cpa64U *pPktSize,
337			     lac_aligment_shift_t alignmentShiftExpected)
338{
339	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
340	Cpa32U numBuffers = 0;
341	CpaStatus status = CPA_STATUS_SUCCESS;
342
343	LAC_CHECK_NULL_PARAM(pUserBufferList);
344	LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
345	LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
346
347	numBuffers = pUserBufferList->numBuffers;
348
349	if (0 == pUserBufferList->numBuffers) {
350		QAT_UTILS_LOG("Number of buffers is 0.\n");
351		return CPA_STATUS_INVALID_PARAM;
352	}
353
354	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
355
356	*pPktSize = 0;
357	while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
358		status = LacBuffDesc_FlatBufferVerify(pCurrClientFlatBuffer,
359						      pPktSize,
360						      alignmentShiftExpected);
361
362		pCurrClientFlatBuffer++;
363		numBuffers--;
364	}
365	return status;
366}
367
368CpaStatus
369LacBuffDesc_BufferListVerifyNull(const CpaBufferList *pUserBufferList,
370				 Cpa64U *pPktSize,
371				 lac_aligment_shift_t alignmentShiftExpected)
372{
373	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
374	Cpa32U numBuffers = 0;
375	CpaStatus status = CPA_STATUS_SUCCESS;
376
377	LAC_CHECK_NULL_PARAM(pUserBufferList);
378	LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
379	LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
380
381	numBuffers = pUserBufferList->numBuffers;
382
383	if (0 == pUserBufferList->numBuffers) {
384		QAT_UTILS_LOG("Number of buffers is 0.\n");
385		return CPA_STATUS_INVALID_PARAM;
386	}
387
388	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
389
390	*pPktSize = 0;
391	while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
392		status =
393		    LacBuffDesc_FlatBufferVerifyNull(pCurrClientFlatBuffer,
394						     pPktSize,
395						     alignmentShiftExpected);
396
397		pCurrClientFlatBuffer++;
398		numBuffers--;
399	}
400	return status;
401}
402
403/**
404 ******************************************************************************
405 * @ingroup LacBufferDesc
406 *****************************************************************************/
407void
408LacBuffDesc_BufferListTotalSizeGet(const CpaBufferList *pUserBufferList,
409				   Cpa64U *pPktSize)
410{
411	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
412	Cpa32U numBuffers = 0;
413
414	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
415	numBuffers = pUserBufferList->numBuffers;
416
417	*pPktSize = 0;
418	while (0 != numBuffers) {
419		*pPktSize += pCurrClientFlatBuffer->dataLenInBytes;
420		pCurrClientFlatBuffer++;
421		numBuffers--;
422	}
423}
424
425void
426LacBuffDesc_BufferListZeroFromOffset(CpaBufferList *pBuffList,
427				     Cpa32U offset,
428				     Cpa32U lenToZero)
429{
430	Cpa32U zeroLen = 0, sizeLeftToZero = 0;
431	Cpa64U currentBufferSize = 0;
432	CpaFlatBuffer *pBuffer = NULL;
433	Cpa8U *pZero = NULL;
434	pBuffer = pBuffList->pBuffers;
435
436	/* Take a copy of total length to zero. */
437	sizeLeftToZero = lenToZero;
438
439	while (sizeLeftToZero > 0) {
440		currentBufferSize = pBuffer->dataLenInBytes;
441		/* check where to start zeroing */
442		if (offset >= currentBufferSize) {
443			/* Need to get to next buffer and reduce
444			 * offset size by data len of buffer */
445			offset = offset - pBuffer->dataLenInBytes;
446			pBuffer++;
447		} else {
448			/* Start to Zero from this position */
449			pZero = (Cpa8U *)pBuffer->pData + offset;
450
451			/* Need to calculate the correct number of bytes to zero
452			 * for this iteration and for this location.
453			 */
454			if (sizeLeftToZero >= pBuffer->dataLenInBytes) {
455				/* The size to zero is spanning buffers, zeroLen
456				 * in
457				 * this case is from pZero (position) to end of
458				 * buffer.
459				 */
460				zeroLen = pBuffer->dataLenInBytes - offset;
461			} else {
462				/* zeroLen is set to sizeLeftToZero, then check
463				 * if zeroLen and
464				 * the offset is greater or equal to the size of
465				 * the buffer, if
466				 * yes, adjust the zeroLen to zero out the
467				 * remainder of this
468				 * buffer.
469				 */
470				zeroLen = sizeLeftToZero;
471				if ((zeroLen + offset) >=
472				    pBuffer->dataLenInBytes) {
473					zeroLen =
474					    pBuffer->dataLenInBytes - offset;
475				}
476			} /* end inner else */
477			memset((void *)pZero, 0, zeroLen);
478			sizeLeftToZero = sizeLeftToZero - zeroLen;
479			/* offset is no longer required as any data left to zero
480			 * is now
481			 * at the start of the next buffer. set offset to zero
482			 * and move on
483			 * the buffer pointer to the next buffer.
484			 */
485			offset = 0;
486			pBuffer++;
487
488		} /* end outer else */
489
490	} /* end while */
491}
492