1/* SPDX-License-Identifier: BSD-3-Clause */
2/* Copyright(c) 2007-2022 Intel Corporation */
3#include "adf_transport_access_macros.h"
4#include "adf_transport_internal.h"
5
6#include "cpa.h"
7#include "icp_adf_init.h"
8#include "icp_adf_transport.h"
9#include "icp_adf_poll.h"
10#include "icp_adf_transport_dp.h"
11#include "icp_sal_poll.h"
12
13/*
14 * adf_modulo
15 * result = data % ( 2 ^ shift )
16 */
17static inline Cpa32U
18adf_modulo(Cpa32U data, Cpa32U shift)
19{
20	Cpa32U div = data >> shift;
21	Cpa32U mult = div << shift;
22
23	return data - mult;
24}
25
26/*
27 * icp_adf_transCreateHandle
28 * crete transport handle for a service
29 * call adf_create_ring from adf driver directly with same parameters
30 */
31CpaStatus
32icp_adf_transCreateHandle(icp_accel_dev_t *adf,
33			  icp_transport_type trans_type,
34			  const char *section,
35			  const uint32_t accel_nr,
36			  const uint32_t bank_nr,
37			  const char *service_name,
38			  const icp_adf_ringInfoService_t info,
39			  icp_trans_callback callback,
40			  icp_resp_deliv_method resp,
41			  const uint32_t num_msgs,
42			  const uint32_t msg_size,
43			  icp_comms_trans_handle *trans_handle)
44{
45	CpaStatus status;
46	int error;
47
48	ICP_CHECK_FOR_NULL_PARAM(trans_handle);
49	ICP_CHECK_FOR_NULL_PARAM(adf);
50
51	error = adf_create_ring(adf->accel_dev,
52				section,
53				bank_nr,
54				num_msgs,
55				msg_size,
56				service_name,
57				callback,
58				((resp == ICP_RESP_TYPE_IRQ) ? 0 : 1),
59				(struct adf_etr_ring_data **)trans_handle);
60	if (!error)
61		status = CPA_STATUS_SUCCESS;
62	else
63		status = CPA_STATUS_FAIL;
64
65	return status;
66}
67
68/*
69 * icp_adf_transReinitHandle
70 * Reinitialize transport handle for a service
71 */
72CpaStatus
73icp_adf_transReinitHandle(icp_accel_dev_t *adf,
74			  icp_transport_type trans_type,
75			  const char *section,
76			  const uint32_t accel_nr,
77			  const uint32_t bank_nr,
78			  const char *service_name,
79			  const icp_adf_ringInfoService_t info,
80			  icp_trans_callback callback,
81			  icp_resp_deliv_method resp,
82			  const uint32_t num_msgs,
83			  const uint32_t msg_size,
84			  icp_comms_trans_handle *trans_handle)
85{
86	return CPA_STATUS_SUCCESS;
87}
88/*
89 * icp_adf_transReleaseHandle
90 * destroy a transport handle, call adf_remove_ring from adf driver directly
91 */
92CpaStatus
93icp_adf_transReleaseHandle(icp_comms_trans_handle trans_handle)
94{
95	struct adf_etr_ring_data *ring = trans_handle;
96
97	ICP_CHECK_FOR_NULL_PARAM(ring);
98	adf_remove_ring(ring);
99
100	return CPA_STATUS_SUCCESS;
101}
102
103/*
104 * icp_adf_transResetHandle
105 * clean a transport handle, call adf_remove_ring from adf driver directly
106 */
107CpaStatus
108icp_adf_transResetHandle(icp_comms_trans_handle trans_handle)
109{
110	return CPA_STATUS_SUCCESS;
111}
112
113/*
114 * icp_adf_transGetRingNum
115 * get ring number from a transport handle
116 */
117CpaStatus
118icp_adf_transGetRingNum(icp_comms_trans_handle trans_handle, uint32_t *ringNum)
119{
120	struct adf_etr_ring_data *ring = trans_handle;
121
122	ICP_CHECK_FOR_NULL_PARAM(ring);
123	ICP_CHECK_FOR_NULL_PARAM(ringNum);
124	*ringNum = (uint32_t)(ring->ring_number);
125
126	return CPA_STATUS_SUCCESS;
127}
128
129/*
130 * icp_adf_transPutMsg
131 * send a request to transport handle
132 * call adf_send_message from adf driver directly
133 */
134CpaStatus
135icp_adf_transPutMsg(icp_comms_trans_handle trans_handle,
136		    uint32_t *inBuf,
137		    uint32_t bufLen)
138{
139	struct adf_etr_ring_data *ring = trans_handle;
140	CpaStatus status = CPA_STATUS_FAIL;
141	int error = EFAULT;
142
143	ICP_CHECK_FOR_NULL_PARAM(ring);
144
145	error = adf_send_message(ring, inBuf);
146	if (EAGAIN == error)
147		status = CPA_STATUS_RETRY;
148	else if (0 == error)
149		status = CPA_STATUS_SUCCESS;
150	else
151		status = CPA_STATUS_FAIL;
152
153	return status;
154}
155
156CpaStatus
157icp_adf_getInflightRequests(icp_comms_trans_handle trans_handle,
158			    Cpa32U *maxInflightRequests,
159			    Cpa32U *numInflightRequests)
160{
161	struct adf_etr_ring_data *ring = trans_handle;
162	ICP_CHECK_FOR_NULL_PARAM(ring);
163	ICP_CHECK_FOR_NULL_PARAM(maxInflightRequests);
164	ICP_CHECK_FOR_NULL_PARAM(numInflightRequests);
165	/*
166	 * XXX: The qat_direct version of this routine returns max - 1, not
167	 * the absolute max.
168	 */
169	*numInflightRequests = (*(uint32_t *)ring->inflights);
170	*maxInflightRequests =
171	    ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);
172	return CPA_STATUS_SUCCESS;
173}
174
175CpaStatus
176icp_adf_dp_getInflightRequests(icp_comms_trans_handle trans_handle,
177			       Cpa32U *maxInflightRequests,
178			       Cpa32U *numInflightRequests)
179{
180	ICP_CHECK_FOR_NULL_PARAM(trans_handle);
181	ICP_CHECK_FOR_NULL_PARAM(maxInflightRequests);
182	ICP_CHECK_FOR_NULL_PARAM(numInflightRequests);
183
184	return icp_adf_getInflightRequests(trans_handle,
185					   maxInflightRequests,
186					   numInflightRequests);
187}
188
189/*
190 * This function allows the user to poll the response ring. The
191 * ring number to be polled is supplied by the user via the
192 * trans handle for that ring. The trans_hnd is a pointer
193 * to an array of trans handles. This ring is
194 * only polled if it contains data.
195 * This method is used as an alternative to the reading messages
196 * via the ISR method.
197 * This function will return RETRY if the ring is empty.
198 */
199CpaStatus
200icp_adf_pollInstance(icp_comms_trans_handle *trans_hnd,
201		     Cpa32U num_transHandles,
202		     Cpa32U response_quota)
203{
204	Cpa32U resp_total = 0;
205	Cpa32U num_resp;
206	struct adf_etr_ring_data *ring = NULL;
207	struct adf_etr_bank_data *bank = NULL;
208	Cpa32U i;
209
210	ICP_CHECK_FOR_NULL_PARAM(trans_hnd);
211
212	for (i = 0; i < num_transHandles; i++) {
213		ring = trans_hnd[i];
214		if (!ring)
215			continue;
216		bank = ring->bank;
217
218		/* If the ring in question is empty try the next ring.*/
219		if (!bank || !bank->ring_mask) {
220			continue;
221		}
222
223		num_resp = adf_handle_response(ring, response_quota);
224		resp_total += num_resp;
225	}
226
227	/* If any of the rings in the instance had data and was polled
228	 * return SUCCESS. */
229	if (resp_total)
230		return CPA_STATUS_SUCCESS;
231	else
232		return CPA_STATUS_RETRY;
233}
234
235/*
236 * This function allows the user to check the response ring. The
237 * ring number to be polled is supplied by the user via the
238 * trans handle for that ring. The trans_hnd is a pointer
239 * to an array of trans handles.
240 * This function now is a empty function.
241 */
242CpaStatus
243icp_adf_check_RespInstance(icp_comms_trans_handle *trans_hnd,
244			   Cpa32U num_transHandles)
245{
246	return CPA_STATUS_SUCCESS;
247}
248
249/*
250 * icp_sal_pollBank
251 * poll bank with id bank_number inside acceleration device with id @accelId
252 */
253CpaStatus
254icp_sal_pollBank(Cpa32U accelId, Cpa32U bank_number, Cpa32U response_quota)
255{
256	int ret;
257
258	ret = adf_poll_bank(accelId, bank_number, response_quota);
259	if (!ret)
260		return CPA_STATUS_SUCCESS;
261	else if (EAGAIN == ret)
262		return CPA_STATUS_RETRY;
263
264	return CPA_STATUS_FAIL;
265}
266
267/*
268 * icp_sal_pollAllBanks
269 * poll all banks inside acceleration device with id @accelId
270 */
271CpaStatus
272icp_sal_pollAllBanks(Cpa32U accelId, Cpa32U response_quota)
273{
274	int ret = 0;
275
276	ret = adf_poll_all_banks(accelId, response_quota);
277	if (!ret)
278		return CPA_STATUS_SUCCESS;
279	else if (ret == EAGAIN)
280		return CPA_STATUS_RETRY;
281
282	return CPA_STATUS_FAIL;
283}
284
285/*
286 * icp_adf_getQueueMemory
287 * Data plane support function - returns the pointer to next message on the ring
288 * or NULL if there is not enough space.
289 */
290void
291icp_adf_getQueueMemory(icp_comms_trans_handle trans_handle,
292		       Cpa32U numberRequests,
293		       void **pCurrentQatMsg)
294{
295	struct adf_etr_ring_data *ring = trans_handle;
296	Cpa64U flight;
297
298	ICP_CHECK_FOR_NULL_PARAM_VOID(ring);
299
300	/* Check if there is enough space in the ring */
301	flight = atomic_add_return(numberRequests, ring->inflights);
302	if (flight > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {
303		atomic_sub(numberRequests, ring->inflights);
304		*pCurrentQatMsg = NULL;
305		return;
306	}
307
308	/* We have enough space - get the address of next message */
309	*pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail);
310}
311
312/*
313 * icp_adf_getSingleQueueAddr
314 * Data plane support function - returns the pointer to next message on the ring
315 * or NULL if there is not enough space - it also updates the shadow tail copy.
316 */
317void
318icp_adf_getSingleQueueAddr(icp_comms_trans_handle trans_handle,
319			   void **pCurrentQatMsg)
320{
321	struct adf_etr_ring_data *ring = trans_handle;
322	Cpa64U flight;
323
324	ICP_CHECK_FOR_NULL_PARAM_VOID(ring);
325	ICP_CHECK_FOR_NULL_PARAM_VOID(pCurrentQatMsg);
326
327	/* Check if there is enough space in the ring */
328	flight = atomic_add_return(1, ring->inflights);
329	if (flight > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {
330		atomic_dec(ring->inflights);
331		*pCurrentQatMsg = NULL;
332		return;
333	}
334
335	/* We have enough space - get the address of next message */
336	*pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail);
337
338	/* Update the shadow tail */
339	ring->tail =
340	    adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size),
341		       ADF_RING_SIZE_MODULO(ring->ring_size));
342}
343
344/*
345 * icp_adf_getQueueNext
346 * Data plane support function - increments the tail pointer and returns
347 * the pointer to next message on the ring.
348 */
349void
350icp_adf_getQueueNext(icp_comms_trans_handle trans_handle, void **pCurrentQatMsg)
351{
352	struct adf_etr_ring_data *ring = trans_handle;
353
354	ICP_CHECK_FOR_NULL_PARAM_VOID(ring);
355	ICP_CHECK_FOR_NULL_PARAM_VOID(pCurrentQatMsg);
356
357	/* Increment tail to next message */
358	ring->tail =
359	    adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size),
360		       ADF_RING_SIZE_MODULO(ring->ring_size));
361
362	/* Get the address of next message */
363	*pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail);
364}
365
366/*
367 * icp_adf_updateQueueTail
368 * Data plane support function - Writes the tail shadow copy to the device.
369 */
370void
371icp_adf_updateQueueTail(icp_comms_trans_handle trans_handle)
372{
373	struct adf_etr_ring_data *ring = trans_handle;
374	struct adf_hw_csr_ops *csr_ops;
375
376	ICP_CHECK_FOR_NULL_PARAM_VOID(ring);
377	ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank);
378	ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank->accel_dev);
379
380	csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
381
382	ICP_CHECK_FOR_NULL_PARAM_VOID(csr_ops);
383
384	csr_ops->write_csr_ring_tail(ring->bank->csr_addr,
385				     ring->bank->bank_number,
386				     ring->ring_number,
387				     ring->tail);
388	ring->csr_tail_offset = ring->tail;
389}
390
391/*
392 * icp_adf_pollQueue
393 * Data plane support function - Poll messages from the queue.
394 */
395CpaStatus
396icp_adf_pollQueue(icp_comms_trans_handle trans_handle, Cpa32U response_quota)
397{
398	Cpa32U num_resp;
399	struct adf_etr_ring_data *ring = trans_handle;
400
401	ICP_CHECK_FOR_NULL_PARAM(ring);
402
403	num_resp = adf_handle_response(ring, response_quota);
404
405	if (num_resp)
406		return CPA_STATUS_SUCCESS;
407	else
408		return CPA_STATUS_RETRY;
409}
410
411/*
412 * icp_adf_queueDataToSend
413 * Data-plane support function - Indicates if there is data on the ring to be
414 * sent. This should only be called on request rings. If the function returns
415 * true then it is ok to call icp_adf_updateQueueTail() function on this ring.
416 */
417CpaBoolean
418icp_adf_queueDataToSend(icp_comms_trans_handle trans_handle)
419{
420	struct adf_etr_ring_data *ring = trans_handle;
421
422	if (ring->tail != ring->csr_tail_offset)
423		return CPA_TRUE;
424	else
425		return CPA_FALSE;
426}
427
428/*
429 * This icp API won't be supported in kernel space currently
430 */
431CpaStatus
432icp_adf_transGetFdForHandle(icp_comms_trans_handle trans_hnd, int *fd)
433{
434	return CPA_STATUS_UNSUPPORTED;
435}
436