1/* SPDX-License-Identifier: BSD-3-Clause */ 2/* Copyright(c) 2007-2022 Intel Corporation */ 3#include "adf_transport_access_macros.h" 4#include "adf_transport_internal.h" 5 6#include "cpa.h" 7#include "icp_adf_init.h" 8#include "icp_adf_transport.h" 9#include "icp_adf_poll.h" 10#include "icp_adf_transport_dp.h" 11#include "icp_sal_poll.h" 12 13/* 14 * adf_modulo 15 * result = data % ( 2 ^ shift ) 16 */ 17static inline Cpa32U 18adf_modulo(Cpa32U data, Cpa32U shift) 19{ 20 Cpa32U div = data >> shift; 21 Cpa32U mult = div << shift; 22 23 return data - mult; 24} 25 26/* 27 * icp_adf_transCreateHandle 28 * crete transport handle for a service 29 * call adf_create_ring from adf driver directly with same parameters 30 */ 31CpaStatus 32icp_adf_transCreateHandle(icp_accel_dev_t *adf, 33 icp_transport_type trans_type, 34 const char *section, 35 const uint32_t accel_nr, 36 const uint32_t bank_nr, 37 const char *service_name, 38 const icp_adf_ringInfoService_t info, 39 icp_trans_callback callback, 40 icp_resp_deliv_method resp, 41 const uint32_t num_msgs, 42 const uint32_t msg_size, 43 icp_comms_trans_handle *trans_handle) 44{ 45 CpaStatus status; 46 int error; 47 48 ICP_CHECK_FOR_NULL_PARAM(trans_handle); 49 ICP_CHECK_FOR_NULL_PARAM(adf); 50 51 error = adf_create_ring(adf->accel_dev, 52 section, 53 bank_nr, 54 num_msgs, 55 msg_size, 56 service_name, 57 callback, 58 ((resp == ICP_RESP_TYPE_IRQ) ? 0 : 1), 59 (struct adf_etr_ring_data **)trans_handle); 60 if (!error) 61 status = CPA_STATUS_SUCCESS; 62 else 63 status = CPA_STATUS_FAIL; 64 65 return status; 66} 67 68/* 69 * icp_adf_transReinitHandle 70 * Reinitialize transport handle for a service 71 */ 72CpaStatus 73icp_adf_transReinitHandle(icp_accel_dev_t *adf, 74 icp_transport_type trans_type, 75 const char *section, 76 const uint32_t accel_nr, 77 const uint32_t bank_nr, 78 const char *service_name, 79 const icp_adf_ringInfoService_t info, 80 icp_trans_callback callback, 81 icp_resp_deliv_method resp, 82 const uint32_t num_msgs, 83 const uint32_t msg_size, 84 icp_comms_trans_handle *trans_handle) 85{ 86 return CPA_STATUS_SUCCESS; 87} 88/* 89 * icp_adf_transReleaseHandle 90 * destroy a transport handle, call adf_remove_ring from adf driver directly 91 */ 92CpaStatus 93icp_adf_transReleaseHandle(icp_comms_trans_handle trans_handle) 94{ 95 struct adf_etr_ring_data *ring = trans_handle; 96 97 ICP_CHECK_FOR_NULL_PARAM(ring); 98 adf_remove_ring(ring); 99 100 return CPA_STATUS_SUCCESS; 101} 102 103/* 104 * icp_adf_transResetHandle 105 * clean a transport handle, call adf_remove_ring from adf driver directly 106 */ 107CpaStatus 108icp_adf_transResetHandle(icp_comms_trans_handle trans_handle) 109{ 110 return CPA_STATUS_SUCCESS; 111} 112 113/* 114 * icp_adf_transGetRingNum 115 * get ring number from a transport handle 116 */ 117CpaStatus 118icp_adf_transGetRingNum(icp_comms_trans_handle trans_handle, uint32_t *ringNum) 119{ 120 struct adf_etr_ring_data *ring = trans_handle; 121 122 ICP_CHECK_FOR_NULL_PARAM(ring); 123 ICP_CHECK_FOR_NULL_PARAM(ringNum); 124 *ringNum = (uint32_t)(ring->ring_number); 125 126 return CPA_STATUS_SUCCESS; 127} 128 129/* 130 * icp_adf_transPutMsg 131 * send a request to transport handle 132 * call adf_send_message from adf driver directly 133 */ 134CpaStatus 135icp_adf_transPutMsg(icp_comms_trans_handle trans_handle, 136 uint32_t *inBuf, 137 uint32_t bufLen) 138{ 139 struct adf_etr_ring_data *ring = trans_handle; 140 CpaStatus status = CPA_STATUS_FAIL; 141 int error = EFAULT; 142 143 ICP_CHECK_FOR_NULL_PARAM(ring); 144 145 error = adf_send_message(ring, inBuf); 146 if (EAGAIN == error) 147 status = CPA_STATUS_RETRY; 148 else if (0 == error) 149 status = CPA_STATUS_SUCCESS; 150 else 151 status = CPA_STATUS_FAIL; 152 153 return status; 154} 155 156CpaStatus 157icp_adf_getInflightRequests(icp_comms_trans_handle trans_handle, 158 Cpa32U *maxInflightRequests, 159 Cpa32U *numInflightRequests) 160{ 161 struct adf_etr_ring_data *ring = trans_handle; 162 ICP_CHECK_FOR_NULL_PARAM(ring); 163 ICP_CHECK_FOR_NULL_PARAM(maxInflightRequests); 164 ICP_CHECK_FOR_NULL_PARAM(numInflightRequests); 165 /* 166 * XXX: The qat_direct version of this routine returns max - 1, not 167 * the absolute max. 168 */ 169 *numInflightRequests = (*(uint32_t *)ring->inflights); 170 *maxInflightRequests = 171 ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size); 172 return CPA_STATUS_SUCCESS; 173} 174 175CpaStatus 176icp_adf_dp_getInflightRequests(icp_comms_trans_handle trans_handle, 177 Cpa32U *maxInflightRequests, 178 Cpa32U *numInflightRequests) 179{ 180 ICP_CHECK_FOR_NULL_PARAM(trans_handle); 181 ICP_CHECK_FOR_NULL_PARAM(maxInflightRequests); 182 ICP_CHECK_FOR_NULL_PARAM(numInflightRequests); 183 184 return icp_adf_getInflightRequests(trans_handle, 185 maxInflightRequests, 186 numInflightRequests); 187} 188 189/* 190 * This function allows the user to poll the response ring. The 191 * ring number to be polled is supplied by the user via the 192 * trans handle for that ring. The trans_hnd is a pointer 193 * to an array of trans handles. This ring is 194 * only polled if it contains data. 195 * This method is used as an alternative to the reading messages 196 * via the ISR method. 197 * This function will return RETRY if the ring is empty. 198 */ 199CpaStatus 200icp_adf_pollInstance(icp_comms_trans_handle *trans_hnd, 201 Cpa32U num_transHandles, 202 Cpa32U response_quota) 203{ 204 Cpa32U resp_total = 0; 205 Cpa32U num_resp; 206 struct adf_etr_ring_data *ring = NULL; 207 struct adf_etr_bank_data *bank = NULL; 208 Cpa32U i; 209 210 ICP_CHECK_FOR_NULL_PARAM(trans_hnd); 211 212 for (i = 0; i < num_transHandles; i++) { 213 ring = trans_hnd[i]; 214 if (!ring) 215 continue; 216 bank = ring->bank; 217 218 /* If the ring in question is empty try the next ring.*/ 219 if (!bank || !bank->ring_mask) { 220 continue; 221 } 222 223 num_resp = adf_handle_response(ring, response_quota); 224 resp_total += num_resp; 225 } 226 227 /* If any of the rings in the instance had data and was polled 228 * return SUCCESS. */ 229 if (resp_total) 230 return CPA_STATUS_SUCCESS; 231 else 232 return CPA_STATUS_RETRY; 233} 234 235/* 236 * This function allows the user to check the response ring. The 237 * ring number to be polled is supplied by the user via the 238 * trans handle for that ring. The trans_hnd is a pointer 239 * to an array of trans handles. 240 * This function now is a empty function. 241 */ 242CpaStatus 243icp_adf_check_RespInstance(icp_comms_trans_handle *trans_hnd, 244 Cpa32U num_transHandles) 245{ 246 return CPA_STATUS_SUCCESS; 247} 248 249/* 250 * icp_sal_pollBank 251 * poll bank with id bank_number inside acceleration device with id @accelId 252 */ 253CpaStatus 254icp_sal_pollBank(Cpa32U accelId, Cpa32U bank_number, Cpa32U response_quota) 255{ 256 int ret; 257 258 ret = adf_poll_bank(accelId, bank_number, response_quota); 259 if (!ret) 260 return CPA_STATUS_SUCCESS; 261 else if (EAGAIN == ret) 262 return CPA_STATUS_RETRY; 263 264 return CPA_STATUS_FAIL; 265} 266 267/* 268 * icp_sal_pollAllBanks 269 * poll all banks inside acceleration device with id @accelId 270 */ 271CpaStatus 272icp_sal_pollAllBanks(Cpa32U accelId, Cpa32U response_quota) 273{ 274 int ret = 0; 275 276 ret = adf_poll_all_banks(accelId, response_quota); 277 if (!ret) 278 return CPA_STATUS_SUCCESS; 279 else if (ret == EAGAIN) 280 return CPA_STATUS_RETRY; 281 282 return CPA_STATUS_FAIL; 283} 284 285/* 286 * icp_adf_getQueueMemory 287 * Data plane support function - returns the pointer to next message on the ring 288 * or NULL if there is not enough space. 289 */ 290void 291icp_adf_getQueueMemory(icp_comms_trans_handle trans_handle, 292 Cpa32U numberRequests, 293 void **pCurrentQatMsg) 294{ 295 struct adf_etr_ring_data *ring = trans_handle; 296 Cpa64U flight; 297 298 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 299 300 /* Check if there is enough space in the ring */ 301 flight = atomic_add_return(numberRequests, ring->inflights); 302 if (flight > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { 303 atomic_sub(numberRequests, ring->inflights); 304 *pCurrentQatMsg = NULL; 305 return; 306 } 307 308 /* We have enough space - get the address of next message */ 309 *pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail); 310} 311 312/* 313 * icp_adf_getSingleQueueAddr 314 * Data plane support function - returns the pointer to next message on the ring 315 * or NULL if there is not enough space - it also updates the shadow tail copy. 316 */ 317void 318icp_adf_getSingleQueueAddr(icp_comms_trans_handle trans_handle, 319 void **pCurrentQatMsg) 320{ 321 struct adf_etr_ring_data *ring = trans_handle; 322 Cpa64U flight; 323 324 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 325 ICP_CHECK_FOR_NULL_PARAM_VOID(pCurrentQatMsg); 326 327 /* Check if there is enough space in the ring */ 328 flight = atomic_add_return(1, ring->inflights); 329 if (flight > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { 330 atomic_dec(ring->inflights); 331 *pCurrentQatMsg = NULL; 332 return; 333 } 334 335 /* We have enough space - get the address of next message */ 336 *pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail); 337 338 /* Update the shadow tail */ 339 ring->tail = 340 adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), 341 ADF_RING_SIZE_MODULO(ring->ring_size)); 342} 343 344/* 345 * icp_adf_getQueueNext 346 * Data plane support function - increments the tail pointer and returns 347 * the pointer to next message on the ring. 348 */ 349void 350icp_adf_getQueueNext(icp_comms_trans_handle trans_handle, void **pCurrentQatMsg) 351{ 352 struct adf_etr_ring_data *ring = trans_handle; 353 354 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 355 ICP_CHECK_FOR_NULL_PARAM_VOID(pCurrentQatMsg); 356 357 /* Increment tail to next message */ 358 ring->tail = 359 adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), 360 ADF_RING_SIZE_MODULO(ring->ring_size)); 361 362 /* Get the address of next message */ 363 *pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail); 364} 365 366/* 367 * icp_adf_updateQueueTail 368 * Data plane support function - Writes the tail shadow copy to the device. 369 */ 370void 371icp_adf_updateQueueTail(icp_comms_trans_handle trans_handle) 372{ 373 struct adf_etr_ring_data *ring = trans_handle; 374 struct adf_hw_csr_ops *csr_ops; 375 376 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 377 ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank); 378 ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank->accel_dev); 379 380 csr_ops = GET_CSR_OPS(ring->bank->accel_dev); 381 382 ICP_CHECK_FOR_NULL_PARAM_VOID(csr_ops); 383 384 csr_ops->write_csr_ring_tail(ring->bank->csr_addr, 385 ring->bank->bank_number, 386 ring->ring_number, 387 ring->tail); 388 ring->csr_tail_offset = ring->tail; 389} 390 391/* 392 * icp_adf_pollQueue 393 * Data plane support function - Poll messages from the queue. 394 */ 395CpaStatus 396icp_adf_pollQueue(icp_comms_trans_handle trans_handle, Cpa32U response_quota) 397{ 398 Cpa32U num_resp; 399 struct adf_etr_ring_data *ring = trans_handle; 400 401 ICP_CHECK_FOR_NULL_PARAM(ring); 402 403 num_resp = adf_handle_response(ring, response_quota); 404 405 if (num_resp) 406 return CPA_STATUS_SUCCESS; 407 else 408 return CPA_STATUS_RETRY; 409} 410 411/* 412 * icp_adf_queueDataToSend 413 * Data-plane support function - Indicates if there is data on the ring to be 414 * sent. This should only be called on request rings. If the function returns 415 * true then it is ok to call icp_adf_updateQueueTail() function on this ring. 416 */ 417CpaBoolean 418icp_adf_queueDataToSend(icp_comms_trans_handle trans_handle) 419{ 420 struct adf_etr_ring_data *ring = trans_handle; 421 422 if (ring->tail != ring->csr_tail_offset) 423 return CPA_TRUE; 424 else 425 return CPA_FALSE; 426} 427 428/* 429 * This icp API won't be supported in kernel space currently 430 */ 431CpaStatus 432icp_adf_transGetFdForHandle(icp_comms_trans_handle trans_hnd, int *fd) 433{ 434 return CPA_STATUS_UNSUPPORTED; 435} 436