1/*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34#include "ena_com.h" 35#ifdef ENA_INTERNAL 36#include "ena_gen_info.h" 37#endif 38 39/*****************************************************************************/ 40/*****************************************************************************/ 41 42/* Timeout in micro-sec */ 43#define ADMIN_CMD_TIMEOUT_US (3000000) 44 45#define ENA_ASYNC_QUEUE_DEPTH 16 46#define ENA_ADMIN_QUEUE_DEPTH 32 47 48#ifdef ENA_EXTENDED_STATS 49 50#define ENA_HISTOGRAM_ACTIVE_MASK_OFFSET 0xF08 51#define ENA_EXTENDED_STAT_GET_FUNCT(_funct_queue) (_funct_queue & 0xFFFF) 52#define ENA_EXTENDED_STAT_GET_QUEUE(_funct_queue) (_funct_queue >> 16) 53 54#endif /* ENA_EXTENDED_STATS */ 55#define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \ 56 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \ 57 | (ENA_COMMON_SPEC_VERSION_MINOR)) 58 59#define ENA_CTRL_MAJOR 0 60#define ENA_CTRL_MINOR 0 61#define ENA_CTRL_SUB_MINOR 1 62 63#define MIN_ENA_CTRL_VER \ 64 (((ENA_CTRL_MAJOR) << \ 65 (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \ 66 ((ENA_CTRL_MINOR) << \ 67 (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \ 68 (ENA_CTRL_SUB_MINOR)) 69 70#define ENA_DMA_ADDR_TO_UINT32_LOW(x) ((u32)((u64)(x))) 71#define ENA_DMA_ADDR_TO_UINT32_HIGH(x) ((u32)(((u64)(x)) >> 32)) 72 73#define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF 74 75#define ENA_COM_BOUNCE_BUFFER_CNTRL_CNT 4 76 77#define ENA_REGS_ADMIN_INTR_MASK 1 78 79/*****************************************************************************/ 80/*****************************************************************************/ 81/*****************************************************************************/ 82 83enum ena_cmd_status { 84 ENA_CMD_SUBMITTED, 85 ENA_CMD_COMPLETED, 86 /* Abort - canceled by the driver */ 87 ENA_CMD_ABORTED, 88}; 89 90struct ena_comp_ctx { 91 ena_wait_event_t wait_event; 92 struct ena_admin_acq_entry *user_cqe; 93 u32 comp_size; 94 enum ena_cmd_status status; 95 /* status from the device */ 96 u8 comp_status; 97 u8 cmd_opcode; 98 bool occupied; 99}; 100 101struct ena_com_stats_ctx { 102 struct ena_admin_aq_get_stats_cmd get_cmd; 103 struct ena_admin_acq_get_stats_resp get_resp; 104}; 105 106static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev, 107 struct ena_common_mem_addr *ena_addr, 108 dma_addr_t addr) 109{ 110 if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) { 111 ena_trc_err("dma address has more bits that the device supports\n"); 112 return ENA_COM_INVAL; 113 } 114 115 ena_addr->mem_addr_low = (u32)addr; 116 ena_addr->mem_addr_high = (u16)((u64)addr >> 32); 117 118 return 0; 119} 120 121static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue) 122{ 123 struct ena_com_admin_sq *sq = &queue->sq; 124 u16 size = ADMIN_SQ_SIZE(queue->q_depth); 125 126 ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, size, sq->entries, sq->dma_addr, 127 sq->mem_handle); 128 129 if (!sq->entries) { 130 ena_trc_err("memory allocation failed"); 131 return ENA_COM_NO_MEM; 132 } 133 134 sq->head = 0; 135 sq->tail = 0; 136 sq->phase = 1; 137 138 sq->db_addr = NULL; 139 140 return 0; 141} 142 143static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue) 144{ 145 struct ena_com_admin_cq *cq = &queue->cq; 146 u16 size = ADMIN_CQ_SIZE(queue->q_depth); 147 148 ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, size, cq->entries, cq->dma_addr, 149 cq->mem_handle); 150 151 if (!cq->entries) { 152 ena_trc_err("memory allocation failed"); 153 return ENA_COM_NO_MEM; 154 } 155 156 cq->head = 0; 157 cq->phase = 1; 158 159 return 0; 160} 161 162static int ena_com_admin_init_aenq(struct ena_com_dev *dev, 163 struct ena_aenq_handlers *aenq_handlers) 164{ 165 struct ena_com_aenq *aenq = &dev->aenq; 166 u32 addr_low, addr_high, aenq_caps; 167 u16 size; 168 169 dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH; 170 size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH); 171 ENA_MEM_ALLOC_COHERENT(dev->dmadev, size, 172 aenq->entries, 173 aenq->dma_addr, 174 aenq->mem_handle); 175 176 if (!aenq->entries) { 177 ena_trc_err("memory allocation failed"); 178 return ENA_COM_NO_MEM; 179 } 180 181 aenq->head = aenq->q_depth; 182 aenq->phase = 1; 183 184 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr); 185 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr); 186 187 ENA_REG_WRITE32(dev->bus, addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF); 188 ENA_REG_WRITE32(dev->bus, addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF); 189 190 aenq_caps = 0; 191 aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK; 192 aenq_caps |= (sizeof(struct ena_admin_aenq_entry) << 193 ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) & 194 ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK; 195 ENA_REG_WRITE32(dev->bus, aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF); 196 197 if (unlikely(!aenq_handlers)) { 198 ena_trc_err("aenq handlers pointer is NULL\n"); 199 return ENA_COM_INVAL; 200 } 201 202 aenq->aenq_handlers = aenq_handlers; 203 204 return 0; 205} 206 207static inline void comp_ctxt_release(struct ena_com_admin_queue *queue, 208 struct ena_comp_ctx *comp_ctx) 209{ 210 comp_ctx->occupied = false; 211 ATOMIC32_DEC(&queue->outstanding_cmds); 212} 213 214static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue, 215 u16 command_id, bool capture) 216{ 217 if (unlikely(command_id >= queue->q_depth)) { 218 ena_trc_err("command id is larger than the queue size. cmd_id: %u queue size %d\n", 219 command_id, queue->q_depth); 220 return NULL; 221 } 222 223 if (unlikely(queue->comp_ctx[command_id].occupied && capture)) { 224 ena_trc_err("Completion context is occupied\n"); 225 return NULL; 226 } 227 228 if (capture) { 229 ATOMIC32_INC(&queue->outstanding_cmds); 230 queue->comp_ctx[command_id].occupied = true; 231 } 232 233 return &queue->comp_ctx[command_id]; 234} 235 236static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, 237 struct ena_admin_aq_entry *cmd, 238 size_t cmd_size_in_bytes, 239 struct ena_admin_acq_entry *comp, 240 size_t comp_size_in_bytes) 241{ 242 struct ena_comp_ctx *comp_ctx; 243 u16 tail_masked, cmd_id; 244 u16 queue_size_mask; 245 u16 cnt; 246 247 queue_size_mask = admin_queue->q_depth - 1; 248 249 tail_masked = admin_queue->sq.tail & queue_size_mask; 250 251 /* In case of queue FULL */ 252 cnt = ATOMIC32_READ(&admin_queue->outstanding_cmds); 253 if (cnt >= admin_queue->q_depth) { 254 ena_trc_dbg("admin queue is full.\n"); 255 admin_queue->stats.out_of_space++; 256 return ERR_PTR(ENA_COM_NO_SPACE); 257 } 258 259 cmd_id = admin_queue->curr_cmd_id; 260 261 cmd->aq_common_descriptor.flags |= admin_queue->sq.phase & 262 ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; 263 264 cmd->aq_common_descriptor.command_id |= cmd_id & 265 ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; 266 267 comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true); 268 if (unlikely(!comp_ctx)) 269 return ERR_PTR(ENA_COM_INVAL); 270 271 comp_ctx->status = ENA_CMD_SUBMITTED; 272 comp_ctx->comp_size = (u32)comp_size_in_bytes; 273 comp_ctx->user_cqe = comp; 274 comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode; 275 276 ENA_WAIT_EVENT_CLEAR(comp_ctx->wait_event); 277 278 memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes); 279 280 admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) & 281 queue_size_mask; 282 283 admin_queue->sq.tail++; 284 admin_queue->stats.submitted_cmd++; 285 286 if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0)) 287 admin_queue->sq.phase = !admin_queue->sq.phase; 288 289 ENA_DB_SYNC(&admin_queue->sq.mem_handle); 290 ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail, 291 admin_queue->sq.db_addr); 292 293 return comp_ctx; 294} 295 296static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue) 297{ 298 size_t size = queue->q_depth * sizeof(struct ena_comp_ctx); 299 struct ena_comp_ctx *comp_ctx; 300 u16 i; 301 302 queue->comp_ctx = ENA_MEM_ALLOC(queue->q_dmadev, size); 303 if (unlikely(!queue->comp_ctx)) { 304 ena_trc_err("memory allocation failed"); 305 return ENA_COM_NO_MEM; 306 } 307 308 for (i = 0; i < queue->q_depth; i++) { 309 comp_ctx = get_comp_ctxt(queue, i, false); 310 if (comp_ctx) 311 ENA_WAIT_EVENT_INIT(comp_ctx->wait_event); 312 } 313 314 return 0; 315} 316 317static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, 318 struct ena_admin_aq_entry *cmd, 319 size_t cmd_size_in_bytes, 320 struct ena_admin_acq_entry *comp, 321 size_t comp_size_in_bytes) 322{ 323 unsigned long flags; 324 struct ena_comp_ctx *comp_ctx; 325 326 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 327 if (unlikely(!admin_queue->running_state)) { 328 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 329 return ERR_PTR(ENA_COM_NO_DEVICE); 330 } 331 comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd, 332 cmd_size_in_bytes, 333 comp, 334 comp_size_in_bytes); 335 if (unlikely(IS_ERR(comp_ctx))) 336 admin_queue->running_state = false; 337 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 338 339 return comp_ctx; 340} 341 342static int ena_com_init_io_sq(struct ena_com_dev *ena_dev, 343 struct ena_com_create_io_ctx *ctx, 344 struct ena_com_io_sq *io_sq) 345{ 346 size_t size; 347 int dev_node = 0; 348 349 memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr)); 350 351 io_sq->desc_entry_size = 352 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? 353 sizeof(struct ena_eth_io_tx_desc) : 354 sizeof(struct ena_eth_io_rx_desc); 355 356 size = io_sq->desc_entry_size * io_sq->q_depth; 357 io_sq->bus = ena_dev->bus; 358 359 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { 360 ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev, 361 size, 362 io_sq->desc_addr.virt_addr, 363 io_sq->desc_addr.phys_addr, 364 io_sq->desc_addr.mem_handle, 365 ctx->numa_node, 366 dev_node); 367 if (!io_sq->desc_addr.virt_addr) { 368 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 369 size, 370 io_sq->desc_addr.virt_addr, 371 io_sq->desc_addr.phys_addr, 372 io_sq->desc_addr.mem_handle); 373 } 374 375 if (!io_sq->desc_addr.virt_addr) { 376 ena_trc_err("memory allocation failed"); 377 return ENA_COM_NO_MEM; 378 } 379 } 380 381 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 382 /* Allocate bounce buffers */ 383 io_sq->bounce_buf_ctrl.buffer_size = ena_dev->llq_info.desc_list_entry_size; 384 io_sq->bounce_buf_ctrl.buffers_num = ENA_COM_BOUNCE_BUFFER_CNTRL_CNT; 385 io_sq->bounce_buf_ctrl.next_to_use = 0; 386 387 size = io_sq->bounce_buf_ctrl.buffer_size * io_sq->bounce_buf_ctrl.buffers_num; 388 389 ENA_MEM_ALLOC_NODE(ena_dev->dmadev, 390 size, 391 io_sq->bounce_buf_ctrl.base_buffer, 392 ctx->numa_node, 393 dev_node); 394 if (!io_sq->bounce_buf_ctrl.base_buffer) 395 io_sq->bounce_buf_ctrl.base_buffer = ENA_MEM_ALLOC(ena_dev->dmadev, size); 396 397 if (!io_sq->bounce_buf_ctrl.base_buffer) { 398 ena_trc_err("bounce buffer memory allocation failed"); 399 return ENA_COM_NO_MEM; 400 } 401 402 memcpy(&io_sq->llq_info, &ena_dev->llq_info, sizeof(io_sq->llq_info)); 403 404 /* Initiate the first bounce buffer */ 405 io_sq->llq_buf_ctrl.curr_bounce_buf = 406 ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); 407 memset(io_sq->llq_buf_ctrl.curr_bounce_buf, 408 0x0, io_sq->llq_info.desc_list_entry_size); 409 io_sq->llq_buf_ctrl.descs_left_in_line = 410 io_sq->llq_info.descs_num_before_header; 411 } 412 413 io_sq->tail = 0; 414 io_sq->next_to_comp = 0; 415 io_sq->phase = 1; 416 417 return 0; 418} 419 420static int ena_com_init_io_cq(struct ena_com_dev *ena_dev, 421 struct ena_com_create_io_ctx *ctx, 422 struct ena_com_io_cq *io_cq) 423{ 424 size_t size; 425 int prev_node = 0; 426 427 memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr)); 428 429 /* Use the basic completion descriptor for Rx */ 430 io_cq->cdesc_entry_size_in_bytes = 431 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? 432 sizeof(struct ena_eth_io_tx_cdesc) : 433 sizeof(struct ena_eth_io_rx_cdesc_base); 434 435 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; 436 io_cq->bus = ena_dev->bus; 437 438 ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev, 439 size, 440 io_cq->cdesc_addr.virt_addr, 441 io_cq->cdesc_addr.phys_addr, 442 io_cq->cdesc_addr.mem_handle, 443 ctx->numa_node, 444 prev_node); 445 if (!io_cq->cdesc_addr.virt_addr) { 446 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 447 size, 448 io_cq->cdesc_addr.virt_addr, 449 io_cq->cdesc_addr.phys_addr, 450 io_cq->cdesc_addr.mem_handle); 451 } 452 453 if (!io_cq->cdesc_addr.virt_addr) { 454 ena_trc_err("memory allocation failed"); 455 return ENA_COM_NO_MEM; 456 } 457 458 io_cq->phase = 1; 459 io_cq->head = 0; 460 461 return 0; 462} 463 464static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue, 465 struct ena_admin_acq_entry *cqe) 466{ 467 struct ena_comp_ctx *comp_ctx; 468 u16 cmd_id; 469 470 cmd_id = cqe->acq_common_descriptor.command & 471 ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; 472 473 comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false); 474 if (unlikely(!comp_ctx)) { 475 ena_trc_err("comp_ctx is NULL. Changing the admin queue running state\n"); 476 admin_queue->running_state = false; 477 return; 478 } 479 480 comp_ctx->status = ENA_CMD_COMPLETED; 481 comp_ctx->comp_status = cqe->acq_common_descriptor.status; 482 483 if (comp_ctx->user_cqe) 484 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size); 485 486 if (!admin_queue->polling) 487 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event); 488} 489 490static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue) 491{ 492 struct ena_admin_acq_entry *cqe = NULL; 493 u16 comp_num = 0; 494 u16 head_masked; 495 u8 phase; 496 497 head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1); 498 phase = admin_queue->cq.phase; 499 500 cqe = &admin_queue->cq.entries[head_masked]; 501 502 /* Go over all the completions */ 503 while ((cqe->acq_common_descriptor.flags & 504 ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) { 505 /* Do not read the rest of the completion entry before the 506 * phase bit was validated 507 */ 508 rmb(); 509 ena_com_handle_single_admin_completion(admin_queue, cqe); 510 511 head_masked++; 512 comp_num++; 513 if (unlikely(head_masked == admin_queue->q_depth)) { 514 head_masked = 0; 515 phase = !phase; 516 } 517 518 cqe = &admin_queue->cq.entries[head_masked]; 519 } 520 521 admin_queue->cq.head += comp_num; 522 admin_queue->cq.phase = phase; 523 admin_queue->sq.head += comp_num; 524 admin_queue->stats.completed_cmd += comp_num; 525} 526 527static int ena_com_comp_status_to_errno(u8 comp_status) 528{ 529 if (unlikely(comp_status != 0)) 530 ena_trc_err("admin command failed[%u]\n", comp_status); 531 532 if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR)) 533 return ENA_COM_INVAL; 534 535 switch (comp_status) { 536 case ENA_ADMIN_SUCCESS: 537 return 0; 538 case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE: 539 return ENA_COM_NO_MEM; 540 case ENA_ADMIN_UNSUPPORTED_OPCODE: 541 return ENA_COM_UNSUPPORTED; 542 case ENA_ADMIN_BAD_OPCODE: 543 case ENA_ADMIN_MALFORMED_REQUEST: 544 case ENA_ADMIN_ILLEGAL_PARAMETER: 545 case ENA_ADMIN_UNKNOWN_ERROR: 546 return ENA_COM_INVAL; 547 } 548 549 return 0; 550} 551 552static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx, 553 struct ena_com_admin_queue *admin_queue) 554{ 555 unsigned long flags, timeout; 556 int ret; 557 558 timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout); 559 560 while (1) { 561 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 562 ena_com_handle_admin_completion(admin_queue); 563 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 564 565 if (comp_ctx->status != ENA_CMD_SUBMITTED) 566 break; 567 568 if (ENA_TIME_EXPIRE(timeout)) { 569 ena_trc_err("Wait for completion (polling) timeout\n"); 570 /* ENA didn't have any completion */ 571 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 572 admin_queue->stats.no_completion++; 573 admin_queue->running_state = false; 574 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 575 576 ret = ENA_COM_TIMER_EXPIRED; 577 goto err; 578 } 579 580 ENA_MSLEEP(100); 581 } 582 583 if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) { 584 ena_trc_err("Command was aborted\n"); 585 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 586 admin_queue->stats.aborted_cmd++; 587 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 588 ret = ENA_COM_NO_DEVICE; 589 goto err; 590 } 591 592 ENA_WARN(comp_ctx->status != ENA_CMD_COMPLETED, 593 "Invalid comp status %d\n", comp_ctx->status); 594 595 ret = ena_com_comp_status_to_errno(comp_ctx->comp_status); 596err: 597 comp_ctxt_release(admin_queue, comp_ctx); 598 return ret; 599} 600 601static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, 602 struct ena_admin_feature_llq_desc *llq_desc) 603{ 604 struct ena_com_llq_info *llq_info = &ena_dev->llq_info; 605 606 memset(llq_info, 0, sizeof(*llq_info)); 607 608 switch (llq_desc->header_location_ctrl) { 609 case ENA_ADMIN_INLINE_HEADER: 610 llq_info->inline_header = true; 611 break; 612 case ENA_ADMIN_HEADER_RING: 613 llq_info->inline_header = false; 614 break; 615 default: 616 ena_trc_err("Invalid header location control\n"); 617 return -EINVAL; 618 } 619 620 switch (llq_desc->entry_size_ctrl) { 621 case ENA_ADMIN_LIST_ENTRY_SIZE_128B: 622 llq_info->desc_list_entry_size = 128; 623 break; 624 case ENA_ADMIN_LIST_ENTRY_SIZE_192B: 625 llq_info->desc_list_entry_size = 192; 626 break; 627 case ENA_ADMIN_LIST_ENTRY_SIZE_256B: 628 llq_info->desc_list_entry_size = 256; 629 break; 630 default: 631 ena_trc_err("Invalid entry_size_ctrl %d\n", 632 llq_desc->entry_size_ctrl); 633 return -EINVAL; 634 } 635 636 if ((llq_info->desc_list_entry_size & 0x7)) { 637 /* The desc list entry size should be whole multiply of 8 638 * This requirement comes from __iowrite64_copy() 639 */ 640 ena_trc_err("illegal entry size %d\n", 641 llq_info->desc_list_entry_size); 642 return -EINVAL; 643 } 644 645 if (llq_info->inline_header) { 646 llq_info->desc_stride_ctrl = llq_desc->descriptors_stride_ctrl; 647 if ((llq_info->desc_stride_ctrl != ENA_ADMIN_SINGLE_DESC_PER_ENTRY) && 648 (llq_info->desc_stride_ctrl != ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY)) { 649 ena_trc_err("Invalid desc_stride_ctrl %d\n", 650 llq_info->desc_stride_ctrl); 651 return -EINVAL; 652 } 653 } else { 654 llq_info->desc_stride_ctrl = ENA_ADMIN_SINGLE_DESC_PER_ENTRY; 655 } 656 657 if (llq_info->desc_stride_ctrl == ENA_ADMIN_SINGLE_DESC_PER_ENTRY) 658 llq_info->descs_per_entry = llq_info->desc_list_entry_size / 659 sizeof(struct ena_eth_io_tx_desc); 660 else 661 llq_info->descs_per_entry = 1; 662 663 llq_info->descs_num_before_header = llq_desc->desc_num_before_header_ctrl; 664 665 return 0; 666} 667 668 669 670static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx, 671 struct ena_com_admin_queue *admin_queue) 672{ 673 unsigned long flags; 674 int ret; 675 676 ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event, 677 admin_queue->completion_timeout); 678 679 /* In case the command wasn't completed find out the root cause. 680 * There might be 2 kinds of errors 681 * 1) No completion (timeout reached) 682 * 2) There is completion but the device didn't get any msi-x interrupt. 683 */ 684 if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) { 685 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 686 ena_com_handle_admin_completion(admin_queue); 687 admin_queue->stats.no_completion++; 688 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 689 690 if (comp_ctx->status == ENA_CMD_COMPLETED) 691 ena_trc_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n", 692 comp_ctx->cmd_opcode); 693 else 694 ena_trc_err("The ena device doesn't send any completion for the admin cmd %d status %d\n", 695 comp_ctx->cmd_opcode, comp_ctx->status); 696 697 admin_queue->running_state = false; 698 ret = ENA_COM_TIMER_EXPIRED; 699 goto err; 700 } 701 702 ret = ena_com_comp_status_to_errno(comp_ctx->comp_status); 703err: 704 comp_ctxt_release(admin_queue, comp_ctx); 705 return ret; 706} 707 708/* This method read the hardware device register through posting writes 709 * and waiting for response 710 * On timeout the function will return ENA_MMIO_READ_TIMEOUT 711 */ 712static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset) 713{ 714 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 715 volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp = 716 mmio_read->read_resp; 717 u32 mmio_read_reg, ret, i; 718 unsigned long flags; 719 u32 timeout = mmio_read->reg_read_to; 720 721 ENA_MIGHT_SLEEP(); 722 723 if (timeout == 0) 724 timeout = ENA_REG_READ_TIMEOUT; 725 726 /* If readless is disabled, perform regular read */ 727 if (!mmio_read->readless_supported) 728 return ENA_REG_READ32(ena_dev->bus, ena_dev->reg_bar + offset); 729 730 ENA_SPINLOCK_LOCK(mmio_read->lock, flags); 731 mmio_read->seq_num++; 732 733 read_resp->req_id = mmio_read->seq_num + 0xDEAD; 734 mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) & 735 ENA_REGS_MMIO_REG_READ_REG_OFF_MASK; 736 mmio_read_reg |= mmio_read->seq_num & 737 ENA_REGS_MMIO_REG_READ_REQ_ID_MASK; 738 739 /* make sure read_resp->req_id get updated before the hw can write 740 * there 741 */ 742 wmb(); 743 744 ENA_REG_WRITE32(ena_dev->bus, mmio_read_reg, ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF); 745 746 for (i = 0; i < timeout; i++) { 747 if (read_resp->req_id == mmio_read->seq_num) 748 break; 749 750 ENA_UDELAY(1); 751 } 752 753 if (unlikely(i == timeout)) { 754 ena_trc_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n", 755 mmio_read->seq_num, 756 offset, 757 read_resp->req_id, 758 read_resp->reg_off); 759 ret = ENA_MMIO_READ_TIMEOUT; 760 goto err; 761 } 762 763 if (read_resp->reg_off != offset) { 764 ena_trc_err("Read failure: wrong offset provided"); 765 ret = ENA_MMIO_READ_TIMEOUT; 766 } else { 767 ret = read_resp->reg_val; 768 } 769err: 770 ENA_SPINLOCK_UNLOCK(mmio_read->lock, flags); 771 772 return ret; 773} 774 775/* There are two types to wait for completion. 776 * Polling mode - wait until the completion is available. 777 * Async mode - wait on wait queue until the completion is ready 778 * (or the timeout expired). 779 * It is expected that the IRQ called ena_com_handle_admin_completion 780 * to mark the completions. 781 */ 782static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx, 783 struct ena_com_admin_queue *admin_queue) 784{ 785 if (admin_queue->polling) 786 return ena_com_wait_and_process_admin_cq_polling(comp_ctx, 787 admin_queue); 788 789 return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx, 790 admin_queue); 791} 792 793static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev, 794 struct ena_com_io_sq *io_sq) 795{ 796 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 797 struct ena_admin_aq_destroy_sq_cmd destroy_cmd; 798 struct ena_admin_acq_destroy_sq_resp_desc destroy_resp; 799 u8 direction; 800 int ret; 801 802 memset(&destroy_cmd, 0x0, sizeof(destroy_cmd)); 803 804 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) 805 direction = ENA_ADMIN_SQ_DIRECTION_TX; 806 else 807 direction = ENA_ADMIN_SQ_DIRECTION_RX; 808 809 destroy_cmd.sq.sq_identity |= (direction << 810 ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) & 811 ENA_ADMIN_SQ_SQ_DIRECTION_MASK; 812 813 destroy_cmd.sq.sq_idx = io_sq->idx; 814 destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ; 815 816 ret = ena_com_execute_admin_command(admin_queue, 817 (struct ena_admin_aq_entry *)&destroy_cmd, 818 sizeof(destroy_cmd), 819 (struct ena_admin_acq_entry *)&destroy_resp, 820 sizeof(destroy_resp)); 821 822 if (unlikely(ret && (ret != ENA_COM_NO_DEVICE))) 823 ena_trc_err("failed to destroy io sq error: %d\n", ret); 824 825 return ret; 826} 827 828static void ena_com_io_queue_free(struct ena_com_dev *ena_dev, 829 struct ena_com_io_sq *io_sq, 830 struct ena_com_io_cq *io_cq) 831{ 832 size_t size; 833 834 if (io_cq->cdesc_addr.virt_addr) { 835 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; 836 837 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 838 size, 839 io_cq->cdesc_addr.virt_addr, 840 io_cq->cdesc_addr.phys_addr, 841 io_cq->cdesc_addr.mem_handle); 842 843 io_cq->cdesc_addr.virt_addr = NULL; 844 } 845 846 if (io_sq->desc_addr.virt_addr) { 847 size = io_sq->desc_entry_size * io_sq->q_depth; 848 849 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 850 size, 851 io_sq->desc_addr.virt_addr, 852 io_sq->desc_addr.phys_addr, 853 io_sq->desc_addr.mem_handle); 854 855 io_sq->desc_addr.virt_addr = NULL; 856 } 857 858 if (io_sq->bounce_buf_ctrl.base_buffer) { 859 size = io_sq->llq_info.desc_list_entry_size * ENA_COM_BOUNCE_BUFFER_CNTRL_CNT; 860 ENA_MEM_FREE(ena_dev->dmadev, io_sq->bounce_buf_ctrl.base_buffer, size); 861 io_sq->bounce_buf_ctrl.base_buffer = NULL; 862 } 863} 864 865static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout, 866 u16 exp_state) 867{ 868 u32 val, i; 869 870 for (i = 0; i < timeout; i++) { 871 val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); 872 873 if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) { 874 ena_trc_err("Reg read timeout occurred\n"); 875 return ENA_COM_TIMER_EXPIRED; 876 } 877 878 if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) == 879 exp_state) 880 return 0; 881 882 /* The resolution of the timeout is 100ms */ 883 ENA_MSLEEP(100); 884 } 885 886 return ENA_COM_TIMER_EXPIRED; 887} 888 889static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev, 890 enum ena_admin_aq_feature_id feature_id) 891{ 892 u32 feature_mask = 1 << feature_id; 893 894 /* Device attributes is always supported */ 895 if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) && 896 !(ena_dev->supported_features & feature_mask)) 897 return false; 898 899 return true; 900} 901 902static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev, 903 struct ena_admin_get_feat_resp *get_resp, 904 enum ena_admin_aq_feature_id feature_id, 905 dma_addr_t control_buf_dma_addr, 906 u32 control_buff_size) 907{ 908 struct ena_com_admin_queue *admin_queue; 909 struct ena_admin_get_feat_cmd get_cmd; 910 int ret; 911 912 if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) { 913 ena_trc_dbg("Feature %d isn't supported\n", feature_id); 914 return ENA_COM_UNSUPPORTED; 915 } 916 917 memset(&get_cmd, 0x0, sizeof(get_cmd)); 918 admin_queue = &ena_dev->admin_queue; 919 920 get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE; 921 922 if (control_buff_size) 923 get_cmd.aq_common_descriptor.flags = 924 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 925 else 926 get_cmd.aq_common_descriptor.flags = 0; 927 928 ret = ena_com_mem_addr_set(ena_dev, 929 &get_cmd.control_buffer.address, 930 control_buf_dma_addr); 931 if (unlikely(ret)) { 932 ena_trc_err("memory address set failed\n"); 933 return ret; 934 } 935 936 get_cmd.control_buffer.length = control_buff_size; 937 938 get_cmd.feat_common.feature_id = feature_id; 939 940 ret = ena_com_execute_admin_command(admin_queue, 941 (struct ena_admin_aq_entry *) 942 &get_cmd, 943 sizeof(get_cmd), 944 (struct ena_admin_acq_entry *) 945 get_resp, 946 sizeof(*get_resp)); 947 948 if (unlikely(ret)) 949 ena_trc_err("Failed to submit get_feature command %d error: %d\n", 950 feature_id, ret); 951 952 return ret; 953} 954 955static int ena_com_get_feature(struct ena_com_dev *ena_dev, 956 struct ena_admin_get_feat_resp *get_resp, 957 enum ena_admin_aq_feature_id feature_id) 958{ 959 return ena_com_get_feature_ex(ena_dev, 960 get_resp, 961 feature_id, 962 0, 963 0); 964} 965 966static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev) 967{ 968 struct ena_rss *rss = &ena_dev->rss; 969 970 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 971 sizeof(*rss->hash_key), 972 rss->hash_key, 973 rss->hash_key_dma_addr, 974 rss->hash_key_mem_handle); 975 976 if (unlikely(!rss->hash_key)) 977 return ENA_COM_NO_MEM; 978 979 return 0; 980} 981 982static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev) 983{ 984 struct ena_rss *rss = &ena_dev->rss; 985 986 if (rss->hash_key) 987 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 988 sizeof(*rss->hash_key), 989 rss->hash_key, 990 rss->hash_key_dma_addr, 991 rss->hash_key_mem_handle); 992 rss->hash_key = NULL; 993} 994 995static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev) 996{ 997 struct ena_rss *rss = &ena_dev->rss; 998 999 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 1000 sizeof(*rss->hash_ctrl), 1001 rss->hash_ctrl, 1002 rss->hash_ctrl_dma_addr, 1003 rss->hash_ctrl_mem_handle); 1004 1005 if (unlikely(!rss->hash_ctrl)) 1006 return ENA_COM_NO_MEM; 1007 1008 return 0; 1009} 1010 1011static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev) 1012{ 1013 struct ena_rss *rss = &ena_dev->rss; 1014 1015 if (rss->hash_ctrl) 1016 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1017 sizeof(*rss->hash_ctrl), 1018 rss->hash_ctrl, 1019 rss->hash_ctrl_dma_addr, 1020 rss->hash_ctrl_mem_handle); 1021 rss->hash_ctrl = NULL; 1022} 1023 1024static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev, 1025 u16 log_size) 1026{ 1027 struct ena_rss *rss = &ena_dev->rss; 1028 struct ena_admin_get_feat_resp get_resp; 1029 size_t tbl_size; 1030 int ret; 1031 1032 ret = ena_com_get_feature(ena_dev, &get_resp, 1033 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG); 1034 if (unlikely(ret)) 1035 return ret; 1036 1037 if ((get_resp.u.ind_table.min_size > log_size) || 1038 (get_resp.u.ind_table.max_size < log_size)) { 1039 ena_trc_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n", 1040 1 << log_size, 1041 1 << get_resp.u.ind_table.min_size, 1042 1 << get_resp.u.ind_table.max_size); 1043 return ENA_COM_INVAL; 1044 } 1045 1046 tbl_size = (1ULL << log_size) * 1047 sizeof(struct ena_admin_rss_ind_table_entry); 1048 1049 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 1050 tbl_size, 1051 rss->rss_ind_tbl, 1052 rss->rss_ind_tbl_dma_addr, 1053 rss->rss_ind_tbl_mem_handle); 1054 if (unlikely(!rss->rss_ind_tbl)) 1055 goto mem_err1; 1056 1057 rss->host_rss_ind_tbl_size = (1ULL << log_size) * sizeof(u16); 1058 rss->host_rss_ind_tbl = 1059 ENA_MEM_ALLOC(ena_dev->dmadev, rss->host_rss_ind_tbl_size); 1060 if (unlikely(!rss->host_rss_ind_tbl)) 1061 goto mem_err2; 1062 1063 rss->tbl_log_size = log_size; 1064 1065 return 0; 1066 1067mem_err2: 1068 tbl_size = (1ULL << log_size) * 1069 sizeof(struct ena_admin_rss_ind_table_entry); 1070 1071 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1072 tbl_size, 1073 rss->rss_ind_tbl, 1074 rss->rss_ind_tbl_dma_addr, 1075 rss->rss_ind_tbl_mem_handle); 1076 rss->rss_ind_tbl = NULL; 1077mem_err1: 1078 rss->tbl_log_size = 0; 1079 return ENA_COM_NO_MEM; 1080} 1081 1082static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev) 1083{ 1084 struct ena_rss *rss = &ena_dev->rss; 1085 size_t tbl_size = (1ULL << rss->tbl_log_size) * 1086 sizeof(struct ena_admin_rss_ind_table_entry); 1087 1088 if (rss->rss_ind_tbl) 1089 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1090 tbl_size, 1091 rss->rss_ind_tbl, 1092 rss->rss_ind_tbl_dma_addr, 1093 rss->rss_ind_tbl_mem_handle); 1094 rss->rss_ind_tbl = NULL; 1095 1096 if (rss->host_rss_ind_tbl) 1097 ENA_MEM_FREE(ena_dev->dmadev, rss->host_rss_ind_tbl, 1098 rss->host_rss_ind_tbl_size); 1099 rss->host_rss_ind_tbl = NULL; 1100} 1101 1102static int ena_com_create_io_sq(struct ena_com_dev *ena_dev, 1103 struct ena_com_io_sq *io_sq, u16 cq_idx) 1104{ 1105 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1106 struct ena_admin_aq_create_sq_cmd create_cmd; 1107 struct ena_admin_acq_create_sq_resp_desc cmd_completion; 1108 u8 direction; 1109 int ret; 1110 1111 memset(&create_cmd, 0x0, sizeof(create_cmd)); 1112 1113 create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ; 1114 1115 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) 1116 direction = ENA_ADMIN_SQ_DIRECTION_TX; 1117 else 1118 direction = ENA_ADMIN_SQ_DIRECTION_RX; 1119 1120 create_cmd.sq_identity |= (direction << 1121 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) & 1122 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK; 1123 1124 create_cmd.sq_caps_2 |= io_sq->mem_queue_type & 1125 ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK; 1126 1127 create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC << 1128 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) & 1129 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK; 1130 1131 create_cmd.sq_caps_3 |= 1132 ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; 1133 1134 create_cmd.cq_idx = cq_idx; 1135 create_cmd.sq_depth = io_sq->q_depth; 1136 1137 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { 1138 ret = ena_com_mem_addr_set(ena_dev, 1139 &create_cmd.sq_ba, 1140 io_sq->desc_addr.phys_addr); 1141 if (unlikely(ret)) { 1142 ena_trc_err("memory address set failed\n"); 1143 return ret; 1144 } 1145 } 1146 1147 ret = ena_com_execute_admin_command(admin_queue, 1148 (struct ena_admin_aq_entry *)&create_cmd, 1149 sizeof(create_cmd), 1150 (struct ena_admin_acq_entry *)&cmd_completion, 1151 sizeof(cmd_completion)); 1152 if (unlikely(ret)) { 1153 ena_trc_err("Failed to create IO SQ. error: %d\n", ret); 1154 return ret; 1155 } 1156 1157 io_sq->idx = cmd_completion.sq_idx; 1158 1159 io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1160 (uintptr_t)cmd_completion.sq_doorbell_offset); 1161 1162 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1163 io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar 1164 + cmd_completion.llq_headers_offset); 1165 1166 io_sq->desc_addr.pbuf_dev_addr = 1167 (u8 __iomem *)((uintptr_t)ena_dev->mem_bar + 1168 cmd_completion.llq_descriptors_offset); 1169 } 1170 1171 ena_trc_dbg("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth); 1172 1173 return ret; 1174} 1175 1176static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev) 1177{ 1178 struct ena_rss *rss = &ena_dev->rss; 1179 struct ena_com_io_sq *io_sq; 1180 u16 qid; 1181 int i; 1182 1183 for (i = 0; i < 1 << rss->tbl_log_size; i++) { 1184 qid = rss->host_rss_ind_tbl[i]; 1185 if (qid >= ENA_TOTAL_NUM_QUEUES) 1186 return ENA_COM_INVAL; 1187 1188 io_sq = &ena_dev->io_sq_queues[qid]; 1189 1190 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX) 1191 return ENA_COM_INVAL; 1192 1193 rss->rss_ind_tbl[i].cq_idx = io_sq->idx; 1194 } 1195 1196 return 0; 1197} 1198 1199static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev) 1200{ 1201 u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { (u16)-1 }; 1202 struct ena_rss *rss = &ena_dev->rss; 1203 u8 idx; 1204 u16 i; 1205 1206 for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++) 1207 dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i; 1208 1209 for (i = 0; i < 1 << rss->tbl_log_size; i++) { 1210 if (rss->rss_ind_tbl[i].cq_idx > ENA_TOTAL_NUM_QUEUES) 1211 return ENA_COM_INVAL; 1212 idx = (u8)rss->rss_ind_tbl[i].cq_idx; 1213 1214 if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES) 1215 return ENA_COM_INVAL; 1216 1217 rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx]; 1218 } 1219 1220 return 0; 1221} 1222 1223static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev) 1224{ 1225 size_t size; 1226 1227 size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS; 1228 1229 ena_dev->intr_moder_tbl = ENA_MEM_ALLOC(ena_dev->dmadev, size); 1230 if (!ena_dev->intr_moder_tbl) 1231 return ENA_COM_NO_MEM; 1232 1233 ena_com_config_default_interrupt_moderation_table(ena_dev); 1234 1235 return 0; 1236} 1237 1238static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev, 1239 u16 intr_delay_resolution) 1240{ 1241 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; 1242 unsigned int i; 1243 1244 if (!intr_delay_resolution) { 1245 ena_trc_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n"); 1246 intr_delay_resolution = 1; 1247 } 1248 ena_dev->intr_delay_resolution = intr_delay_resolution; 1249 1250 /* update Rx */ 1251 for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++) 1252 intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution; 1253 1254 /* update Tx */ 1255 ena_dev->intr_moder_tx_interval /= intr_delay_resolution; 1256} 1257 1258/*****************************************************************************/ 1259/******************************* API ******************************/ 1260/*****************************************************************************/ 1261 1262int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue, 1263 struct ena_admin_aq_entry *cmd, 1264 size_t cmd_size, 1265 struct ena_admin_acq_entry *comp, 1266 size_t comp_size) 1267{ 1268 struct ena_comp_ctx *comp_ctx; 1269 int ret; 1270 1271 comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size, 1272 comp, comp_size); 1273 if (unlikely(IS_ERR(comp_ctx))) { 1274 if (comp_ctx == ERR_PTR(ENA_COM_NO_DEVICE)) 1275 ena_trc_dbg("Failed to submit command [%ld]\n", 1276 PTR_ERR(comp_ctx)); 1277 else 1278 ena_trc_err("Failed to submit command [%ld]\n", 1279 PTR_ERR(comp_ctx)); 1280 1281 return PTR_ERR(comp_ctx); 1282 } 1283 1284 ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue); 1285 if (unlikely(ret)) { 1286 if (admin_queue->running_state) 1287 ena_trc_err("Failed to process command. ret = %d\n", 1288 ret); 1289 else 1290 ena_trc_dbg("Failed to process command. ret = %d\n", 1291 ret); 1292 } 1293 return ret; 1294} 1295 1296int ena_com_create_io_cq(struct ena_com_dev *ena_dev, 1297 struct ena_com_io_cq *io_cq) 1298{ 1299 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1300 struct ena_admin_aq_create_cq_cmd create_cmd; 1301 struct ena_admin_acq_create_cq_resp_desc cmd_completion; 1302 int ret; 1303 1304 memset(&create_cmd, 0x0, sizeof(create_cmd)); 1305 1306 create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ; 1307 1308 create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) & 1309 ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; 1310 create_cmd.cq_caps_1 |= 1311 ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK; 1312 1313 create_cmd.msix_vector = io_cq->msix_vector; 1314 create_cmd.cq_depth = io_cq->q_depth; 1315 1316 ret = ena_com_mem_addr_set(ena_dev, 1317 &create_cmd.cq_ba, 1318 io_cq->cdesc_addr.phys_addr); 1319 if (unlikely(ret)) { 1320 ena_trc_err("memory address set failed\n"); 1321 return ret; 1322 } 1323 1324 ret = ena_com_execute_admin_command(admin_queue, 1325 (struct ena_admin_aq_entry *)&create_cmd, 1326 sizeof(create_cmd), 1327 (struct ena_admin_acq_entry *)&cmd_completion, 1328 sizeof(cmd_completion)); 1329 if (unlikely(ret)) { 1330 ena_trc_err("Failed to create IO CQ. error: %d\n", ret); 1331 return ret; 1332 } 1333 1334 io_cq->idx = cmd_completion.cq_idx; 1335 1336 io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1337 cmd_completion.cq_interrupt_unmask_register_offset); 1338 1339 if (cmd_completion.cq_head_db_register_offset) 1340 io_cq->cq_head_db_reg = 1341 (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1342 cmd_completion.cq_head_db_register_offset); 1343 1344 if (cmd_completion.numa_node_register_offset) 1345 io_cq->numa_node_cfg_reg = 1346 (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1347 cmd_completion.numa_node_register_offset); 1348 1349 ena_trc_dbg("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth); 1350 1351 return ret; 1352} 1353 1354int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid, 1355 struct ena_com_io_sq **io_sq, 1356 struct ena_com_io_cq **io_cq) 1357{ 1358 if (qid >= ENA_TOTAL_NUM_QUEUES) { 1359 ena_trc_err("Invalid queue number %d but the max is %d\n", 1360 qid, ENA_TOTAL_NUM_QUEUES); 1361 return ENA_COM_INVAL; 1362 } 1363 1364 *io_sq = &ena_dev->io_sq_queues[qid]; 1365 *io_cq = &ena_dev->io_cq_queues[qid]; 1366 1367 return 0; 1368} 1369 1370void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev) 1371{ 1372 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1373 struct ena_comp_ctx *comp_ctx; 1374 u16 i; 1375 1376 if (!admin_queue->comp_ctx) 1377 return; 1378 1379 for (i = 0; i < admin_queue->q_depth; i++) { 1380 comp_ctx = get_comp_ctxt(admin_queue, i, false); 1381 if (unlikely(!comp_ctx)) 1382 break; 1383 1384 comp_ctx->status = ENA_CMD_ABORTED; 1385 1386 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event); 1387 } 1388} 1389 1390void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev) 1391{ 1392 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1393 unsigned long flags; 1394 1395 /* 1396 * XXX: workaround for missing synchronization mechanism of AENQ handler 1397 * Wait 20ms for safety though it have not panicked actually. 1398 */ 1399 ENA_MSLEEP(20); 1400 1401 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 1402 while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) { 1403 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 1404 ENA_MSLEEP(20); 1405 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 1406 } 1407 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 1408} 1409 1410int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev, 1411 struct ena_com_io_cq *io_cq) 1412{ 1413 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1414 struct ena_admin_aq_destroy_cq_cmd destroy_cmd; 1415 struct ena_admin_acq_destroy_cq_resp_desc destroy_resp; 1416 int ret; 1417 1418 memset(&destroy_cmd, 0x0, sizeof(destroy_cmd)); 1419 1420 destroy_cmd.cq_idx = io_cq->idx; 1421 destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ; 1422 1423 ret = ena_com_execute_admin_command(admin_queue, 1424 (struct ena_admin_aq_entry *)&destroy_cmd, 1425 sizeof(destroy_cmd), 1426 (struct ena_admin_acq_entry *)&destroy_resp, 1427 sizeof(destroy_resp)); 1428 1429 if (unlikely(ret && (ret != ENA_COM_NO_DEVICE))) 1430 ena_trc_err("Failed to destroy IO CQ. error: %d\n", ret); 1431 1432 return ret; 1433} 1434 1435bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev) 1436{ 1437 return ena_dev->admin_queue.running_state; 1438} 1439 1440void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state) 1441{ 1442 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1443 unsigned long flags; 1444 1445 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 1446 ena_dev->admin_queue.running_state = state; 1447 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 1448} 1449 1450void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev) 1451{ 1452 u16 depth = ena_dev->aenq.q_depth; 1453 1454 ENA_WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n"); 1455 1456 /* Init head_db to mark that all entries in the queue 1457 * are initially available 1458 */ 1459 ENA_REG_WRITE32(ena_dev->bus, depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); 1460} 1461 1462int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag) 1463{ 1464 struct ena_com_admin_queue *admin_queue; 1465 struct ena_admin_set_feat_cmd cmd; 1466 struct ena_admin_set_feat_resp resp; 1467 struct ena_admin_get_feat_resp get_resp; 1468 int ret; 1469 1470 ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG); 1471 if (ret) { 1472 ena_trc_info("Can't get aenq configuration\n"); 1473 return ret; 1474 } 1475 1476 if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) { 1477 ena_trc_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n", 1478 get_resp.u.aenq.supported_groups, 1479 groups_flag); 1480 return ENA_COM_UNSUPPORTED; 1481 } 1482 1483 memset(&cmd, 0x0, sizeof(cmd)); 1484 admin_queue = &ena_dev->admin_queue; 1485 1486 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 1487 cmd.aq_common_descriptor.flags = 0; 1488 cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG; 1489 cmd.u.aenq.enabled_groups = groups_flag; 1490 1491 ret = ena_com_execute_admin_command(admin_queue, 1492 (struct ena_admin_aq_entry *)&cmd, 1493 sizeof(cmd), 1494 (struct ena_admin_acq_entry *)&resp, 1495 sizeof(resp)); 1496 1497 if (unlikely(ret)) 1498 ena_trc_err("Failed to config AENQ ret: %d\n", ret); 1499 1500 return ret; 1501} 1502 1503int ena_com_get_dma_width(struct ena_com_dev *ena_dev) 1504{ 1505 u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF); 1506 int width; 1507 1508 if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) { 1509 ena_trc_err("Reg read timeout occurred\n"); 1510 return ENA_COM_TIMER_EXPIRED; 1511 } 1512 1513 width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >> 1514 ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT; 1515 1516 ena_trc_dbg("ENA dma width: %d\n", width); 1517 1518 if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) { 1519 ena_trc_err("DMA width illegal value: %d\n", width); 1520 return ENA_COM_INVAL; 1521 } 1522 1523 ena_dev->dma_addr_bits = width; 1524 1525 return width; 1526} 1527 1528int ena_com_validate_version(struct ena_com_dev *ena_dev) 1529{ 1530 u32 ver; 1531 u32 ctrl_ver; 1532 u32 ctrl_ver_masked; 1533 1534 /* Make sure the ENA version and the controller version are at least 1535 * as the driver expects 1536 */ 1537 ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF); 1538 ctrl_ver = ena_com_reg_bar_read32(ena_dev, 1539 ENA_REGS_CONTROLLER_VERSION_OFF); 1540 1541 if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) || 1542 (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) { 1543 ena_trc_err("Reg read timeout occurred\n"); 1544 return ENA_COM_TIMER_EXPIRED; 1545 } 1546 1547 ena_trc_info("ena device version: %d.%d\n", 1548 (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >> 1549 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT, 1550 ver & ENA_REGS_VERSION_MINOR_VERSION_MASK); 1551 1552 if (ver < MIN_ENA_VER) { 1553 ena_trc_err("ENA version is lower than the minimal version the driver supports\n"); 1554 return -1; 1555 } 1556 1557 ena_trc_info("ena controller version: %d.%d.%d implementation version %d\n", 1558 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) 1559 >> ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT, 1560 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) 1561 >> ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT, 1562 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK), 1563 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >> 1564 ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT); 1565 1566 ctrl_ver_masked = 1567 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) | 1568 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) | 1569 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK); 1570 1571 /* Validate the ctrl version without the implementation ID */ 1572 if (ctrl_ver_masked < MIN_ENA_CTRL_VER) { 1573 ena_trc_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n"); 1574 return -1; 1575 } 1576 1577 return 0; 1578} 1579 1580void ena_com_admin_destroy(struct ena_com_dev *ena_dev) 1581{ 1582 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1583 struct ena_com_admin_cq *cq = &admin_queue->cq; 1584 struct ena_com_admin_sq *sq = &admin_queue->sq; 1585 struct ena_com_aenq *aenq = &ena_dev->aenq; 1586 u16 size; 1587 int i; 1588 1589 ENA_SPINLOCK_DESTROY(admin_queue->q_lock); 1590 1591 if (admin_queue->comp_ctx) { 1592 size_t s; 1593 1594 for (i = 0; i < admin_queue->q_depth; i++) { 1595 struct ena_comp_ctx *comp_ctx = get_comp_ctxt(admin_queue, i, false); 1596 if (comp_ctx != NULL) 1597 ENA_WAIT_EVENT_DESTROY(comp_ctx->wait_event); 1598 } 1599 1600 s = admin_queue->q_depth * sizeof(struct ena_comp_ctx); 1601 ENA_MEM_FREE(ena_dev->dmadev, admin_queue->comp_ctx, s); 1602 } 1603 admin_queue->comp_ctx = NULL; 1604 size = ADMIN_SQ_SIZE(admin_queue->q_depth); 1605 if (sq->entries) 1606 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, sq->entries, 1607 sq->dma_addr, sq->mem_handle); 1608 sq->entries = NULL; 1609 1610 size = ADMIN_CQ_SIZE(admin_queue->q_depth); 1611 if (cq->entries) 1612 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, cq->entries, 1613 cq->dma_addr, cq->mem_handle); 1614 cq->entries = NULL; 1615 1616 size = ADMIN_AENQ_SIZE(aenq->q_depth); 1617 if (ena_dev->aenq.entries) 1618 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, aenq->entries, 1619 aenq->dma_addr, aenq->mem_handle); 1620 aenq->entries = NULL; 1621} 1622 1623void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling) 1624{ 1625 u32 mask_value = 0; 1626 1627 if (polling) 1628 mask_value = ENA_REGS_ADMIN_INTR_MASK; 1629 1630 ENA_REG_WRITE32(ena_dev->bus, mask_value, ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF); 1631 ena_dev->admin_queue.polling = polling; 1632} 1633 1634int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev) 1635{ 1636 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1637 1638 ENA_SPINLOCK_INIT(mmio_read->lock); 1639 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 1640 sizeof(*mmio_read->read_resp), 1641 mmio_read->read_resp, 1642 mmio_read->read_resp_dma_addr, 1643 mmio_read->read_resp_mem_handle); 1644 if (unlikely(!mmio_read->read_resp)) 1645 return ENA_COM_NO_MEM; 1646 1647 ena_com_mmio_reg_read_request_write_dev_addr(ena_dev); 1648 1649 mmio_read->read_resp->req_id = 0x0; 1650 mmio_read->seq_num = 0x0; 1651 mmio_read->readless_supported = true; 1652 1653 return 0; 1654} 1655 1656void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported) 1657{ 1658 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1659 1660 mmio_read->readless_supported = readless_supported; 1661} 1662 1663void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev) 1664{ 1665 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1666 1667 ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); 1668 ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); 1669 1670 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1671 sizeof(*mmio_read->read_resp), 1672 mmio_read->read_resp, 1673 mmio_read->read_resp_dma_addr, 1674 mmio_read->read_resp_mem_handle); 1675 1676 mmio_read->read_resp = NULL; 1677 1678 ENA_SPINLOCK_DESTROY(mmio_read->lock); 1679} 1680 1681void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev) 1682{ 1683 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1684 u32 addr_low, addr_high; 1685 1686 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr); 1687 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr); 1688 1689 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); 1690 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); 1691} 1692 1693int ena_com_admin_init(struct ena_com_dev *ena_dev, 1694 struct ena_aenq_handlers *aenq_handlers, 1695 bool init_spinlock) 1696{ 1697 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1698 u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high; 1699 int ret; 1700 1701#ifdef ENA_INTERNAL 1702 ena_trc_info("ena_defs : Version:[%s] Build date [%s]", 1703 ENA_GEN_COMMIT, ENA_GEN_DATE); 1704#endif 1705 dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); 1706 1707 if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) { 1708 ena_trc_err("Reg read timeout occurred\n"); 1709 return ENA_COM_TIMER_EXPIRED; 1710 } 1711 1712 if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) { 1713 ena_trc_err("Device isn't ready, abort com init\n"); 1714 return ENA_COM_NO_DEVICE; 1715 } 1716 1717 admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH; 1718 1719 admin_queue->bus = ena_dev->bus; 1720 admin_queue->q_dmadev = ena_dev->dmadev; 1721 admin_queue->polling = false; 1722 admin_queue->curr_cmd_id = 0; 1723 1724 ATOMIC32_SET(&admin_queue->outstanding_cmds, 0); 1725 1726 if (init_spinlock) 1727 ENA_SPINLOCK_INIT(admin_queue->q_lock); 1728 1729 ret = ena_com_init_comp_ctxt(admin_queue); 1730 if (ret) 1731 goto error; 1732 1733 ret = ena_com_admin_init_sq(admin_queue); 1734 if (ret) 1735 goto error; 1736 1737 ret = ena_com_admin_init_cq(admin_queue); 1738 if (ret) 1739 goto error; 1740 1741 admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1742 ENA_REGS_AQ_DB_OFF); 1743 1744 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr); 1745 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr); 1746 1747 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF); 1748 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF); 1749 1750 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr); 1751 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr); 1752 1753 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF); 1754 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF); 1755 1756 aq_caps = 0; 1757 aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK; 1758 aq_caps |= (sizeof(struct ena_admin_aq_entry) << 1759 ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) & 1760 ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK; 1761 1762 acq_caps = 0; 1763 acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK; 1764 acq_caps |= (sizeof(struct ena_admin_acq_entry) << 1765 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) & 1766 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK; 1767 1768 ENA_REG_WRITE32(ena_dev->bus, aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF); 1769 ENA_REG_WRITE32(ena_dev->bus, acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF); 1770 ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers); 1771 if (ret) 1772 goto error; 1773 1774 admin_queue->running_state = true; 1775 1776 return 0; 1777error: 1778 ena_com_admin_destroy(ena_dev); 1779 1780 return ret; 1781} 1782 1783int ena_com_create_io_queue(struct ena_com_dev *ena_dev, 1784 struct ena_com_create_io_ctx *ctx) 1785{ 1786 struct ena_com_io_sq *io_sq; 1787 struct ena_com_io_cq *io_cq; 1788 int ret; 1789 1790 if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) { 1791 ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n", 1792 ctx->qid, ENA_TOTAL_NUM_QUEUES); 1793 return ENA_COM_INVAL; 1794 } 1795 1796 io_sq = &ena_dev->io_sq_queues[ctx->qid]; 1797 io_cq = &ena_dev->io_cq_queues[ctx->qid]; 1798 1799 memset(io_sq, 0x0, sizeof(*io_sq)); 1800 memset(io_cq, 0x0, sizeof(*io_cq)); 1801 1802 /* Init CQ */ 1803 io_cq->q_depth = ctx->queue_size; 1804 io_cq->direction = ctx->direction; 1805 io_cq->qid = ctx->qid; 1806 1807 io_cq->msix_vector = ctx->msix_vector; 1808 1809 io_sq->q_depth = ctx->queue_size; 1810 io_sq->direction = ctx->direction; 1811 io_sq->qid = ctx->qid; 1812 1813 io_sq->mem_queue_type = ctx->mem_queue_type; 1814 1815 if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) 1816 /* header length is limited to 8 bits */ 1817 io_sq->tx_max_header_size = 1818 ENA_MIN32(ena_dev->tx_max_header_size, SZ_256); 1819 1820 ret = ena_com_init_io_sq(ena_dev, ctx, io_sq); 1821 if (ret) 1822 goto error; 1823 ret = ena_com_init_io_cq(ena_dev, ctx, io_cq); 1824 if (ret) 1825 goto error; 1826 1827 ret = ena_com_create_io_cq(ena_dev, io_cq); 1828 if (ret) 1829 goto error; 1830 1831 ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx); 1832 if (ret) 1833 goto destroy_io_cq; 1834 1835 return 0; 1836 1837destroy_io_cq: 1838 ena_com_destroy_io_cq(ena_dev, io_cq); 1839error: 1840 ena_com_io_queue_free(ena_dev, io_sq, io_cq); 1841 return ret; 1842} 1843 1844void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid) 1845{ 1846 struct ena_com_io_sq *io_sq; 1847 struct ena_com_io_cq *io_cq; 1848 1849 if (qid >= ENA_TOTAL_NUM_QUEUES) { 1850 ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n", 1851 qid, ENA_TOTAL_NUM_QUEUES); 1852 return; 1853 } 1854 1855 io_sq = &ena_dev->io_sq_queues[qid]; 1856 io_cq = &ena_dev->io_cq_queues[qid]; 1857 1858 ena_com_destroy_io_sq(ena_dev, io_sq); 1859 ena_com_destroy_io_cq(ena_dev, io_cq); 1860 1861 ena_com_io_queue_free(ena_dev, io_sq, io_cq); 1862} 1863 1864int ena_com_get_link_params(struct ena_com_dev *ena_dev, 1865 struct ena_admin_get_feat_resp *resp) 1866{ 1867 return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG); 1868} 1869 1870int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, 1871 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1872{ 1873 struct ena_admin_get_feat_resp get_resp; 1874 int rc; 1875 1876 rc = ena_com_get_feature(ena_dev, &get_resp, 1877 ENA_ADMIN_DEVICE_ATTRIBUTES); 1878 if (rc) 1879 return rc; 1880 1881 memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr, 1882 sizeof(get_resp.u.dev_attr)); 1883 ena_dev->supported_features = get_resp.u.dev_attr.supported_features; 1884 1885 rc = ena_com_get_feature(ena_dev, &get_resp, 1886 ENA_ADMIN_MAX_QUEUES_NUM); 1887 if (rc) 1888 return rc; 1889 1890 memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue, 1891 sizeof(get_resp.u.max_queue)); 1892 ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size; 1893 1894 rc = ena_com_get_feature(ena_dev, &get_resp, 1895 ENA_ADMIN_AENQ_CONFIG); 1896 if (rc) 1897 return rc; 1898 1899 memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq, 1900 sizeof(get_resp.u.aenq)); 1901 1902 rc = ena_com_get_feature(ena_dev, &get_resp, 1903 ENA_ADMIN_STATELESS_OFFLOAD_CONFIG); 1904 if (rc) 1905 return rc; 1906 1907 memcpy(&get_feat_ctx->offload, &get_resp.u.offload, 1908 sizeof(get_resp.u.offload)); 1909 1910 /* Driver hints isn't mandatory admin command. So in case the 1911 * command isn't supported set driver hints to 0 1912 */ 1913 rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS); 1914 1915 if (!rc) 1916 memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints, 1917 sizeof(get_resp.u.hw_hints)); 1918 else if (rc == ENA_COM_UNSUPPORTED) 1919 memset(&get_feat_ctx->hw_hints, 0x0, sizeof(get_feat_ctx->hw_hints)); 1920 else 1921 return rc; 1922 1923 rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_LLQ); 1924 if (!rc) 1925 memcpy(&get_feat_ctx->llq, &get_resp.u.llq, 1926 sizeof(get_resp.u.llq)); 1927 else if (rc == ENA_COM_UNSUPPORTED) 1928 memset(&get_feat_ctx->llq, 0x0, sizeof(get_feat_ctx->llq)); 1929 else 1930 return rc; 1931 1932 return 0; 1933} 1934 1935void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev) 1936{ 1937 ena_com_handle_admin_completion(&ena_dev->admin_queue); 1938} 1939 1940/* ena_handle_specific_aenq_event: 1941 * return the handler that is relevant to the specific event group 1942 */ 1943static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev, 1944 u16 group) 1945{ 1946 struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers; 1947 1948 if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group]) 1949 return aenq_handlers->handlers[group]; 1950 1951 return aenq_handlers->unimplemented_handler; 1952} 1953 1954/* ena_aenq_intr_handler: 1955 * handles the aenq incoming events. 1956 * pop events from the queue and apply the specific handler 1957 */ 1958void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data) 1959{ 1960 struct ena_admin_aenq_entry *aenq_e; 1961 struct ena_admin_aenq_common_desc *aenq_common; 1962 struct ena_com_aenq *aenq = &dev->aenq; 1963 ena_aenq_handler handler_cb; 1964 unsigned long long timestamp; 1965 u16 masked_head, processed = 0; 1966 u8 phase; 1967 1968 masked_head = aenq->head & (aenq->q_depth - 1); 1969 phase = aenq->phase; 1970 aenq_e = &aenq->entries[masked_head]; /* Get first entry */ 1971 aenq_common = &aenq_e->aenq_common_desc; 1972 1973 /* Go over all the events */ 1974 while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == 1975 phase) { 1976 timestamp = (unsigned long long)aenq_common->timestamp_low | 1977 ((unsigned long long)aenq_common->timestamp_high << 32); 1978 ena_trc_dbg("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n", 1979 aenq_common->group, 1980 aenq_common->syndrom, 1981 timestamp); 1982 1983 /* Handle specific event*/ 1984 handler_cb = ena_com_get_specific_aenq_cb(dev, 1985 aenq_common->group); 1986 handler_cb(data, aenq_e); /* call the actual event handler*/ 1987 1988 /* Get next event entry */ 1989 masked_head++; 1990 processed++; 1991 1992 if (unlikely(masked_head == aenq->q_depth)) { 1993 masked_head = 0; 1994 phase = !phase; 1995 } 1996 aenq_e = &aenq->entries[masked_head]; 1997 aenq_common = &aenq_e->aenq_common_desc; 1998 } 1999 2000 aenq->head += processed; 2001 aenq->phase = phase; 2002 2003 /* Don't update aenq doorbell if there weren't any processed events */ 2004 if (!processed) 2005 return; 2006 2007 /* write the aenq doorbell after all AENQ descriptors were read */ 2008 mb(); 2009 ENA_REG_WRITE32(dev->bus, (u32)aenq->head, dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); 2010} 2011#ifdef ENA_EXTENDED_STATS 2012/* 2013 * Sets the function Idx and Queue Idx to be used for 2014 * get full statistics feature 2015 * 2016 */ 2017int ena_com_extended_stats_set_func_queue(struct ena_com_dev *ena_dev, 2018 u32 func_queue) 2019{ 2020 2021 /* Function & Queue is acquired from user in the following format : 2022 * Bottom Half word: funct 2023 * Top Half Word: queue 2024 */ 2025 ena_dev->stats_func = ENA_EXTENDED_STAT_GET_FUNCT(func_queue); 2026 ena_dev->stats_queue = ENA_EXTENDED_STAT_GET_QUEUE(func_queue); 2027 2028 return 0; 2029} 2030 2031#endif /* ENA_EXTENDED_STATS */ 2032 2033int ena_com_dev_reset(struct ena_com_dev *ena_dev, 2034 enum ena_regs_reset_reason_types reset_reason) 2035{ 2036 u32 stat, timeout, cap, reset_val; 2037 int rc; 2038 2039 stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); 2040 cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF); 2041 2042 if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) || 2043 (cap == ENA_MMIO_READ_TIMEOUT))) { 2044 ena_trc_err("Reg read32 timeout occurred\n"); 2045 return ENA_COM_TIMER_EXPIRED; 2046 } 2047 2048 if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) { 2049 ena_trc_err("Device isn't ready, can't reset device\n"); 2050 return ENA_COM_INVAL; 2051 } 2052 2053 timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >> 2054 ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT; 2055 if (timeout == 0) { 2056 ena_trc_err("Invalid timeout value\n"); 2057 return ENA_COM_INVAL; 2058 } 2059 2060 /* start reset */ 2061 reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK; 2062 reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) & 2063 ENA_REGS_DEV_CTL_RESET_REASON_MASK; 2064 ENA_REG_WRITE32(ena_dev->bus, reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); 2065 2066 /* Write again the MMIO read request address */ 2067 ena_com_mmio_reg_read_request_write_dev_addr(ena_dev); 2068 2069 rc = wait_for_reset_state(ena_dev, timeout, 2070 ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK); 2071 if (rc != 0) { 2072 ena_trc_err("Reset indication didn't turn on\n"); 2073 return rc; 2074 } 2075 2076 /* reset done */ 2077 ENA_REG_WRITE32(ena_dev->bus, 0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); 2078 rc = wait_for_reset_state(ena_dev, timeout, 0); 2079 if (rc != 0) { 2080 ena_trc_err("Reset indication didn't turn off\n"); 2081 return rc; 2082 } 2083 2084 timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >> 2085 ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT; 2086 if (timeout) 2087 /* the resolution of timeout reg is 100ms */ 2088 ena_dev->admin_queue.completion_timeout = timeout * 100000; 2089 else 2090 ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US; 2091 2092 return 0; 2093} 2094 2095static int ena_get_dev_stats(struct ena_com_dev *ena_dev, 2096 struct ena_com_stats_ctx *ctx, 2097 enum ena_admin_get_stats_type type) 2098{ 2099 struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; 2100 struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; 2101 struct ena_com_admin_queue *admin_queue; 2102 int ret; 2103 2104 admin_queue = &ena_dev->admin_queue; 2105 2106 get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS; 2107 get_cmd->aq_common_descriptor.flags = 0; 2108 get_cmd->type = type; 2109 2110 ret = ena_com_execute_admin_command(admin_queue, 2111 (struct ena_admin_aq_entry *)get_cmd, 2112 sizeof(*get_cmd), 2113 (struct ena_admin_acq_entry *)get_resp, 2114 sizeof(*get_resp)); 2115 2116 if (unlikely(ret)) 2117 ena_trc_err("Failed to get stats. error: %d\n", ret); 2118 2119 return ret; 2120} 2121 2122int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, 2123 struct ena_admin_basic_stats *stats) 2124{ 2125 struct ena_com_stats_ctx ctx; 2126 int ret; 2127 2128 memset(&ctx, 0x0, sizeof(ctx)); 2129 ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC); 2130 if (likely(ret == 0)) 2131 memcpy(stats, &ctx.get_resp.basic_stats, 2132 sizeof(ctx.get_resp.basic_stats)); 2133 2134 return ret; 2135} 2136#ifdef ENA_EXTENDED_STATS 2137 2138int ena_com_get_dev_extended_stats(struct ena_com_dev *ena_dev, char *buff, 2139 u32 len) 2140{ 2141 struct ena_com_stats_ctx ctx; 2142 struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx.get_cmd; 2143 ena_mem_handle_t mem_handle; 2144 void *virt_addr; 2145 dma_addr_t phys_addr; 2146 int ret; 2147 2148 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, len, 2149 virt_addr, phys_addr, mem_handle); 2150 if (!virt_addr) { 2151 ret = ENA_COM_NO_MEM; 2152 goto done; 2153 } 2154 memset(&ctx, 0x0, sizeof(ctx)); 2155 ret = ena_com_mem_addr_set(ena_dev, 2156 &get_cmd->u.control_buffer.address, 2157 phys_addr); 2158 if (unlikely(ret)) { 2159 ena_trc_err("memory address set failed\n"); 2160 return ret; 2161 } 2162 get_cmd->u.control_buffer.length = len; 2163 2164 get_cmd->device_id = ena_dev->stats_func; 2165 get_cmd->queue_idx = ena_dev->stats_queue; 2166 2167 ret = ena_get_dev_stats(ena_dev, &ctx, 2168 ENA_ADMIN_GET_STATS_TYPE_EXTENDED); 2169 if (ret < 0) 2170 goto free_ext_stats_mem; 2171 2172 ret = snprintf(buff, len, "%s", (char *)virt_addr); 2173 2174free_ext_stats_mem: 2175 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, len, virt_addr, phys_addr, 2176 mem_handle); 2177done: 2178 return ret; 2179} 2180#endif 2181 2182int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu) 2183{ 2184 struct ena_com_admin_queue *admin_queue; 2185 struct ena_admin_set_feat_cmd cmd; 2186 struct ena_admin_set_feat_resp resp; 2187 int ret; 2188 2189 if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) { 2190 ena_trc_dbg("Feature %d isn't supported\n", ENA_ADMIN_MTU); 2191 return ENA_COM_UNSUPPORTED; 2192 } 2193 2194 memset(&cmd, 0x0, sizeof(cmd)); 2195 admin_queue = &ena_dev->admin_queue; 2196 2197 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2198 cmd.aq_common_descriptor.flags = 0; 2199 cmd.feat_common.feature_id = ENA_ADMIN_MTU; 2200 cmd.u.mtu.mtu = mtu; 2201 2202 ret = ena_com_execute_admin_command(admin_queue, 2203 (struct ena_admin_aq_entry *)&cmd, 2204 sizeof(cmd), 2205 (struct ena_admin_acq_entry *)&resp, 2206 sizeof(resp)); 2207 2208 if (unlikely(ret)) 2209 ena_trc_err("Failed to set mtu %d. error: %d\n", mtu, ret); 2210 2211 return ret; 2212} 2213 2214int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, 2215 struct ena_admin_feature_offload_desc *offload) 2216{ 2217 int ret; 2218 struct ena_admin_get_feat_resp resp; 2219 2220 ret = ena_com_get_feature(ena_dev, &resp, 2221 ENA_ADMIN_STATELESS_OFFLOAD_CONFIG); 2222 if (unlikely(ret)) { 2223 ena_trc_err("Failed to get offload capabilities %d\n", ret); 2224 return ret; 2225 } 2226 2227 memcpy(offload, &resp.u.offload, sizeof(resp.u.offload)); 2228 2229 return 0; 2230} 2231 2232int ena_com_set_hash_function(struct ena_com_dev *ena_dev) 2233{ 2234 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 2235 struct ena_rss *rss = &ena_dev->rss; 2236 struct ena_admin_set_feat_cmd cmd; 2237 struct ena_admin_set_feat_resp resp; 2238 struct ena_admin_get_feat_resp get_resp; 2239 int ret; 2240 2241 if (!ena_com_check_supported_feature_id(ena_dev, 2242 ENA_ADMIN_RSS_HASH_FUNCTION)) { 2243 ena_trc_dbg("Feature %d isn't supported\n", 2244 ENA_ADMIN_RSS_HASH_FUNCTION); 2245 return ENA_COM_UNSUPPORTED; 2246 } 2247 2248 /* Validate hash function is supported */ 2249 ret = ena_com_get_feature(ena_dev, &get_resp, 2250 ENA_ADMIN_RSS_HASH_FUNCTION); 2251 if (unlikely(ret)) 2252 return ret; 2253 2254 if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) { 2255 ena_trc_err("Func hash %d isn't supported by device, abort\n", 2256 rss->hash_func); 2257 return ENA_COM_UNSUPPORTED; 2258 } 2259 2260 memset(&cmd, 0x0, sizeof(cmd)); 2261 2262 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2263 cmd.aq_common_descriptor.flags = 2264 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 2265 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION; 2266 cmd.u.flow_hash_func.init_val = rss->hash_init_val; 2267 cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func; 2268 2269 ret = ena_com_mem_addr_set(ena_dev, 2270 &cmd.control_buffer.address, 2271 rss->hash_key_dma_addr); 2272 if (unlikely(ret)) { 2273 ena_trc_err("memory address set failed\n"); 2274 return ret; 2275 } 2276 2277 cmd.control_buffer.length = sizeof(*rss->hash_key); 2278 2279 ret = ena_com_execute_admin_command(admin_queue, 2280 (struct ena_admin_aq_entry *)&cmd, 2281 sizeof(cmd), 2282 (struct ena_admin_acq_entry *)&resp, 2283 sizeof(resp)); 2284 if (unlikely(ret)) { 2285 ena_trc_err("Failed to set hash function %d. error: %d\n", 2286 rss->hash_func, ret); 2287 return ENA_COM_INVAL; 2288 } 2289 2290 return 0; 2291} 2292 2293int ena_com_fill_hash_function(struct ena_com_dev *ena_dev, 2294 enum ena_admin_hash_functions func, 2295 const u8 *key, u16 key_len, u32 init_val) 2296{ 2297 struct ena_rss *rss = &ena_dev->rss; 2298 struct ena_admin_get_feat_resp get_resp; 2299 struct ena_admin_feature_rss_flow_hash_control *hash_key = 2300 rss->hash_key; 2301 int rc; 2302 2303 /* Make sure size is a mult of DWs */ 2304 if (unlikely(key_len & 0x3)) 2305 return ENA_COM_INVAL; 2306 2307 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2308 ENA_ADMIN_RSS_HASH_FUNCTION, 2309 rss->hash_key_dma_addr, 2310 sizeof(*rss->hash_key)); 2311 if (unlikely(rc)) 2312 return rc; 2313 2314 if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) { 2315 ena_trc_err("Flow hash function %d isn't supported\n", func); 2316 return ENA_COM_UNSUPPORTED; 2317 } 2318 2319 switch (func) { 2320 case ENA_ADMIN_TOEPLITZ: 2321 if (key_len > sizeof(hash_key->key)) { 2322 ena_trc_err("key len (%hu) is bigger than the max supported (%zu)\n", 2323 key_len, sizeof(hash_key->key)); 2324 return ENA_COM_INVAL; 2325 } 2326 2327 memcpy(hash_key->key, key, key_len); 2328 rss->hash_init_val = init_val; 2329 hash_key->keys_num = key_len >> 2; 2330 break; 2331 case ENA_ADMIN_CRC32: 2332 rss->hash_init_val = init_val; 2333 break; 2334 default: 2335 ena_trc_err("Invalid hash function (%d)\n", func); 2336 return ENA_COM_INVAL; 2337 } 2338 2339 rc = ena_com_set_hash_function(ena_dev); 2340 2341 /* Restore the old function */ 2342 if (unlikely(rc)) 2343 ena_com_get_hash_function(ena_dev, NULL, NULL); 2344 2345 return rc; 2346} 2347 2348int ena_com_get_hash_function(struct ena_com_dev *ena_dev, 2349 enum ena_admin_hash_functions *func, 2350 u8 *key) 2351{ 2352 struct ena_rss *rss = &ena_dev->rss; 2353 struct ena_admin_get_feat_resp get_resp; 2354 struct ena_admin_feature_rss_flow_hash_control *hash_key = 2355 rss->hash_key; 2356 int rc; 2357 2358 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2359 ENA_ADMIN_RSS_HASH_FUNCTION, 2360 rss->hash_key_dma_addr, 2361 sizeof(*rss->hash_key)); 2362 if (unlikely(rc)) 2363 return rc; 2364 2365 rss->hash_func = get_resp.u.flow_hash_func.selected_func; 2366 if (func) 2367 *func = rss->hash_func; 2368 2369 if (key) 2370 memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2); 2371 2372 return 0; 2373} 2374 2375int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev, 2376 enum ena_admin_flow_hash_proto proto, 2377 u16 *fields) 2378{ 2379 struct ena_rss *rss = &ena_dev->rss; 2380 struct ena_admin_get_feat_resp get_resp; 2381 int rc; 2382 2383 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2384 ENA_ADMIN_RSS_HASH_INPUT, 2385 rss->hash_ctrl_dma_addr, 2386 sizeof(*rss->hash_ctrl)); 2387 if (unlikely(rc)) 2388 return rc; 2389 2390 if (fields) 2391 *fields = rss->hash_ctrl->selected_fields[proto].fields; 2392 2393 return 0; 2394} 2395 2396int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev) 2397{ 2398 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 2399 struct ena_rss *rss = &ena_dev->rss; 2400 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl; 2401 struct ena_admin_set_feat_cmd cmd; 2402 struct ena_admin_set_feat_resp resp; 2403 int ret; 2404 2405 if (!ena_com_check_supported_feature_id(ena_dev, 2406 ENA_ADMIN_RSS_HASH_INPUT)) { 2407 ena_trc_dbg("Feature %d isn't supported\n", 2408 ENA_ADMIN_RSS_HASH_INPUT); 2409 return ENA_COM_UNSUPPORTED; 2410 } 2411 2412 memset(&cmd, 0x0, sizeof(cmd)); 2413 2414 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2415 cmd.aq_common_descriptor.flags = 2416 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 2417 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT; 2418 cmd.u.flow_hash_input.enabled_input_sort = 2419 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK | 2420 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK; 2421 2422 ret = ena_com_mem_addr_set(ena_dev, 2423 &cmd.control_buffer.address, 2424 rss->hash_ctrl_dma_addr); 2425 if (unlikely(ret)) { 2426 ena_trc_err("memory address set failed\n"); 2427 return ret; 2428 } 2429 cmd.control_buffer.length = sizeof(*hash_ctrl); 2430 2431 ret = ena_com_execute_admin_command(admin_queue, 2432 (struct ena_admin_aq_entry *)&cmd, 2433 sizeof(cmd), 2434 (struct ena_admin_acq_entry *)&resp, 2435 sizeof(resp)); 2436 if (unlikely(ret)) 2437 ena_trc_err("Failed to set hash input. error: %d\n", ret); 2438 2439 return ret; 2440} 2441 2442int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev) 2443{ 2444 struct ena_rss *rss = &ena_dev->rss; 2445 struct ena_admin_feature_rss_hash_control *hash_ctrl = 2446 rss->hash_ctrl; 2447 u16 available_fields = 0; 2448 int rc, i; 2449 2450 /* Get the supported hash input */ 2451 rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL); 2452 if (unlikely(rc)) 2453 return rc; 2454 2455 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields = 2456 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2457 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2458 2459 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields = 2460 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2461 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2462 2463 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields = 2464 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2465 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2466 2467 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields = 2468 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2469 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2470 2471 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields = 2472 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA; 2473 2474 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields = 2475 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA; 2476 2477 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields = 2478 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA; 2479 2480 hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields = 2481 ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA; 2482 2483 for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) { 2484 available_fields = hash_ctrl->selected_fields[i].fields & 2485 hash_ctrl->supported_fields[i].fields; 2486 if (available_fields != hash_ctrl->selected_fields[i].fields) { 2487 ena_trc_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n", 2488 i, hash_ctrl->supported_fields[i].fields, 2489 hash_ctrl->selected_fields[i].fields); 2490 return ENA_COM_UNSUPPORTED; 2491 } 2492 } 2493 2494 rc = ena_com_set_hash_ctrl(ena_dev); 2495 2496 /* In case of failure, restore the old hash ctrl */ 2497 if (unlikely(rc)) 2498 ena_com_get_hash_ctrl(ena_dev, 0, NULL); 2499 2500 return rc; 2501} 2502 2503int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev, 2504 enum ena_admin_flow_hash_proto proto, 2505 u16 hash_fields) 2506{ 2507 struct ena_rss *rss = &ena_dev->rss; 2508 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl; 2509 u16 supported_fields; 2510 int rc; 2511 2512 if (proto >= ENA_ADMIN_RSS_PROTO_NUM) { 2513 ena_trc_err("Invalid proto num (%u)\n", proto); 2514 return ENA_COM_INVAL; 2515 } 2516 2517 /* Get the ctrl table */ 2518 rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL); 2519 if (unlikely(rc)) 2520 return rc; 2521 2522 /* Make sure all the fields are supported */ 2523 supported_fields = hash_ctrl->supported_fields[proto].fields; 2524 if ((hash_fields & supported_fields) != hash_fields) { 2525 ena_trc_err("proto %d doesn't support the required fields %x. supports only: %x\n", 2526 proto, hash_fields, supported_fields); 2527 } 2528 2529 hash_ctrl->selected_fields[proto].fields = hash_fields; 2530 2531 rc = ena_com_set_hash_ctrl(ena_dev); 2532 2533 /* In case of failure, restore the old hash ctrl */ 2534 if (unlikely(rc)) 2535 ena_com_get_hash_ctrl(ena_dev, 0, NULL); 2536 2537 return 0; 2538} 2539 2540int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev, 2541 u16 entry_idx, u16 entry_value) 2542{ 2543 struct ena_rss *rss = &ena_dev->rss; 2544 2545 if (unlikely(entry_idx >= (1 << rss->tbl_log_size))) 2546 return ENA_COM_INVAL; 2547 2548 if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES))) 2549 return ENA_COM_INVAL; 2550 2551 rss->host_rss_ind_tbl[entry_idx] = entry_value; 2552 2553 return 0; 2554} 2555 2556int ena_com_indirect_table_set(struct ena_com_dev *ena_dev) 2557{ 2558 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 2559 struct ena_rss *rss = &ena_dev->rss; 2560 struct ena_admin_set_feat_cmd cmd; 2561 struct ena_admin_set_feat_resp resp; 2562 int ret; 2563 2564 if (!ena_com_check_supported_feature_id(ena_dev, 2565 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) { 2566 ena_trc_dbg("Feature %d isn't supported\n", 2567 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG); 2568 return ENA_COM_UNSUPPORTED; 2569 } 2570 2571 ret = ena_com_ind_tbl_convert_to_device(ena_dev); 2572 if (ret) { 2573 ena_trc_err("Failed to convert host indirection table to device table\n"); 2574 return ret; 2575 } 2576 2577 memset(&cmd, 0x0, sizeof(cmd)); 2578 2579 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2580 cmd.aq_common_descriptor.flags = 2581 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 2582 cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG; 2583 cmd.u.ind_table.size = rss->tbl_log_size; 2584 cmd.u.ind_table.inline_index = 0xFFFFFFFF; 2585 2586 ret = ena_com_mem_addr_set(ena_dev, 2587 &cmd.control_buffer.address, 2588 rss->rss_ind_tbl_dma_addr); 2589 if (unlikely(ret)) { 2590 ena_trc_err("memory address set failed\n"); 2591 return ret; 2592 } 2593 2594 cmd.control_buffer.length = (1ULL << rss->tbl_log_size) * 2595 sizeof(struct ena_admin_rss_ind_table_entry); 2596 2597 ret = ena_com_execute_admin_command(admin_queue, 2598 (struct ena_admin_aq_entry *)&cmd, 2599 sizeof(cmd), 2600 (struct ena_admin_acq_entry *)&resp, 2601 sizeof(resp)); 2602 2603 if (unlikely(ret)) 2604 ena_trc_err("Failed to set indirect table. error: %d\n", ret); 2605 2606 return ret; 2607} 2608 2609int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl) 2610{ 2611 struct ena_rss *rss = &ena_dev->rss; 2612 struct ena_admin_get_feat_resp get_resp; 2613 u32 tbl_size; 2614 int i, rc; 2615 2616 tbl_size = (1ULL << rss->tbl_log_size) * 2617 sizeof(struct ena_admin_rss_ind_table_entry); 2618 2619 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2620 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG, 2621 rss->rss_ind_tbl_dma_addr, 2622 tbl_size); 2623 if (unlikely(rc)) 2624 return rc; 2625 2626 if (!ind_tbl) 2627 return 0; 2628 2629 rc = ena_com_ind_tbl_convert_from_device(ena_dev); 2630 if (unlikely(rc)) 2631 return rc; 2632 2633 for (i = 0; i < (1 << rss->tbl_log_size); i++) 2634 ind_tbl[i] = rss->host_rss_ind_tbl[i]; 2635 2636 return 0; 2637} 2638 2639int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size) 2640{ 2641 int rc; 2642 2643 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss)); 2644 2645 rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size); 2646 if (unlikely(rc)) 2647 goto err_indr_tbl; 2648 2649 rc = ena_com_hash_key_allocate(ena_dev); 2650 if (unlikely(rc)) 2651 goto err_hash_key; 2652 2653 rc = ena_com_hash_ctrl_init(ena_dev); 2654 if (unlikely(rc)) 2655 goto err_hash_ctrl; 2656 2657 return 0; 2658 2659err_hash_ctrl: 2660 ena_com_hash_key_destroy(ena_dev); 2661err_hash_key: 2662 ena_com_indirect_table_destroy(ena_dev); 2663err_indr_tbl: 2664 2665 return rc; 2666} 2667 2668void ena_com_rss_destroy(struct ena_com_dev *ena_dev) 2669{ 2670 ena_com_indirect_table_destroy(ena_dev); 2671 ena_com_hash_key_destroy(ena_dev); 2672 ena_com_hash_ctrl_destroy(ena_dev); 2673 2674 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss)); 2675} 2676 2677int ena_com_allocate_host_info(struct ena_com_dev *ena_dev) 2678{ 2679 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2680 2681 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 2682 SZ_4K, 2683 host_attr->host_info, 2684 host_attr->host_info_dma_addr, 2685 host_attr->host_info_dma_handle); 2686 if (unlikely(!host_attr->host_info)) 2687 return ENA_COM_NO_MEM; 2688 2689 return 0; 2690} 2691 2692int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, 2693 u32 debug_area_size) 2694{ 2695 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2696 2697 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 2698 debug_area_size, 2699 host_attr->debug_area_virt_addr, 2700 host_attr->debug_area_dma_addr, 2701 host_attr->debug_area_dma_handle); 2702 if (unlikely(!host_attr->debug_area_virt_addr)) { 2703 host_attr->debug_area_size = 0; 2704 return ENA_COM_NO_MEM; 2705 } 2706 2707 host_attr->debug_area_size = debug_area_size; 2708 2709 return 0; 2710} 2711 2712void ena_com_delete_host_info(struct ena_com_dev *ena_dev) 2713{ 2714 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2715 2716 if (host_attr->host_info) { 2717 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 2718 SZ_4K, 2719 host_attr->host_info, 2720 host_attr->host_info_dma_addr, 2721 host_attr->host_info_dma_handle); 2722 host_attr->host_info = NULL; 2723 } 2724} 2725 2726void ena_com_delete_debug_area(struct ena_com_dev *ena_dev) 2727{ 2728 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2729 2730 if (host_attr->debug_area_virt_addr) { 2731 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 2732 host_attr->debug_area_size, 2733 host_attr->debug_area_virt_addr, 2734 host_attr->debug_area_dma_addr, 2735 host_attr->debug_area_dma_handle); 2736 host_attr->debug_area_virt_addr = NULL; 2737 } 2738} 2739 2740int ena_com_set_host_attributes(struct ena_com_dev *ena_dev) 2741{ 2742 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2743 struct ena_com_admin_queue *admin_queue; 2744 struct ena_admin_set_feat_cmd cmd; 2745 struct ena_admin_set_feat_resp resp; 2746 2747 int ret; 2748 2749 /* Host attribute config is called before ena_com_get_dev_attr_feat 2750 * so ena_com can't check if the feature is supported. 2751 */ 2752 2753 memset(&cmd, 0x0, sizeof(cmd)); 2754 admin_queue = &ena_dev->admin_queue; 2755 2756 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2757 cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG; 2758 2759 ret = ena_com_mem_addr_set(ena_dev, 2760 &cmd.u.host_attr.debug_ba, 2761 host_attr->debug_area_dma_addr); 2762 if (unlikely(ret)) { 2763 ena_trc_err("memory address set failed\n"); 2764 return ret; 2765 } 2766 2767 ret = ena_com_mem_addr_set(ena_dev, 2768 &cmd.u.host_attr.os_info_ba, 2769 host_attr->host_info_dma_addr); 2770 if (unlikely(ret)) { 2771 ena_trc_err("memory address set failed\n"); 2772 return ret; 2773 } 2774 2775 cmd.u.host_attr.debug_area_size = host_attr->debug_area_size; 2776 2777 ret = ena_com_execute_admin_command(admin_queue, 2778 (struct ena_admin_aq_entry *)&cmd, 2779 sizeof(cmd), 2780 (struct ena_admin_acq_entry *)&resp, 2781 sizeof(resp)); 2782 2783 if (unlikely(ret)) 2784 ena_trc_err("Failed to set host attributes: %d\n", ret); 2785 2786 return ret; 2787} 2788 2789/* Interrupt moderation */ 2790bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev) 2791{ 2792 return ena_com_check_supported_feature_id(ena_dev, 2793 ENA_ADMIN_INTERRUPT_MODERATION); 2794} 2795 2796int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, 2797 u32 tx_coalesce_usecs) 2798{ 2799 if (!ena_dev->intr_delay_resolution) { 2800 ena_trc_err("Illegal interrupt delay granularity value\n"); 2801 return ENA_COM_FAULT; 2802 } 2803 2804 ena_dev->intr_moder_tx_interval = tx_coalesce_usecs / 2805 ena_dev->intr_delay_resolution; 2806 2807 return 0; 2808} 2809 2810int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, 2811 u32 rx_coalesce_usecs) 2812{ 2813 if (!ena_dev->intr_delay_resolution) { 2814 ena_trc_err("Illegal interrupt delay granularity value\n"); 2815 return ENA_COM_FAULT; 2816 } 2817 2818 /* We use LOWEST entry of moderation table for storing 2819 * nonadaptive interrupt coalescing values 2820 */ 2821 ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval = 2822 rx_coalesce_usecs / ena_dev->intr_delay_resolution; 2823 2824 return 0; 2825} 2826 2827void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev) 2828{ 2829 size_t size; 2830 2831 size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS; 2832 if (ena_dev->intr_moder_tbl) 2833 ENA_MEM_FREE(ena_dev->dmadev, ena_dev->intr_moder_tbl, size); 2834 ena_dev->intr_moder_tbl = NULL; 2835} 2836 2837int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev) 2838{ 2839 struct ena_admin_get_feat_resp get_resp; 2840 u16 delay_resolution; 2841 int rc; 2842 2843 rc = ena_com_get_feature(ena_dev, &get_resp, 2844 ENA_ADMIN_INTERRUPT_MODERATION); 2845 2846 if (rc) { 2847 if (rc == ENA_COM_UNSUPPORTED) { 2848 ena_trc_dbg("Feature %d isn't supported\n", 2849 ENA_ADMIN_INTERRUPT_MODERATION); 2850 rc = 0; 2851 } else { 2852 ena_trc_err("Failed to get interrupt moderation admin cmd. rc: %d\n", 2853 rc); 2854 } 2855 2856 /* no moderation supported, disable adaptive support */ 2857 ena_com_disable_adaptive_moderation(ena_dev); 2858 return rc; 2859 } 2860 2861 rc = ena_com_init_interrupt_moderation_table(ena_dev); 2862 if (rc) 2863 goto err; 2864 2865 /* if moderation is supported by device we set adaptive moderation */ 2866 delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution; 2867 ena_com_update_intr_delay_resolution(ena_dev, delay_resolution); 2868 ena_com_enable_adaptive_moderation(ena_dev); 2869 2870 return 0; 2871err: 2872 ena_com_destroy_interrupt_moderation(ena_dev); 2873 return rc; 2874} 2875 2876void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev) 2877{ 2878 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; 2879 2880 if (!intr_moder_tbl) 2881 return; 2882 2883 intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval = 2884 ENA_INTR_LOWEST_USECS; 2885 intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval = 2886 ENA_INTR_LOWEST_PKTS; 2887 intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval = 2888 ENA_INTR_LOWEST_BYTES; 2889 2890 intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval = 2891 ENA_INTR_LOW_USECS; 2892 intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval = 2893 ENA_INTR_LOW_PKTS; 2894 intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval = 2895 ENA_INTR_LOW_BYTES; 2896 2897 intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval = 2898 ENA_INTR_MID_USECS; 2899 intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval = 2900 ENA_INTR_MID_PKTS; 2901 intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval = 2902 ENA_INTR_MID_BYTES; 2903 2904 intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval = 2905 ENA_INTR_HIGH_USECS; 2906 intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval = 2907 ENA_INTR_HIGH_PKTS; 2908 intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval = 2909 ENA_INTR_HIGH_BYTES; 2910 2911 intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval = 2912 ENA_INTR_HIGHEST_USECS; 2913 intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval = 2914 ENA_INTR_HIGHEST_PKTS; 2915 intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval = 2916 ENA_INTR_HIGHEST_BYTES; 2917} 2918 2919unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev) 2920{ 2921 return ena_dev->intr_moder_tx_interval; 2922} 2923 2924unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev) 2925{ 2926 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; 2927 2928 if (intr_moder_tbl) 2929 return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval; 2930 2931 return 0; 2932} 2933 2934void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev, 2935 enum ena_intr_moder_level level, 2936 struct ena_intr_moder_entry *entry) 2937{ 2938 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; 2939 2940 if (level >= ENA_INTR_MAX_NUM_OF_LEVELS) 2941 return; 2942 2943 intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval; 2944 if (ena_dev->intr_delay_resolution) 2945 intr_moder_tbl[level].intr_moder_interval /= 2946 ena_dev->intr_delay_resolution; 2947 intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval; 2948 2949 /* use hardcoded value until ethtool supports bytecount parameter */ 2950 if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED) 2951 intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval; 2952} 2953 2954void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev, 2955 enum ena_intr_moder_level level, 2956 struct ena_intr_moder_entry *entry) 2957{ 2958 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; 2959 2960 if (level >= ENA_INTR_MAX_NUM_OF_LEVELS) 2961 return; 2962 2963 entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval; 2964 if (ena_dev->intr_delay_resolution) 2965 entry->intr_moder_interval *= ena_dev->intr_delay_resolution; 2966 entry->pkts_per_interval = 2967 intr_moder_tbl[level].pkts_per_interval; 2968 entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval; 2969} 2970 2971int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, 2972 struct ena_admin_feature_llq_desc *llq) 2973{ 2974 int rc; 2975 int size; 2976 2977 if (llq->max_llq_num == 0) { 2978 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2979 return 0; 2980 } 2981 2982 rc = ena_com_config_llq_info(ena_dev, llq); 2983 if (rc) 2984 return rc; 2985 2986 /* Validate the descriptor is not too big */ 2987 size = ena_dev->tx_max_header_size; 2988 size += ena_dev->llq_info.descs_num_before_header * 2989 sizeof(struct ena_eth_io_tx_desc); 2990 2991 if (unlikely(ena_dev->llq_info.desc_list_entry_size < size)) { 2992 ena_trc_err("the size of the LLQ entry is smaller than needed\n"); 2993 return ENA_COM_INVAL; 2994 } 2995 2996 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 2997 2998 return 0; 2999} 3000