1/* 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Cavium, Inc. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33/*$FreeBSD: stable/11/sys/dev/liquidio/base/lio_request_manager.c 325618 2017-11-09 19:52:56Z sbruno $*/ 34 35#include "lio_bsd.h" 36#include "lio_common.h" 37#include "lio_droq.h" 38#include "lio_iq.h" 39#include "lio_response_manager.h" 40#include "lio_device.h" 41#include "lio_main.h" 42#include "lio_network.h" 43#include "cn23xx_pf_device.h" 44#include "lio_rxtx.h" 45 46struct lio_iq_post_status { 47 int status; 48 int index; 49}; 50 51static void lio_check_db_timeout(void *arg, int pending); 52static void __lio_check_db_timeout(struct octeon_device *oct, 53 uint64_t iq_no); 54 55/* Return 0 on success, 1 on failure */ 56int 57lio_init_instr_queue(struct octeon_device *oct, union octeon_txpciq txpciq, 58 uint32_t num_descs) 59{ 60 struct lio_instr_queue *iq; 61 struct lio_iq_config *conf = NULL; 62 struct lio_tq *db_tq; 63 struct lio_request_list *request_buf; 64 bus_size_t max_size; 65 uint32_t iq_no = (uint32_t)txpciq.s.q_no; 66 uint32_t q_size; 67 int error, i; 68 69 if (LIO_CN23XX_PF(oct)) 70 conf = &(LIO_GET_IQ_CFG(LIO_CHIP_CONF(oct, cn23xx_pf))); 71 if (conf == NULL) { 72 lio_dev_err(oct, "Unsupported Chip %x\n", oct->chip_id); 73 return (1); 74 } 75 76 q_size = (uint32_t)conf->instr_type * num_descs; 77 iq = oct->instr_queue[iq_no]; 78 iq->oct_dev = oct; 79 80 max_size = LIO_CN23XX_PKI_MAX_FRAME_SIZE * num_descs; 81 82 error = bus_dma_tag_create(bus_get_dma_tag(oct->device), /* parent */ 83 1, 0, /* alignment, bounds */ 84 BUS_SPACE_MAXADDR, /* lowaddr */ 85 BUS_SPACE_MAXADDR, /* highaddr */ 86 NULL, NULL, /* filter, filterarg */ 87 max_size, /* maxsize */ 88 LIO_MAX_SG, /* nsegments */ 89 PAGE_SIZE, /* maxsegsize */ 90 0, /* flags */ 91 NULL, /* lockfunc */ 92 NULL, /* lockfuncarg */ 93 &iq->txtag); 94 if (error) { 95 lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n", 96 iq_no); 97 return (1); 98 } 99 100 iq->base_addr = lio_dma_alloc(q_size, (vm_paddr_t *)&iq->base_addr_dma); 101 if (!iq->base_addr) { 102 lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n", 103 iq_no); 104 return (1); 105 } 106 107 iq->max_count = num_descs; 108 109 /* 110 * Initialize a list to holds requests that have been posted to 111 * Octeon but has yet to be fetched by octeon 112 */ 113 iq->request_list = malloc(sizeof(*iq->request_list) * num_descs, 114 M_DEVBUF, M_NOWAIT | M_ZERO); 115 if (iq->request_list == NULL) { 116 lio_dev_err(oct, "Alloc failed for IQ[%d] nr free list\n", 117 iq_no); 118 return (1); 119 } 120 121 lio_dev_dbg(oct, "IQ[%d]: base: %p basedma: %llx count: %d\n", 122 iq_no, iq->base_addr, LIO_CAST64(iq->base_addr_dma), 123 iq->max_count); 124 125 /* Create the descriptor buffer dma maps */ 126 request_buf = iq->request_list; 127 for (i = 0; i < num_descs; i++, request_buf++) { 128 error = bus_dmamap_create(iq->txtag, 0, &request_buf->map); 129 if (error) { 130 lio_dev_err(oct, "Unable to create TX DMA map\n"); 131 return (1); 132 } 133 } 134 135 iq->txpciq.txpciq64 = txpciq.txpciq64; 136 iq->fill_cnt = 0; 137 iq->host_write_index = 0; 138 iq->octeon_read_index = 0; 139 iq->flush_index = 0; 140 iq->last_db_time = 0; 141 iq->db_timeout = (uint32_t)conf->db_timeout; 142 atomic_store_rel_int(&iq->instr_pending, 0); 143 144 /* Initialize the lock for this instruction queue */ 145 mtx_init(&iq->lock, "Tx_lock", NULL, MTX_DEF); 146 mtx_init(&iq->post_lock, "iq_post_lock", NULL, MTX_DEF); 147 mtx_init(&iq->enq_lock, "enq_lock", NULL, MTX_DEF); 148 149 mtx_init(&iq->iq_flush_running_lock, "iq_flush_running_lock", NULL, 150 MTX_DEF); 151 152 oct->io_qmask.iq |= BIT_ULL(iq_no); 153 154 /* Set the 32B/64B mode for each input queue */ 155 oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no); 156 iq->iqcmd_64B = (conf->instr_type == 64); 157 158 oct->fn_list.setup_iq_regs(oct, iq_no); 159 160 db_tq = &oct->check_db_tq[iq_no]; 161 db_tq->tq = taskqueue_create("lio_check_db_timeout", M_WAITOK, 162 taskqueue_thread_enqueue, &db_tq->tq); 163 if (db_tq->tq == NULL) { 164 lio_dev_err(oct, "check db wq create failed for iq %d\n", 165 iq_no); 166 return (1); 167 } 168 169 TIMEOUT_TASK_INIT(db_tq->tq, &db_tq->work, 0, lio_check_db_timeout, 170 (void *)db_tq); 171 db_tq->ctxul = iq_no; 172 db_tq->ctxptr = oct; 173 174 taskqueue_start_threads(&db_tq->tq, 1, PI_NET, 175 "lio%d_check_db_timeout:%d", 176 oct->octeon_id, iq_no); 177 taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work, 1); 178 179 /* Allocate a buf ring */ 180 oct->instr_queue[iq_no]->br = 181 buf_ring_alloc(LIO_BR_SIZE, M_DEVBUF, M_WAITOK, 182 &oct->instr_queue[iq_no]->enq_lock); 183 if (oct->instr_queue[iq_no]->br == NULL) { 184 lio_dev_err(oct, "Critical Failure setting up buf ring\n"); 185 return (1); 186 } 187 188 return (0); 189} 190 191int 192lio_delete_instr_queue(struct octeon_device *oct, uint32_t iq_no) 193{ 194 struct lio_instr_queue *iq = oct->instr_queue[iq_no]; 195 struct lio_request_list *request_buf; 196 struct lio_mbuf_free_info *finfo; 197 uint64_t desc_size = 0, q_size; 198 int i; 199 200 lio_dev_dbg(oct, "%s[%d]\n", __func__, iq_no); 201 202 if (oct->check_db_tq[iq_no].tq != NULL) { 203 while (taskqueue_cancel_timeout(oct->check_db_tq[iq_no].tq, 204 &oct->check_db_tq[iq_no].work, 205 NULL)) 206 taskqueue_drain_timeout(oct->check_db_tq[iq_no].tq, 207 &oct->check_db_tq[iq_no].work); 208 taskqueue_free(oct->check_db_tq[iq_no].tq); 209 oct->check_db_tq[iq_no].tq = NULL; 210 } 211 212 if (LIO_CN23XX_PF(oct)) 213 desc_size = 214 LIO_GET_IQ_INSTR_TYPE_CFG(LIO_CHIP_CONF(oct, cn23xx_pf)); 215 216 request_buf = iq->request_list; 217 for (i = 0; i < iq->max_count; i++, request_buf++) { 218 if ((request_buf->reqtype == LIO_REQTYPE_NORESP_NET) || 219 (request_buf->reqtype == LIO_REQTYPE_NORESP_NET_SG)) { 220 if (request_buf->buf != NULL) { 221 finfo = request_buf->buf; 222 bus_dmamap_sync(iq->txtag, request_buf->map, 223 BUS_DMASYNC_POSTWRITE); 224 bus_dmamap_unload(iq->txtag, 225 request_buf->map); 226 m_freem(finfo->mb); 227 request_buf->buf = NULL; 228 if (request_buf->map != NULL) { 229 bus_dmamap_destroy(iq->txtag, 230 request_buf->map); 231 request_buf->map = NULL; 232 } 233 } else if (request_buf->map != NULL) { 234 bus_dmamap_unload(iq->txtag, request_buf->map); 235 bus_dmamap_destroy(iq->txtag, request_buf->map); 236 request_buf->map = NULL; 237 } 238 } 239 } 240 241 if (iq->br != NULL) { 242 buf_ring_free(iq->br, M_DEVBUF); 243 iq->br = NULL; 244 } 245 246 if (iq->request_list != NULL) { 247 free(iq->request_list, M_DEVBUF); 248 iq->request_list = NULL; 249 } 250 251 if (iq->txtag != NULL) { 252 bus_dma_tag_destroy(iq->txtag); 253 iq->txtag = NULL; 254 } 255 256 if (iq->base_addr) { 257 q_size = iq->max_count * desc_size; 258 lio_dma_free((uint32_t)q_size, iq->base_addr); 259 260 oct->io_qmask.iq &= ~(1ULL << iq_no); 261 bzero(oct->instr_queue[iq_no], sizeof(struct lio_instr_queue)); 262 oct->num_iqs--; 263 264 return (0); 265 } 266 267 return (1); 268} 269 270/* Return 0 on success, 1 on failure */ 271int 272lio_setup_iq(struct octeon_device *oct, int ifidx, int q_index, 273 union octeon_txpciq txpciq, uint32_t num_descs) 274{ 275 uint32_t iq_no = (uint32_t)txpciq.s.q_no; 276 277 if (oct->instr_queue[iq_no]->oct_dev != NULL) { 278 lio_dev_dbg(oct, "IQ is in use. Cannot create the IQ: %d again\n", 279 iq_no); 280 oct->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64; 281 return (0); 282 } 283 284 oct->instr_queue[iq_no]->q_index = q_index; 285 oct->instr_queue[iq_no]->ifidx = ifidx; 286 287 if (lio_init_instr_queue(oct, txpciq, num_descs)) { 288 lio_delete_instr_queue(oct, iq_no); 289 return (1); 290 } 291 292 oct->num_iqs++; 293 if (oct->fn_list.enable_io_queues(oct)) 294 return (1); 295 296 return (0); 297} 298 299int 300lio_wait_for_instr_fetch(struct octeon_device *oct) 301{ 302 int i, retry = 1000, pending, instr_cnt = 0; 303 304 do { 305 instr_cnt = 0; 306 307 for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) { 308 if (!(oct->io_qmask.iq & BIT_ULL(i))) 309 continue; 310 pending = atomic_load_acq_int( 311 &oct->instr_queue[i]->instr_pending); 312 if (pending) 313 __lio_check_db_timeout(oct, i); 314 instr_cnt += pending; 315 } 316 317 if (instr_cnt == 0) 318 break; 319 320 lio_sleep_timeout(1); 321 322 } while (retry-- && instr_cnt); 323 324 return (instr_cnt); 325} 326 327static inline void 328lio_ring_doorbell(struct octeon_device *oct, struct lio_instr_queue *iq) 329{ 330 331 if (atomic_load_acq_int(&oct->status) == LIO_DEV_RUNNING) { 332 lio_write_csr32(oct, iq->doorbell_reg, iq->fill_cnt); 333 /* make sure doorbell write goes through */ 334 __compiler_membar(); 335 iq->fill_cnt = 0; 336 iq->last_db_time = ticks; 337 return; 338 } 339} 340 341static inline void 342__lio_copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd) 343{ 344 uint8_t *iqptr, cmdsize; 345 346 cmdsize = ((iq->iqcmd_64B) ? 64 : 32); 347 iqptr = iq->base_addr + (cmdsize * iq->host_write_index); 348 349 memcpy(iqptr, cmd, cmdsize); 350} 351 352static inline struct lio_iq_post_status 353__lio_post_command2(struct lio_instr_queue *iq, uint8_t *cmd) 354{ 355 struct lio_iq_post_status st; 356 357 st.status = LIO_IQ_SEND_OK; 358 359 /* 360 * This ensures that the read index does not wrap around to the same 361 * position if queue gets full before Octeon could fetch any instr. 362 */ 363 if (atomic_load_acq_int(&iq->instr_pending) >= 364 (int32_t)(iq->max_count - 1)) { 365 st.status = LIO_IQ_SEND_FAILED; 366 st.index = -1; 367 return (st); 368 } 369 370 if (atomic_load_acq_int(&iq->instr_pending) >= 371 (int32_t)(iq->max_count - 2)) 372 st.status = LIO_IQ_SEND_STOP; 373 374 __lio_copy_cmd_into_iq(iq, cmd); 375 376 /* "index" is returned, host_write_index is modified. */ 377 st.index = iq->host_write_index; 378 iq->host_write_index = lio_incr_index(iq->host_write_index, 1, 379 iq->max_count); 380 iq->fill_cnt++; 381 382 /* 383 * Flush the command into memory. We need to be sure the data is in 384 * memory before indicating that the instruction is pending. 385 */ 386 wmb(); 387 388 atomic_add_int(&iq->instr_pending, 1); 389 390 return (st); 391} 392 393static inline void 394__lio_add_to_request_list(struct lio_instr_queue *iq, int idx, void *buf, 395 int reqtype) 396{ 397 398 iq->request_list[idx].buf = buf; 399 iq->request_list[idx].reqtype = reqtype; 400} 401 402/* Can only run in process context */ 403int 404lio_process_iq_request_list(struct octeon_device *oct, 405 struct lio_instr_queue *iq, uint32_t budget) 406{ 407 struct lio_soft_command *sc; 408 struct octeon_instr_irh *irh = NULL; 409 struct lio_mbuf_free_info *finfo; 410 void *buf; 411 uint32_t inst_count = 0; 412 uint32_t old = iq->flush_index; 413 int reqtype; 414 415 while (old != iq->octeon_read_index) { 416 reqtype = iq->request_list[old].reqtype; 417 buf = iq->request_list[old].buf; 418 finfo = buf; 419 420 if (reqtype == LIO_REQTYPE_NONE) 421 goto skip_this; 422 423 switch (reqtype) { 424 case LIO_REQTYPE_NORESP_NET: 425 lio_free_mbuf(iq, buf); 426 break; 427 case LIO_REQTYPE_NORESP_NET_SG: 428 lio_free_sgmbuf(iq, buf); 429 break; 430 case LIO_REQTYPE_RESP_NET: 431 case LIO_REQTYPE_SOFT_COMMAND: 432 sc = buf; 433 if (LIO_CN23XX_PF(oct)) 434 irh = (struct octeon_instr_irh *) 435 &sc->cmd.cmd3.irh; 436 if (irh->rflag) { 437 /* 438 * We're expecting a response from Octeon. 439 * It's up to lio_process_ordered_list() to 440 * process sc. Add sc to the ordered soft 441 * command response list because we expect 442 * a response from Octeon. 443 */ 444 mtx_lock(&oct->response_list 445 [LIO_ORDERED_SC_LIST].lock); 446 atomic_add_int(&oct->response_list 447 [LIO_ORDERED_SC_LIST]. 448 pending_req_count, 1); 449 STAILQ_INSERT_TAIL(&oct->response_list 450 [LIO_ORDERED_SC_LIST]. 451 head, &sc->node, entries); 452 mtx_unlock(&oct->response_list 453 [LIO_ORDERED_SC_LIST].lock); 454 } else { 455 if (sc->callback != NULL) { 456 /* This callback must not sleep */ 457 sc->callback(oct, LIO_REQUEST_DONE, 458 sc->callback_arg); 459 } 460 } 461 462 break; 463 default: 464 lio_dev_err(oct, "%s Unknown reqtype: %d buf: %p at idx %d\n", 465 __func__, reqtype, buf, old); 466 } 467 468 iq->request_list[old].buf = NULL; 469 iq->request_list[old].reqtype = 0; 470 471skip_this: 472 inst_count++; 473 old = lio_incr_index(old, 1, iq->max_count); 474 475 if ((budget) && (inst_count >= budget)) 476 break; 477 } 478 479 iq->flush_index = old; 480 481 return (inst_count); 482} 483 484/* Can only be called from process context */ 485int 486lio_flush_iq(struct octeon_device *oct, struct lio_instr_queue *iq, 487 uint32_t budget) 488{ 489 uint32_t inst_processed = 0; 490 uint32_t tot_inst_processed = 0; 491 int tx_done = 1; 492 493 if (!mtx_trylock(&iq->iq_flush_running_lock)) 494 return (tx_done); 495 496 mtx_lock(&iq->lock); 497 498 iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq); 499 500 do { 501 /* Process any outstanding IQ packets. */ 502 if (iq->flush_index == iq->octeon_read_index) 503 break; 504 505 if (budget) 506 inst_processed = 507 lio_process_iq_request_list(oct, iq, 508 budget - 509 tot_inst_processed); 510 else 511 inst_processed = 512 lio_process_iq_request_list(oct, iq, 0); 513 514 if (inst_processed) { 515 atomic_subtract_int(&iq->instr_pending, inst_processed); 516 iq->stats.instr_processed += inst_processed; 517 } 518 tot_inst_processed += inst_processed; 519 inst_processed = 0; 520 521 } while (tot_inst_processed < budget); 522 523 if (budget && (tot_inst_processed >= budget)) 524 tx_done = 0; 525 526 iq->last_db_time = ticks; 527 528 mtx_unlock(&iq->lock); 529 530 mtx_unlock(&iq->iq_flush_running_lock); 531 532 return (tx_done); 533} 534 535/* 536 * Process instruction queue after timeout. 537 * This routine gets called from a taskqueue or when removing the module. 538 */ 539static void 540__lio_check_db_timeout(struct octeon_device *oct, uint64_t iq_no) 541{ 542 struct lio_instr_queue *iq; 543 uint64_t next_time; 544 545 if (oct == NULL) 546 return; 547 548 iq = oct->instr_queue[iq_no]; 549 if (iq == NULL) 550 return; 551 552 if (atomic_load_acq_int(&iq->instr_pending)) { 553 /* If ticks - last_db_time < db_timeout do nothing */ 554 next_time = iq->last_db_time + lio_ms_to_ticks(iq->db_timeout); 555 if (!lio_check_timeout(ticks, next_time)) 556 return; 557 558 iq->last_db_time = ticks; 559 560 /* Flush the instruction queue */ 561 lio_flush_iq(oct, iq, 0); 562 563 lio_enable_irq(NULL, iq); 564 } 565 566 if (oct->props.ifp != NULL && iq->br != NULL) { 567 if (mtx_trylock(&iq->enq_lock)) { 568 if (!drbr_empty(oct->props.ifp, iq->br)) 569 lio_mq_start_locked(oct->props.ifp, iq); 570 571 mtx_unlock(&iq->enq_lock); 572 } 573 } 574} 575 576/* 577 * Called by the Poll thread at regular intervals to check the instruction 578 * queue for commands to be posted and for commands that were fetched by Octeon. 579 */ 580static void 581lio_check_db_timeout(void *arg, int pending) 582{ 583 struct lio_tq *db_tq = (struct lio_tq *)arg; 584 struct octeon_device *oct = db_tq->ctxptr; 585 uint64_t iq_no = db_tq->ctxul; 586 uint32_t delay = 10; 587 588 __lio_check_db_timeout(oct, iq_no); 589 taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work, 590 lio_ms_to_ticks(delay)); 591} 592 593int 594lio_send_command(struct octeon_device *oct, uint32_t iq_no, 595 uint32_t force_db, void *cmd, void *buf, 596 uint32_t datasize, uint32_t reqtype) 597{ 598 struct lio_iq_post_status st; 599 struct lio_instr_queue *iq = oct->instr_queue[iq_no]; 600 601 /* 602 * Get the lock and prevent other tasks and tx interrupt handler 603 * from running. 604 */ 605 mtx_lock(&iq->post_lock); 606 607 st = __lio_post_command2(iq, cmd); 608 609 if (st.status != LIO_IQ_SEND_FAILED) { 610 __lio_add_to_request_list(iq, st.index, buf, reqtype); 611 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize); 612 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1); 613 614 if (force_db || (st.status == LIO_IQ_SEND_STOP)) 615 lio_ring_doorbell(oct, iq); 616 } else { 617 LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1); 618 } 619 620 mtx_unlock(&iq->post_lock); 621 622 /* 623 * This is only done here to expedite packets being flushed for 624 * cases where there are no IQ completion interrupts. 625 */ 626 627 return (st.status); 628} 629 630void 631lio_prepare_soft_command(struct octeon_device *oct, struct lio_soft_command *sc, 632 uint8_t opcode, uint8_t subcode, uint32_t irh_ossp, 633 uint64_t ossp0, uint64_t ossp1) 634{ 635 struct lio_config *lio_cfg; 636 struct octeon_instr_ih3 *ih3; 637 struct octeon_instr_pki_ih3 *pki_ih3; 638 struct octeon_instr_irh *irh; 639 struct octeon_instr_rdp *rdp; 640 641 KASSERT(opcode <= 15, ("%s, %d, opcode > 15", __func__, __LINE__)); 642 KASSERT(subcode <= 127, ("%s, %d, opcode > 127", __func__, __LINE__)); 643 644 lio_cfg = lio_get_conf(oct); 645 646 if (LIO_CN23XX_PF(oct)) { 647 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; 648 649 ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind; 650 651 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3; 652 653 pki_ih3->w = 1; 654 pki_ih3->raw = 1; 655 pki_ih3->utag = 1; 656 pki_ih3->uqpg = oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg; 657 pki_ih3->utt = 1; 658 pki_ih3->tag = LIO_CONTROL; 659 pki_ih3->tagtype = LIO_ATOMIC_TAG; 660 pki_ih3->qpg = oct->instr_queue[sc->iq_no]->txpciq.s.qpg; 661 pki_ih3->pm = 0x7; 662 pki_ih3->sl = 8; 663 664 if (sc->datasize) 665 ih3->dlengsz = sc->datasize; 666 667 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; 668 irh->opcode = opcode; 669 irh->subcode = subcode; 670 671 /* opcode/subcode specific parameters (ossp) */ 672 irh->ossp = irh_ossp; 673 sc->cmd.cmd3.ossp[0] = ossp0; 674 sc->cmd.cmd3.ossp[1] = ossp1; 675 676 if (sc->rdatasize) { 677 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp; 678 rdp->pcie_port = oct->pcie_port; 679 rdp->rlen = sc->rdatasize; 680 681 irh->rflag = 1; 682 /* PKI IH3 */ 683 /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */ 684 ih3->fsz = LIO_SOFTCMDRESP_IH3; 685 } else { 686 irh->rflag = 0; 687 /* PKI IH3 */ 688 /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */ 689 ih3->fsz = LIO_PCICMD_O3; 690 } 691 } 692} 693 694int 695lio_send_soft_command(struct octeon_device *oct, struct lio_soft_command *sc) 696{ 697 struct octeon_instr_ih3 *ih3; 698 struct octeon_instr_irh *irh; 699 uint32_t len = 0; 700 701 if (LIO_CN23XX_PF(oct)) { 702 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; 703 if (ih3->dlengsz) { 704 KASSERT(sc->dmadptr, ("%s, %d, sc->dmadptr is NULL", 705 __func__, __LINE__)); 706 sc->cmd.cmd3.dptr = sc->dmadptr; 707 } 708 709 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; 710 if (irh->rflag) { 711 KASSERT(sc->dmarptr, ("%s, %d, sc->dmarptr is NULL", 712 __func__, __LINE__)); 713 KASSERT(sc->status_word, ("%s, %d, sc->status_word is NULL", 714 __func__, __LINE__)); 715 *sc->status_word = COMPLETION_WORD_INIT; 716 sc->cmd.cmd3.rptr = sc->dmarptr; 717 } 718 len = (uint32_t)ih3->dlengsz; 719 } 720 if (sc->wait_time) 721 sc->timeout = ticks + lio_ms_to_ticks(sc->wait_time); 722 723 return (lio_send_command(oct, sc->iq_no, 1, &sc->cmd, sc, 724 len, LIO_REQTYPE_SOFT_COMMAND)); 725} 726 727int 728lio_setup_sc_buffer_pool(struct octeon_device *oct) 729{ 730 struct lio_soft_command *sc; 731 uint64_t dma_addr; 732 int i; 733 734 STAILQ_INIT(&oct->sc_buf_pool.head); 735 mtx_init(&oct->sc_buf_pool.lock, "sc_pool_lock", NULL, MTX_DEF); 736 atomic_store_rel_int(&oct->sc_buf_pool.alloc_buf_count, 0); 737 738 for (i = 0; i < LIO_MAX_SOFT_COMMAND_BUFFERS; i++) { 739 sc = (struct lio_soft_command *) 740 lio_dma_alloc(LIO_SOFT_COMMAND_BUFFER_SIZE, (vm_paddr_t *)&dma_addr); 741 if (sc == NULL) { 742 lio_free_sc_buffer_pool(oct); 743 return (1); 744 } 745 746 sc->dma_addr = dma_addr; 747 sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE; 748 749 STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries); 750 } 751 752 return (0); 753} 754 755int 756lio_free_sc_buffer_pool(struct octeon_device *oct) 757{ 758 struct lio_stailq_node *tmp, *tmp2; 759 struct lio_soft_command *sc; 760 761 mtx_lock(&oct->sc_buf_pool.lock); 762 763 STAILQ_FOREACH_SAFE(tmp, &oct->sc_buf_pool.head, entries, tmp2) { 764 sc = LIO_STAILQ_FIRST_ENTRY(&oct->sc_buf_pool.head, 765 struct lio_soft_command, node); 766 767 STAILQ_REMOVE_HEAD(&oct->sc_buf_pool.head, entries); 768 769 lio_dma_free(sc->size, sc); 770 } 771 772 STAILQ_INIT(&oct->sc_buf_pool.head); 773 774 mtx_unlock(&oct->sc_buf_pool.lock); 775 776 return (0); 777} 778 779struct lio_soft_command * 780lio_alloc_soft_command(struct octeon_device *oct, uint32_t datasize, 781 uint32_t rdatasize, uint32_t ctxsize) 782{ 783 struct lio_soft_command *sc = NULL; 784 struct lio_stailq_node *tmp; 785 uint64_t dma_addr; 786 uint32_t size; 787 uint32_t offset = sizeof(struct lio_soft_command); 788 789 KASSERT((offset + datasize + rdatasize + ctxsize) <= 790 LIO_SOFT_COMMAND_BUFFER_SIZE, 791 ("%s, %d, offset + datasize + rdatasize + ctxsize > LIO_SOFT_COMMAND_BUFFER_SIZE", 792 __func__, __LINE__)); 793 794 mtx_lock(&oct->sc_buf_pool.lock); 795 796 if (STAILQ_EMPTY(&oct->sc_buf_pool.head)) { 797 mtx_unlock(&oct->sc_buf_pool.lock); 798 return (NULL); 799 } 800 tmp = STAILQ_LAST(&oct->sc_buf_pool.head, lio_stailq_node, entries); 801 802 STAILQ_REMOVE(&oct->sc_buf_pool.head, tmp, lio_stailq_node, entries); 803 804 atomic_add_int(&oct->sc_buf_pool.alloc_buf_count, 1); 805 806 mtx_unlock(&oct->sc_buf_pool.lock); 807 808 sc = (struct lio_soft_command *)tmp; 809 810 dma_addr = sc->dma_addr; 811 size = sc->size; 812 813 bzero(sc, sc->size); 814 815 sc->dma_addr = dma_addr; 816 sc->size = size; 817 818 if (ctxsize) { 819 sc->ctxptr = (uint8_t *)sc + offset; 820 sc->ctxsize = ctxsize; 821 } 822 823 /* Start data at 128 byte boundary */ 824 offset = (offset + ctxsize + 127) & 0xffffff80; 825 826 if (datasize) { 827 sc->virtdptr = (uint8_t *)sc + offset; 828 sc->dmadptr = dma_addr + offset; 829 sc->datasize = datasize; 830 } 831 /* Start rdata at 128 byte boundary */ 832 offset = (offset + datasize + 127) & 0xffffff80; 833 834 if (rdatasize) { 835 KASSERT(rdatasize >= 16, ("%s, %d, rdatasize < 16", __func__, 836 __LINE__)); 837 sc->virtrptr = (uint8_t *)sc + offset; 838 sc->dmarptr = dma_addr + offset; 839 sc->rdatasize = rdatasize; 840 sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) + 841 rdatasize - 8); 842 } 843 return (sc); 844} 845 846void 847lio_free_soft_command(struct octeon_device *oct, 848 struct lio_soft_command *sc) 849{ 850 851 mtx_lock(&oct->sc_buf_pool.lock); 852 853 STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries); 854 855 atomic_subtract_int(&oct->sc_buf_pool.alloc_buf_count, 1); 856 857 mtx_unlock(&oct->sc_buf_pool.lock); 858} 859