405 q->rspq_size = RSPQ_Q_SIZE; 406 q->fl_size = fl_q_size; 407 q->jumbo_size = jumbo_q_size; 408 q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; 409 q->txq_size[TXQ_OFLD] = 1024; 410 q->txq_size[TXQ_CTRL] = 256; 411 q->cong_thres = 0; 412 } 413} 414 415int 416t3_sge_alloc(adapter_t *sc) 417{ 418 419 /* The parent tag. */ 420 if (bus_dma_tag_create( NULL, /* parent */ 421 1, 0, /* algnmnt, boundary */ 422 BUS_SPACE_MAXADDR, /* lowaddr */ 423 BUS_SPACE_MAXADDR, /* highaddr */ 424 NULL, NULL, /* filter, filterarg */ 425 BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 426 BUS_SPACE_UNRESTRICTED, /* nsegments */ 427 BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 428 0, /* flags */ 429 NULL, NULL, /* lock, lockarg */ 430 &sc->parent_dmat)) { 431 device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); 432 return (ENOMEM); 433 } 434 435 /* 436 * DMA tag for normal sized RX frames 437 */ 438 if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR, 439 BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, 440 MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) { 441 device_printf(sc->dev, "Cannot allocate RX DMA tag\n"); 442 return (ENOMEM); 443 } 444 445 /* 446 * DMA tag for jumbo sized RX frames. 447 */ 448 if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR, 449 BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES, 450 BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) { 451 device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n"); 452 return (ENOMEM); 453 } 454 455 /* 456 * DMA tag for TX frames. 457 */ 458 if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR, 459 BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, 460 TX_MAX_SIZE, BUS_DMA_ALLOCNOW, 461 NULL, NULL, &sc->tx_dmat)) { 462 device_printf(sc->dev, "Cannot allocate TX DMA tag\n"); 463 return (ENOMEM); 464 } 465 466 return (0); 467} 468 469int 470t3_sge_free(struct adapter * sc) 471{ 472 473 if (sc->tx_dmat != NULL) 474 bus_dma_tag_destroy(sc->tx_dmat); 475 476 if (sc->rx_jumbo_dmat != NULL) 477 bus_dma_tag_destroy(sc->rx_jumbo_dmat); 478 479 if (sc->rx_dmat != NULL) 480 bus_dma_tag_destroy(sc->rx_dmat); 481 482 if (sc->parent_dmat != NULL) 483 bus_dma_tag_destroy(sc->parent_dmat); 484 485 return (0); 486} 487 488void 489t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) 490{ 491 492 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U); 493 qs->rspq.polling = 0 /* p->polling */; 494} 495 496#if !defined(__i386__) && !defined(__amd64__) 497static void 498refill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 499{ 500 struct refill_fl_cb_arg *cb_arg = arg; 501 502 cb_arg->error = error; 503 cb_arg->seg = segs[0]; 504 cb_arg->nseg = nseg; 505 506} 507#endif 508/** 509 * refill_fl - refill an SGE free-buffer list 510 * @sc: the controller softc 511 * @q: the free-list to refill 512 * @n: the number of new buffers to allocate 513 * 514 * (Re)populate an SGE free-buffer list with up to @n new packet buffers. 515 * The caller must assure that @n does not exceed the queue's capacity. 516 */ 517static void 518refill_fl(adapter_t *sc, struct sge_fl *q, int n) 519{ 520 struct rx_sw_desc *sd = &q->sdesc[q->pidx]; 521 struct rx_desc *d = &q->desc[q->pidx]; 522 struct refill_fl_cb_arg cb_arg; 523 caddr_t cl; 524 int err, count = 0; 525 int header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t); 526 527 cb_arg.error = 0; 528 while (n--) { 529 /* 530 * We only allocate a cluster, mbuf allocation happens after rx 531 */ 532 if ((cl = cxgb_cache_get(q->zone)) == NULL) { 533 log(LOG_WARNING, "Failed to allocate cluster\n"); 534 goto done; 535 } 536 537 if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { 538 if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { 539 log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); 540 uma_zfree(q->zone, cl); 541 goto done; 542 } 543 sd->flags |= RX_SW_DESC_MAP_CREATED; 544 } 545#if !defined(__i386__) && !defined(__amd64__) 546 err = bus_dmamap_load(q->entry_tag, sd->map, 547 cl + header_size, q->buf_size, 548 refill_fl_cb, &cb_arg, 0); 549 550 if (err != 0 || cb_arg.error) { 551 log(LOG_WARNING, "failure in refill_fl %d\n", cb_arg.error); 552 /* 553 * XXX free cluster 554 */ 555 return; 556 } 557#else 558 cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)(cl + header_size)); 559#endif 560 sd->flags |= RX_SW_DESC_INUSE; 561 sd->rxsd_cl = cl; 562 sd->data = cl + header_size; 563 d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); 564 d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); 565 d->len_gen = htobe32(V_FLD_GEN1(q->gen)); 566 d->gen2 = htobe32(V_FLD_GEN2(q->gen)); 567 568 d++; 569 sd++; 570 571 if (++q->pidx == q->size) { 572 q->pidx = 0; 573 q->gen ^= 1; 574 sd = q->sdesc; 575 d = q->desc; 576 } 577 q->credits++; 578 count++; 579 } 580 581done: 582 if (count) 583 t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 584} 585 586 587/** 588 * free_rx_bufs - free the Rx buffers on an SGE free list 589 * @sc: the controle softc 590 * @q: the SGE free list to clean up 591 * 592 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from 593 * this queue should be stopped before calling this function. 594 */ 595static void 596free_rx_bufs(adapter_t *sc, struct sge_fl *q) 597{ 598 u_int cidx = q->cidx; 599 600 while (q->credits--) { 601 struct rx_sw_desc *d = &q->sdesc[cidx]; 602 603 if (d->flags & RX_SW_DESC_INUSE) { 604 bus_dmamap_unload(q->entry_tag, d->map); 605 bus_dmamap_destroy(q->entry_tag, d->map); 606 uma_zfree(q->zone, d->rxsd_cl); 607 } 608 d->rxsd_cl = NULL; 609 if (++cidx == q->size) 610 cidx = 0; 611 } 612} 613 614static __inline void 615__refill_fl(adapter_t *adap, struct sge_fl *fl) 616{ 617 refill_fl(adap, fl, min(16U, fl->size - fl->credits)); 618} 619 620static __inline void 621__refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max) 622{ 623 if ((fl->size - fl->credits) < max) 624 refill_fl(adap, fl, min(max, fl->size - fl->credits)); 625} 626 627void 628refill_fl_service(adapter_t *adap, struct sge_fl *fl) 629{ 630 __refill_fl_lt(adap, fl, 512); 631} 632 633/** 634 * recycle_rx_buf - recycle a receive buffer 635 * @adapter: the adapter 636 * @q: the SGE free list 637 * @idx: index of buffer to recycle 638 * 639 * Recycles the specified buffer on the given free list by adding it at 640 * the next available slot on the list. 641 */ 642static void 643recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx) 644{ 645 struct rx_desc *from = &q->desc[idx]; 646 struct rx_desc *to = &q->desc[q->pidx]; 647 648 q->sdesc[q->pidx] = q->sdesc[idx]; 649 to->addr_lo = from->addr_lo; // already big endian 650 to->addr_hi = from->addr_hi; // likewise 651 wmb(); 652 to->len_gen = htobe32(V_FLD_GEN1(q->gen)); 653 to->gen2 = htobe32(V_FLD_GEN2(q->gen)); 654 q->credits++; 655 656 if (++q->pidx == q->size) { 657 q->pidx = 0; 658 q->gen ^= 1; 659 } 660 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 661} 662 663static void 664alloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 665{ 666 uint32_t *addr; 667 668 addr = arg; 669 *addr = segs[0].ds_addr; 670} 671 672static int 673alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size, 674 bus_addr_t *phys, void *desc, void *sdesc, bus_dma_tag_t *tag, 675 bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag) 676{ 677 size_t len = nelem * elem_size; 678 void *s = NULL; 679 void *p = NULL; 680 int err; 681 682 if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0, 683 BUS_SPACE_MAXADDR_32BIT, 684 BUS_SPACE_MAXADDR, NULL, NULL, len, 1, 685 len, 0, NULL, NULL, tag)) != 0) { 686 device_printf(sc->dev, "Cannot allocate descriptor tag\n"); 687 return (ENOMEM); 688 } 689 690 if ((err = bus_dmamem_alloc(*tag, (void **)&p, BUS_DMA_NOWAIT, 691 map)) != 0) { 692 device_printf(sc->dev, "Cannot allocate descriptor memory\n"); 693 return (ENOMEM); 694 } 695 696 bus_dmamap_load(*tag, *map, p, len, alloc_ring_cb, phys, 0); 697 bzero(p, len); 698 *(void **)desc = p; 699 700 if (sw_size) { 701 len = nelem * sw_size; 702 s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO); 703 *(void **)sdesc = s; 704 } 705 if (parent_entry_tag == NULL) 706 return (0); 707 708 if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0, 709 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 710 NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, 711 TX_MAX_SIZE, BUS_DMA_ALLOCNOW, 712 NULL, NULL, entry_tag)) != 0) { 713 device_printf(sc->dev, "Cannot allocate descriptor entry tag\n"); 714 return (ENOMEM); 715 } 716 return (0); 717} 718 719static void 720sge_slow_intr_handler(void *arg, int ncount) 721{ 722 adapter_t *sc = arg; 723 724 t3_slow_intr_handler(sc); 725} 726 727/** 728 * sge_timer_cb - perform periodic maintenance of an SGE qset 729 * @data: the SGE queue set to maintain 730 * 731 * Runs periodically from a timer to perform maintenance of an SGE queue 732 * set. It performs two tasks: 733 * 734 * a) Cleans up any completed Tx descriptors that may still be pending. 735 * Normal descriptor cleanup happens when new packets are added to a Tx 736 * queue so this timer is relatively infrequent and does any cleanup only 737 * if the Tx queue has not seen any new packets in a while. We make a 738 * best effort attempt to reclaim descriptors, in that we don't wait 739 * around if we cannot get a queue's lock (which most likely is because 740 * someone else is queueing new packets and so will also handle the clean 741 * up). Since control queues use immediate data exclusively we don't 742 * bother cleaning them up here. 743 * 744 * b) Replenishes Rx queues that have run out due to memory shortage. 745 * Normally new Rx buffers are added when existing ones are consumed but 746 * when out of memory a queue can become empty. We try to add only a few 747 * buffers here, the queue will be replenished fully as these new buffers 748 * are used up if memory shortage has subsided. 749 * 750 * c) Return coalesced response queue credits in case a response queue is 751 * starved. 752 * 753 * d) Ring doorbells for T304 tunnel queues since we have seen doorbell 754 * fifo overflows and the FW doesn't implement any recovery scheme yet. 755 */ 756static void 757sge_timer_cb(void *arg) 758{ 759 adapter_t *sc = arg; 760#ifndef IFNET_MULTIQUEUE 761 struct port_info *pi; 762 struct sge_qset *qs; 763 struct sge_txq *txq; 764 int i, j; 765 int reclaim_ofl, refill_rx; 766 767 for (i = 0; i < sc->params.nports; i++) 768 for (j = 0; j < sc->port[i].nqsets; j++) { 769 qs = &sc->sge.qs[i + j]; 770 txq = &qs->txq[0]; 771 reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; 772 refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || 773 (qs->fl[1].credits < qs->fl[1].size)); 774 if (reclaim_ofl || refill_rx) { 775 pi = &sc->port[i]; 776 taskqueue_enqueue(pi->tq, &pi->timer_reclaim_task); 777 break; 778 } 779 } 780#endif 781 if (sc->params.nports > 2) { 782 int i; 783 784 for_each_port(sc, i) { 785 struct port_info *pi = &sc->port[i]; 786 787 t3_write_reg(sc, A_SG_KDOORBELL, 788 F_SELEGRCNTX | 789 (FW_TUNNEL_SGEEC_START + pi->first_qset)); 790 } 791 } 792 if (sc->open_device_map != 0) 793 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 794} 795 796/* 797 * This is meant to be a catch-all function to keep sge state private 798 * to sge.c 799 * 800 */ 801int 802t3_sge_init_adapter(adapter_t *sc) 803{ 804 callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE); 805 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 806 TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); 807 mi_init(); 808 cxgb_cache_init(); 809 return (0); 810} 811 812int 813t3_sge_reset_adapter(adapter_t *sc) 814{ 815 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 816 return (0); 817} 818 819int 820t3_sge_init_port(struct port_info *pi) 821{ 822 TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi); 823 return (0); 824} 825 826void 827t3_sge_deinit_sw(adapter_t *sc) 828{ 829 830 mi_deinit(); 831} 832 833/** 834 * refill_rspq - replenish an SGE response queue 835 * @adapter: the adapter 836 * @q: the response queue to replenish 837 * @credits: how many new responses to make available 838 * 839 * Replenishes a response queue by making the supplied number of responses 840 * available to HW. 841 */ 842static __inline void 843refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits) 844{ 845 846 /* mbufs are allocated on demand when a rspq entry is processed. */ 847 t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN, 848 V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); 849} 850 851static __inline void 852sge_txq_reclaim_(struct sge_txq *txq, int force) 853{ 854 855 if (desc_reclaimable(txq) < 16) 856 return; 857 if (mtx_trylock(&txq->lock) == 0) 858 return; 859 reclaim_completed_tx_(txq, 16); 860 mtx_unlock(&txq->lock); 861 862} 863 864static void 865sge_txq_reclaim_handler(void *arg, int ncount) 866{ 867 struct sge_txq *q = arg; 868 869 sge_txq_reclaim_(q, TRUE); 870} 871 872 873 874static void 875sge_timer_reclaim(void *arg, int ncount) 876{ 877 struct port_info *pi = arg; 878 int i, nqsets = pi->nqsets; 879 adapter_t *sc = pi->adapter; 880 struct sge_qset *qs; 881 struct sge_txq *txq; 882 struct mtx *lock; 883 884#ifdef IFNET_MULTIQUEUE 885 panic("%s should not be called with multiqueue support\n", __FUNCTION__); 886#endif 887 for (i = 0; i < nqsets; i++) { 888 qs = &sc->sge.qs[i]; 889 890 txq = &qs->txq[TXQ_OFLD]; 891 sge_txq_reclaim_(txq, FALSE); 892 893 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : 894 &sc->sge.qs[0].rspq.lock; 895 896 if (mtx_trylock(lock)) { 897 /* XXX currently assume that we are *NOT* polling */ 898 uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS); 899 900 if (qs->fl[0].credits < qs->fl[0].size - 16) 901 __refill_fl(sc, &qs->fl[0]); 902 if (qs->fl[1].credits < qs->fl[1].size - 16) 903 __refill_fl(sc, &qs->fl[1]); 904 905 if (status & (1 << qs->rspq.cntxt_id)) { 906 if (qs->rspq.credits) { 907 refill_rspq(sc, &qs->rspq, 1); 908 qs->rspq.credits--; 909 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, 910 1 << qs->rspq.cntxt_id); 911 } 912 } 913 mtx_unlock(lock); 914 } 915 } 916} 917 918/** 919 * init_qset_cntxt - initialize an SGE queue set context info 920 * @qs: the queue set 921 * @id: the queue set id 922 * 923 * Initializes the TIDs and context ids for the queues of a queue set. 924 */ 925static void 926init_qset_cntxt(struct sge_qset *qs, u_int id) 927{ 928 929 qs->rspq.cntxt_id = id; 930 qs->fl[0].cntxt_id = 2 * id; 931 qs->fl[1].cntxt_id = 2 * id + 1; 932 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; 933 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; 934 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; 935 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; 936 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; 937 938 mbufq_init(&qs->txq[TXQ_ETH].sendq); 939 mbufq_init(&qs->txq[TXQ_OFLD].sendq); 940 mbufq_init(&qs->txq[TXQ_CTRL].sendq); 941} 942 943 944static void 945txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs) 946{ 947 txq->in_use += ndesc; 948 /* 949 * XXX we don't handle stopping of queue 950 * presumably start handles this when we bump against the end 951 */ 952 txqs->gen = txq->gen; 953 txq->unacked += ndesc; 954 txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5); 955 txq->unacked &= 31; 956 txqs->pidx = txq->pidx; 957 txq->pidx += ndesc; 958#ifdef INVARIANTS 959 if (((txqs->pidx > txq->cidx) && 960 (txq->pidx < txqs->pidx) && 961 (txq->pidx >= txq->cidx)) || 962 ((txqs->pidx < txq->cidx) && 963 (txq->pidx >= txq-> cidx)) || 964 ((txqs->pidx < txq->cidx) && 965 (txq->cidx < txqs->pidx))) 966 panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d", 967 txqs->pidx, txq->pidx, txq->cidx); 968#endif 969 if (txq->pidx >= txq->size) { 970 txq->pidx -= txq->size; 971 txq->gen ^= 1; 972 } 973 974} 975 976/** 977 * calc_tx_descs - calculate the number of Tx descriptors for a packet 978 * @m: the packet mbufs 979 * @nsegs: the number of segments 980 * 981 * Returns the number of Tx descriptors needed for the given Ethernet 982 * packet. Ethernet packets require addition of WR and CPL headers. 983 */ 984static __inline unsigned int 985calc_tx_descs(const struct mbuf *m, int nsegs) 986{ 987 unsigned int flits; 988 989 if (m->m_pkthdr.len <= WR_LEN - sizeof(struct cpl_tx_pkt)) 990 return 1; 991 992 flits = sgl_len(nsegs) + 2; 993#ifdef TSO_SUPPORTED 994 if (m->m_pkthdr.csum_flags & CSUM_TSO) 995 flits++; 996#endif 997 return flits_to_desc(flits); 998} 999 1000static unsigned int 1001busdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, 1002 struct tx_sw_desc *txsd, bus_dma_segment_t *segs, int *nsegs) 1003{ 1004 struct mbuf *m0; 1005 int err, pktlen, pass = 0; 1006 1007retry: 1008 err = 0; 1009 m0 = *m; 1010 pktlen = m0->m_pkthdr.len; 1011#if defined(__i386__) || defined(__amd64__) 1012 if (busdma_map_sg_collapse(m, segs, nsegs) == 0) { 1013 goto done; 1014 } else 1015#endif 1016 err = bus_dmamap_load_mbuf_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0); 1017 1018 if (err == 0) { 1019 goto done; 1020 } 1021 if (err == EFBIG && pass == 0) { 1022 pass = 1; 1023 /* Too many segments, try to defrag */ 1024 m0 = m_defrag(m0, M_DONTWAIT); 1025 if (m0 == NULL) { 1026 m_freem(*m); 1027 *m = NULL; 1028 return (ENOBUFS); 1029 } 1030 *m = m0; 1031 goto retry; 1032 } else if (err == ENOMEM) { 1033 return (err); 1034 } if (err) { 1035 if (cxgb_debug) 1036 printf("map failure err=%d pktlen=%d\n", err, pktlen); 1037 m_freem(m0); 1038 *m = NULL; 1039 return (err); 1040 } 1041done: 1042#if !defined(__i386__) && !defined(__amd64__) 1043 bus_dmamap_sync(txq->entry_tag, txsd->map, BUS_DMASYNC_PREWRITE); 1044#endif 1045 txsd->flags |= TX_SW_DESC_MAPPED; 1046 1047 return (0); 1048} 1049 1050/** 1051 * make_sgl - populate a scatter/gather list for a packet 1052 * @sgp: the SGL to populate 1053 * @segs: the packet dma segments 1054 * @nsegs: the number of segments 1055 * 1056 * Generates a scatter/gather list for the buffers that make up a packet 1057 * and returns the SGL size in 8-byte words. The caller must size the SGL 1058 * appropriately. 1059 */ 1060static __inline void 1061make_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs) 1062{ 1063 int i, idx; 1064 1065 for (idx = 0, i = 0; i < nsegs; i++) { 1066 /* 1067 * firmware doesn't like empty segments 1068 */ 1069 if (segs[i].ds_len == 0) 1070 continue; 1071 if (i && idx == 0) 1072 ++sgp; 1073 1074 sgp->len[idx] = htobe32(segs[i].ds_len); 1075 sgp->addr[idx] = htobe64(segs[i].ds_addr); 1076 idx ^= 1; 1077 } 1078 1079 if (idx) { 1080 sgp->len[idx] = 0; 1081 sgp->addr[idx] = 0; 1082 } 1083} 1084 1085/** 1086 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell 1087 * @adap: the adapter 1088 * @q: the Tx queue 1089 * 1090 * Ring the doorbel if a Tx queue is asleep. There is a natural race, 1091 * where the HW is going to sleep just after we checked, however, 1092 * then the interrupt handler will detect the outstanding TX packet 1093 * and ring the doorbell for us. 1094 * 1095 * When GTS is disabled we unconditionally ring the doorbell. 1096 */ 1097static __inline void 1098check_ring_tx_db(adapter_t *adap, struct sge_txq *q) 1099{ 1100#if USE_GTS 1101 clear_bit(TXQ_LAST_PKT_DB, &q->flags); 1102 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { 1103 set_bit(TXQ_LAST_PKT_DB, &q->flags); 1104#ifdef T3_TRACE 1105 T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", 1106 q->cntxt_id); 1107#endif 1108 t3_write_reg(adap, A_SG_KDOORBELL, 1109 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1110 } 1111#else 1112 wmb(); /* write descriptors before telling HW */ 1113 t3_write_reg(adap, A_SG_KDOORBELL, 1114 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1115#endif 1116} 1117 1118static __inline void 1119wr_gen2(struct tx_desc *d, unsigned int gen) 1120{ 1121#if SGE_NUM_GENBITS == 2 1122 d->flit[TX_DESC_FLITS - 1] = htobe64(gen); 1123#endif 1124} 1125 1126/** 1127 * write_wr_hdr_sgl - write a WR header and, optionally, SGL 1128 * @ndesc: number of Tx descriptors spanned by the SGL 1129 * @txd: first Tx descriptor to be written 1130 * @txqs: txq state (generation and producer index) 1131 * @txq: the SGE Tx queue 1132 * @sgl: the SGL 1133 * @flits: number of flits to the start of the SGL in the first descriptor 1134 * @sgl_flits: the SGL size in flits 1135 * @wr_hi: top 32 bits of WR header based on WR type (big endian) 1136 * @wr_lo: low 32 bits of WR header based on WR type (big endian) 1137 * 1138 * Write a work request header and an associated SGL. If the SGL is 1139 * small enough to fit into one Tx descriptor it has already been written 1140 * and we just need to write the WR header. Otherwise we distribute the 1141 * SGL across the number of descriptors it spans. 1142 */ 1143static void 1144write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs, 1145 const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits, 1146 unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo) 1147{ 1148 1149 struct work_request_hdr *wrp = (struct work_request_hdr *)txd; 1150 struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx]; 1151 1152 if (__predict_true(ndesc == 1)) { 1153 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | 1154 V_WR_SGLSFLT(flits)) | wr_hi; 1155 wmb(); 1156 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) | 1157 V_WR_GEN(txqs->gen)) | wr_lo; 1158 /* XXX gen? */ 1159 wr_gen2(txd, txqs->gen); 1160 1161 } else { 1162 unsigned int ogen = txqs->gen; 1163 const uint64_t *fp = (const uint64_t *)sgl; 1164 struct work_request_hdr *wp = wrp; 1165 1166 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | 1167 V_WR_SGLSFLT(flits)) | wr_hi; 1168 1169 while (sgl_flits) { 1170 unsigned int avail = WR_FLITS - flits; 1171 1172 if (avail > sgl_flits) 1173 avail = sgl_flits; 1174 memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); 1175 sgl_flits -= avail; 1176 ndesc--; 1177 if (!sgl_flits) 1178 break; 1179 1180 fp += avail; 1181 txd++; 1182 txsd++; 1183 if (++txqs->pidx == txq->size) { 1184 txqs->pidx = 0; 1185 txqs->gen ^= 1; 1186 txd = txq->desc; 1187 txsd = txq->sdesc; 1188 } 1189 1190 /* 1191 * when the head of the mbuf chain 1192 * is freed all clusters will be freed 1193 * with it 1194 */ 1195 KASSERT(txsd->mi.mi_base == NULL, 1196 ("overwriting valid entry mi_base==%p", txsd->mi.mi_base)); 1197 wrp = (struct work_request_hdr *)txd; 1198 wrp->wr_hi = htonl(V_WR_DATATYPE(1) | 1199 V_WR_SGLSFLT(1)) | wr_hi; 1200 wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS, 1201 sgl_flits + 1)) | 1202 V_WR_GEN(txqs->gen)) | wr_lo; 1203 wr_gen2(txd, txqs->gen); 1204 flits = 1; 1205 } 1206 wrp->wr_hi |= htonl(F_WR_EOP); 1207 wmb(); 1208 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; 1209 wr_gen2((struct tx_desc *)wp, ogen); 1210 } 1211} 1212 1213/* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */ 1214#define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20) 1215 1216#ifdef VLAN_SUPPORTED 1217#define GET_VTAG(cntrl, m) \ 1218do { \ 1219 if ((m)->m_flags & M_VLANTAG) \ 1220 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \ 1221} while (0) 1222 1223#define GET_VTAG_MI(cntrl, mi) \ 1224do { \ 1225 if ((mi)->mi_flags & M_VLANTAG) \ 1226 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((mi)->mi_ether_vtag); \ 1227} while (0) 1228#else 1229#define GET_VTAG(cntrl, m) 1230#define GET_VTAG_MI(cntrl, m) 1231#endif 1232 1233int 1234t3_encap(struct sge_qset *qs, struct mbuf **m, int count) 1235{ 1236 adapter_t *sc; 1237 struct mbuf *m0; 1238 struct sge_txq *txq; 1239 struct txq_state txqs; 1240 struct port_info *pi; 1241 unsigned int ndesc, flits, cntrl, mlen; 1242 int err, nsegs, tso_info = 0; 1243 1244 struct work_request_hdr *wrp; 1245 struct tx_sw_desc *txsd; 1246 struct sg_ent *sgp, *sgl; 1247 uint32_t wr_hi, wr_lo, sgl_flits; 1248 bus_dma_segment_t segs[TX_MAX_SEGS]; 1249 1250 struct tx_desc *txd; 1251 struct mbuf_vec *mv; 1252 struct mbuf_iovec *mi; 1253 1254 DPRINTF("t3_encap cpu=%d ", curcpu); 1255 1256 mi = NULL; 1257 pi = qs->port; 1258 sc = pi->adapter; 1259 txq = &qs->txq[TXQ_ETH]; 1260 txd = &txq->desc[txq->pidx]; 1261 txsd = &txq->sdesc[txq->pidx]; 1262 sgl = txq->txq_sgl; 1263 m0 = *m; 1264 1265 DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset); 1266 DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan); 1267 if (cxgb_debug) 1268 printf("mi_base=%p cidx=%d pidx=%d\n\n", txsd->mi.mi_base, txq->cidx, txq->pidx); 1269 1270 mtx_assert(&txq->lock, MA_OWNED); 1271 cntrl = V_TXPKT_INTF(pi->txpkt_intf); 1272/* 1273 * XXX need to add VLAN support for 6.x 1274 */ 1275#ifdef VLAN_SUPPORTED 1276 if (m0->m_pkthdr.csum_flags & (CSUM_TSO)) 1277 tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); 1278#endif 1279 KASSERT(txsd->mi.mi_base == NULL, 1280 ("overwriting valid entry mi_base==%p", txsd->mi.mi_base)); 1281 if (count > 1) { 1282 panic("count > 1 not support in CVS\n"); 1283 if ((err = busdma_map_sg_vec(m, &m0, segs, count))) 1284 return (err); 1285 nsegs = count; 1286 } else if ((err = busdma_map_sg_collapse(&m0, segs, &nsegs))) { 1287 if (cxgb_debug) 1288 printf("failed ... err=%d\n", err); 1289 return (err); 1290 } 1291 KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d count=%d", nsegs, count)); 1292 1293 if (!(m0->m_pkthdr.len <= PIO_LEN)) { 1294 mi_collapse_mbuf(&txsd->mi, m0); 1295 mi = &txsd->mi; 1296 } 1297 if (count > 1) { 1298 struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd; 1299 int i, fidx; 1300 struct mbuf_iovec *batchmi; 1301 1302 mv = mtomv(m0); 1303 batchmi = mv->mv_vec; 1304 1305 wrp = (struct work_request_hdr *)txd; 1306 1307 flits = count*2 + 1; 1308 txq_prod(txq, 1, &txqs); 1309 1310 for (fidx = 1, i = 0; i < count; i++, batchmi++, fidx += 2) { 1311 struct cpl_tx_pkt_batch_entry *cbe = &cpl_batch->pkt_entry[i]; 1312 1313 cntrl = V_TXPKT_INTF(pi->txpkt_intf); 1314 GET_VTAG_MI(cntrl, batchmi); 1315 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); 1316 if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP))) 1317 cntrl |= F_TXPKT_IPCSUM_DIS; 1318 if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)))) 1319 cntrl |= F_TXPKT_L4CSUM_DIS; 1320 cbe->cntrl = htonl(cntrl); 1321 cbe->len = htonl(batchmi->mi_len | 0x80000000); 1322 cbe->addr = htobe64(segs[i].ds_addr); 1323 txd->flit[fidx] |= htobe64(1 << 24); 1324 } 1325 1326 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | 1327 V_WR_SGLSFLT(flits)) | htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); 1328 wmb(); 1329 wrp->wr_lo = htonl(V_WR_LEN(flits) | 1330 V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token)); 1331 /* XXX gen? */ 1332 wr_gen2(txd, txqs.gen); 1333 check_ring_tx_db(sc, txq); 1334 1335 return (0); 1336 } else if (tso_info) { 1337 int min_size = TCPPKTHDRSIZE, eth_type, tagged; 1338 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd; 1339 struct ip *ip; 1340 struct tcphdr *tcp; 1341 char *pkthdr; 1342 1343 txd->flit[2] = 0; 1344 GET_VTAG(cntrl, m0); 1345 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); 1346 hdr->cntrl = htonl(cntrl); 1347 mlen = m0->m_pkthdr.len; 1348 hdr->len = htonl(mlen | 0x80000000); 1349 1350 DPRINTF("tso buf len=%d\n", mlen); 1351 1352 tagged = m0->m_flags & M_VLANTAG; 1353 if (!tagged) 1354 min_size -= ETHER_VLAN_ENCAP_LEN; 1355 1356 if (__predict_false(mlen < min_size)) { 1357 printf("mbuf=%p,len=%d,tso_segsz=%d,csum_flags=%#x,flags=%#x", 1358 m0, mlen, m0->m_pkthdr.tso_segsz, 1359 m0->m_pkthdr.csum_flags, m0->m_flags); 1360 panic("tx tso packet too small"); 1361 } 1362 1363 /* Make sure that ether, ip, tcp headers are all in m0 */ 1364 if (__predict_false(m0->m_len < min_size)) { 1365 m0 = m_pullup(m0, min_size); 1366 if (__predict_false(m0 == NULL)) { 1367 /* XXX panic probably an overreaction */ 1368 panic("couldn't fit header into mbuf"); 1369 } 1370 } 1371 pkthdr = m0->m_data; 1372 1373 if (tagged) { 1374 eth_type = CPL_ETH_II_VLAN; 1375 ip = (struct ip *)(pkthdr + ETHER_HDR_LEN + 1376 ETHER_VLAN_ENCAP_LEN); 1377 } else { 1378 eth_type = CPL_ETH_II; 1379 ip = (struct ip *)(pkthdr + ETHER_HDR_LEN); 1380 } 1381 tcp = (struct tcphdr *)((uint8_t *)ip + 1382 sizeof(*ip)); 1383 1384 tso_info |= V_LSO_ETH_TYPE(eth_type) | 1385 V_LSO_IPHDR_WORDS(ip->ip_hl) | 1386 V_LSO_TCPHDR_WORDS(tcp->th_off); 1387 hdr->lso_info = htonl(tso_info); 1388 1389 if (__predict_false(mlen <= PIO_LEN)) { 1390 /* pkt not undersized but fits in PIO_LEN */ 1391 printf("**5592 Fix** mbuf=%p,len=%d,tso_segsz=%d,csum_flags=%#x,flags=%#x", 1392 m0, mlen, m0->m_pkthdr.tso_segsz, m0->m_pkthdr.csum_flags, m0->m_flags); 1393 txq_prod(txq, 1, &txqs); 1394 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[3]); 1395 m_freem(m0); 1396 m0 = NULL; 1397 flits = (mlen + 7) / 8 + 3; 1398 hdr->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | 1399 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | 1400 F_WR_SOP | F_WR_EOP | txqs.compl); 1401 wmb(); 1402 hdr->wr.wr_lo = htonl(V_WR_LEN(flits) | 1403 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); 1404 1405 wr_gen2(txd, txqs.gen); 1406 check_ring_tx_db(sc, txq); 1407 return (0); 1408 } 1409 flits = 3; 1410 } else { 1411 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd; 1412 1413 GET_VTAG(cntrl, m0); 1414 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); 1415 if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP))) 1416 cntrl |= F_TXPKT_IPCSUM_DIS; 1417 if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)))) 1418 cntrl |= F_TXPKT_L4CSUM_DIS; 1419 cpl->cntrl = htonl(cntrl); 1420 mlen = m0->m_pkthdr.len; 1421 cpl->len = htonl(mlen | 0x80000000); 1422 1423 if (mlen <= PIO_LEN) { 1424 txq_prod(txq, 1, &txqs); 1425 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); 1426 m_freem(m0); 1427 m0 = NULL; 1428 flits = (mlen + 7) / 8 + 2; 1429 cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | 1430 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | 1431 F_WR_SOP | F_WR_EOP | txqs.compl); 1432 wmb(); 1433 cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | 1434 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); 1435 1436 wr_gen2(txd, txqs.gen); 1437 check_ring_tx_db(sc, txq); 1438 DPRINTF("pio buf\n"); 1439 return (0); 1440 } 1441 DPRINTF("regular buf\n"); 1442 flits = 2; 1443 } 1444 wrp = (struct work_request_hdr *)txd; 1445 1446#ifdef nomore 1447 /* 1448 * XXX need to move into one of the helper routines above 1449 * 1450 */ 1451 if ((err = busdma_map_mbufs(m, txq, txsd, segs, &nsegs)) != 0) 1452 return (err); 1453 m0 = *m; 1454#endif 1455 ndesc = calc_tx_descs(m0, nsegs); 1456 1457 sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; 1458 make_sgl(sgp, segs, nsegs); 1459 1460 sgl_flits = sgl_len(nsegs); 1461 1462 DPRINTF("make_sgl success nsegs==%d ndesc==%d\n", nsegs, ndesc); 1463 txq_prod(txq, ndesc, &txqs); 1464 wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); 1465 wr_lo = htonl(V_WR_TID(txq->token)); 1466 write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); 1467 check_ring_tx_db(pi->adapter, txq); 1468 1469 if ((m0->m_type == MT_DATA) && 1470 ((m0->m_flags & (M_EXT|M_NOFREE)) == M_EXT) && 1471 (m0->m_ext.ext_type != EXT_PACKET)) { 1472 m0->m_flags &= ~M_EXT ; 1473 cxgb_mbufs_outstanding--; 1474 m_free(m0); 1475 } 1476 1477 return (0); 1478} 1479 1480 1481/** 1482 * write_imm - write a packet into a Tx descriptor as immediate data 1483 * @d: the Tx descriptor to write 1484 * @m: the packet 1485 * @len: the length of packet data to write as immediate data 1486 * @gen: the generation bit value to write 1487 * 1488 * Writes a packet as immediate data into a Tx descriptor. The packet 1489 * contains a work request at its beginning. We must write the packet 1490 * carefully so the SGE doesn't read accidentally before it's written in 1491 * its entirety. 1492 */ 1493static __inline void 1494write_imm(struct tx_desc *d, struct mbuf *m, 1495 unsigned int len, unsigned int gen) 1496{ 1497 struct work_request_hdr *from = mtod(m, struct work_request_hdr *); 1498 struct work_request_hdr *to = (struct work_request_hdr *)d; 1499 1500 if (len > WR_LEN) 1501 panic("len too big %d\n", len); 1502 if (len < sizeof(*from)) 1503 panic("len too small %d", len); 1504 1505 memcpy(&to[1], &from[1], len - sizeof(*from)); 1506 to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP | 1507 V_WR_BCNTLFLT(len & 7)); 1508 wmb(); 1509 to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) | 1510 V_WR_LEN((len + 7) / 8)); 1511 wr_gen2(d, gen); 1512 1513 /* 1514 * This check is a hack we should really fix the logic so 1515 * that this can't happen 1516 */ 1517 if (m->m_type != MT_DONTFREE) 1518 m_freem(m); 1519 1520} 1521 1522/** 1523 * check_desc_avail - check descriptor availability on a send queue 1524 * @adap: the adapter 1525 * @q: the TX queue 1526 * @m: the packet needing the descriptors 1527 * @ndesc: the number of Tx descriptors needed 1528 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL) 1529 * 1530 * Checks if the requested number of Tx descriptors is available on an 1531 * SGE send queue. If the queue is already suspended or not enough 1532 * descriptors are available the packet is queued for later transmission. 1533 * Must be called with the Tx queue locked. 1534 * 1535 * Returns 0 if enough descriptors are available, 1 if there aren't 1536 * enough descriptors and the packet has been queued, and 2 if the caller 1537 * needs to retry because there weren't enough descriptors at the 1538 * beginning of the call but some freed up in the mean time. 1539 */ 1540static __inline int 1541check_desc_avail(adapter_t *adap, struct sge_txq *q, 1542 struct mbuf *m, unsigned int ndesc, 1543 unsigned int qid) 1544{ 1545 /* 1546 * XXX We currently only use this for checking the control queue 1547 * the control queue is only used for binding qsets which happens 1548 * at init time so we are guaranteed enough descriptors 1549 */ 1550 if (__predict_false(!mbufq_empty(&q->sendq))) { 1551addq_exit: mbufq_tail(&q->sendq, m); 1552 return 1; 1553 } 1554 if (__predict_false(q->size - q->in_use < ndesc)) { 1555 1556 struct sge_qset *qs = txq_to_qset(q, qid); 1557 1558 printf("stopping q\n"); 1559 1560 setbit(&qs->txq_stopped, qid); 1561 smp_mb(); 1562 1563 if (should_restart_tx(q) && 1564 test_and_clear_bit(qid, &qs->txq_stopped)) 1565 return 2; 1566 1567 q->stops++; 1568 goto addq_exit; 1569 } 1570 return 0; 1571} 1572 1573 1574/** 1575 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs 1576 * @q: the SGE control Tx queue 1577 * 1578 * This is a variant of reclaim_completed_tx() that is used for Tx queues 1579 * that send only immediate data (presently just the control queues) and 1580 * thus do not have any mbufs 1581 */ 1582static __inline void 1583reclaim_completed_tx_imm(struct sge_txq *q) 1584{ 1585 unsigned int reclaim = q->processed - q->cleaned; 1586 1587 mtx_assert(&q->lock, MA_OWNED); 1588 1589 q->in_use -= reclaim; 1590 q->cleaned += reclaim; 1591} 1592 1593static __inline int 1594immediate(const struct mbuf *m) 1595{ 1596 return m->m_len <= WR_LEN && m->m_pkthdr.len <= WR_LEN ; 1597} 1598 1599/** 1600 * ctrl_xmit - send a packet through an SGE control Tx queue 1601 * @adap: the adapter 1602 * @q: the control queue 1603 * @m: the packet 1604 * 1605 * Send a packet through an SGE control Tx queue. Packets sent through 1606 * a control queue must fit entirely as immediate data in a single Tx 1607 * descriptor and have no page fragments. 1608 */ 1609static int 1610ctrl_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) 1611{ 1612 int ret; 1613 struct work_request_hdr *wrp = mtod(m, struct work_request_hdr *); 1614 1615 if (__predict_false(!immediate(m))) { 1616 m_freem(m); 1617 return 0; 1618 } 1619 1620 wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP); 1621 wrp->wr_lo = htonl(V_WR_TID(q->token)); 1622 1623 mtx_lock(&q->lock); 1624again: reclaim_completed_tx_imm(q); 1625 1626 ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL); 1627 if (__predict_false(ret)) { 1628 if (ret == 1) { 1629 mtx_unlock(&q->lock); 1630 log(LOG_ERR, "no desc available\n"); 1631 return (ENOSPC); 1632 } 1633 goto again; 1634 } 1635 write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1636 1637 q->in_use++; 1638 if (++q->pidx >= q->size) { 1639 q->pidx = 0; 1640 q->gen ^= 1; 1641 } 1642 mtx_unlock(&q->lock); 1643 wmb(); 1644 t3_write_reg(adap, A_SG_KDOORBELL, 1645 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1646 return (0); 1647} 1648 1649 1650/** 1651 * restart_ctrlq - restart a suspended control queue 1652 * @qs: the queue set cotaining the control queue 1653 * 1654 * Resumes transmission on a suspended Tx control queue. 1655 */ 1656static void 1657restart_ctrlq(void *data, int npending) 1658{ 1659 struct mbuf *m; 1660 struct sge_qset *qs = (struct sge_qset *)data; 1661 struct sge_txq *q = &qs->txq[TXQ_CTRL]; 1662 adapter_t *adap = qs->port->adapter; 1663 1664 log(LOG_WARNING, "Restart_ctrlq in_use=%d\n", q->in_use); 1665 1666 mtx_lock(&q->lock); 1667again: reclaim_completed_tx_imm(q); 1668 1669 while (q->in_use < q->size && 1670 (m = mbufq_dequeue(&q->sendq)) != NULL) { 1671 1672 write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1673 1674 if (++q->pidx >= q->size) { 1675 q->pidx = 0; 1676 q->gen ^= 1; 1677 } 1678 q->in_use++; 1679 } 1680 if (!mbufq_empty(&q->sendq)) { 1681 setbit(&qs->txq_stopped, TXQ_CTRL); 1682 smp_mb(); 1683 1684 if (should_restart_tx(q) && 1685 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) 1686 goto again; 1687 q->stops++; 1688 } 1689 mtx_unlock(&q->lock); 1690 wmb(); 1691 t3_write_reg(adap, A_SG_KDOORBELL, 1692 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1693} 1694 1695 1696/* 1697 * Send a management message through control queue 0 1698 */ 1699int 1700t3_mgmt_tx(struct adapter *adap, struct mbuf *m) 1701{ 1702 return ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], m); 1703} 1704 1705 1706/** 1707 * free_qset - free the resources of an SGE queue set 1708 * @sc: the controller owning the queue set 1709 * @q: the queue set 1710 * 1711 * Release the HW and SW resources associated with an SGE queue set, such 1712 * as HW contexts, packet buffers, and descriptor rings. Traffic to the 1713 * queue set must be quiesced prior to calling this. 1714 */ 1715void 1716t3_free_qset(adapter_t *sc, struct sge_qset *q) 1717{ 1718 int i; 1719 1720 t3_free_tx_desc_all(&q->txq[TXQ_ETH]); 1721 1722 for (i = 0; i < SGE_TXQ_PER_SET; i++) 1723 if (q->txq[i].txq_mr.br_ring != NULL) { 1724 free(q->txq[i].txq_mr.br_ring, M_DEVBUF); 1725 mtx_destroy(&q->txq[i].txq_mr.br_lock); 1726 } 1727 for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 1728 if (q->fl[i].desc) { 1729 mtx_lock_spin(&sc->sge.reg_lock); 1730 t3_sge_disable_fl(sc, q->fl[i].cntxt_id); 1731 mtx_unlock_spin(&sc->sge.reg_lock); 1732 bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); 1733 bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc, 1734 q->fl[i].desc_map); 1735 bus_dma_tag_destroy(q->fl[i].desc_tag); 1736 bus_dma_tag_destroy(q->fl[i].entry_tag); 1737 } 1738 if (q->fl[i].sdesc) { 1739 free_rx_bufs(sc, &q->fl[i]); 1740 free(q->fl[i].sdesc, M_DEVBUF); 1741 } 1742 } 1743 1744 for (i = 0; i < SGE_TXQ_PER_SET; i++) { 1745 if (q->txq[i].desc) { 1746 mtx_lock_spin(&sc->sge.reg_lock); 1747 t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); 1748 mtx_unlock_spin(&sc->sge.reg_lock); 1749 bus_dmamap_unload(q->txq[i].desc_tag, 1750 q->txq[i].desc_map); 1751 bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc, 1752 q->txq[i].desc_map); 1753 bus_dma_tag_destroy(q->txq[i].desc_tag); 1754 bus_dma_tag_destroy(q->txq[i].entry_tag); 1755 MTX_DESTROY(&q->txq[i].lock); 1756 } 1757 if (q->txq[i].sdesc) { 1758 free(q->txq[i].sdesc, M_DEVBUF); 1759 } 1760 } 1761 1762 if (q->rspq.desc) { 1763 mtx_lock_spin(&sc->sge.reg_lock); 1764 t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); 1765 mtx_unlock_spin(&sc->sge.reg_lock); 1766 1767 bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); 1768 bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc, 1769 q->rspq.desc_map); 1770 bus_dma_tag_destroy(q->rspq.desc_tag); 1771 MTX_DESTROY(&q->rspq.lock); 1772 } 1773 1774 tcp_lro_free(&q->lro.ctrl); 1775 1776 bzero(q, sizeof(*q)); 1777} 1778 1779/** 1780 * t3_free_sge_resources - free SGE resources 1781 * @sc: the adapter softc 1782 * 1783 * Frees resources used by the SGE queue sets. 1784 */ 1785void 1786t3_free_sge_resources(adapter_t *sc) 1787{ 1788 int i, nqsets; 1789 1790#ifdef IFNET_MULTIQUEUE 1791 panic("%s should not be called when IFNET_MULTIQUEUE is defined", __FUNCTION__); 1792#endif 1793 for (nqsets = i = 0; i < (sc)->params.nports; i++) 1794 nqsets += sc->port[i].nqsets; 1795 1796 for (i = 0; i < nqsets; ++i) 1797 t3_free_qset(sc, &sc->sge.qs[i]); 1798} 1799 1800/** 1801 * t3_sge_start - enable SGE 1802 * @sc: the controller softc 1803 * 1804 * Enables the SGE for DMAs. This is the last step in starting packet 1805 * transfers. 1806 */ 1807void 1808t3_sge_start(adapter_t *sc) 1809{ 1810 t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE); 1811} 1812 1813/** 1814 * t3_sge_stop - disable SGE operation 1815 * @sc: the adapter 1816 * 1817 * Disables the DMA engine. This can be called in emeregencies (e.g., 1818 * from error interrupts) or from normal process context. In the latter 1819 * case it also disables any pending queue restart tasklets. Note that 1820 * if it is called in interrupt context it cannot disable the restart 1821 * tasklets as it cannot wait, however the tasklets will have no effect 1822 * since the doorbells are disabled and the driver will call this again 1823 * later from process context, at which time the tasklets will be stopped 1824 * if they are still running. 1825 */ 1826void 1827t3_sge_stop(adapter_t *sc) 1828{ 1829 int i, nqsets; 1830 1831 t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0); 1832 1833 if (sc->tq == NULL) 1834 return; 1835 1836 for (nqsets = i = 0; i < (sc)->params.nports; i++) 1837 nqsets += sc->port[i].nqsets; 1838#ifdef notyet 1839 /* 1840 * 1841 * XXX 1842 */ 1843 for (i = 0; i < nqsets; ++i) { 1844 struct sge_qset *qs = &sc->sge.qs[i]; 1845 1846 taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); 1847 taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); 1848 } 1849#endif 1850} 1851 1852/** 1853 * t3_free_tx_desc - reclaims Tx descriptors and their buffers 1854 * @adapter: the adapter 1855 * @q: the Tx queue to reclaim descriptors from 1856 * @reclaimable: the number of descriptors to reclaim 1857 * @m_vec_size: maximum number of buffers to reclaim 1858 * @desc_reclaimed: returns the number of descriptors reclaimed 1859 * 1860 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated 1861 * Tx buffers. Called with the Tx queue lock held. 1862 * 1863 * Returns number of buffers of reclaimed 1864 */ 1865void 1866t3_free_tx_desc(struct sge_txq *q, int reclaimable) 1867{ 1868 struct tx_sw_desc *txsd; 1869 unsigned int cidx; 1870 1871#ifdef T3_TRACE 1872 T3_TRACE2(sc->tb[q->cntxt_id & 7], 1873 "reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx); 1874#endif 1875 cidx = q->cidx; 1876 txsd = &q->sdesc[cidx]; 1877 DPRINTF("reclaiming %d WR\n", reclaimable); 1878 mtx_assert(&q->lock, MA_OWNED); 1879 while (reclaimable--) { 1880 DPRINTF("cidx=%d d=%p\n", cidx, txsd); 1881 if (txsd->mi.mi_base != NULL) { 1882 if (txsd->flags & TX_SW_DESC_MAPPED) { 1883 bus_dmamap_unload(q->entry_tag, txsd->map); 1884 txsd->flags &= ~TX_SW_DESC_MAPPED; 1885 } 1886 m_freem_iovec(&txsd->mi); 1887 buf_ring_scan(&q->txq_mr, txsd->mi.mi_base, __FILE__, __LINE__); 1888 txsd->mi.mi_base = NULL; 1889 1890#if defined(DIAGNOSTIC) && 0 1891 if (m_get_priority(txsd->m[0]) != cidx) 1892 printf("pri=%d cidx=%d\n", 1893 (int)m_get_priority(txsd->m[0]), cidx); 1894#endif 1895 1896 } else 1897 q->txq_skipped++; 1898 1899 ++txsd; 1900 if (++cidx == q->size) { 1901 cidx = 0; 1902 txsd = q->sdesc; 1903 } 1904 } 1905 q->cidx = cidx; 1906 1907} 1908 1909void 1910t3_free_tx_desc_all(struct sge_txq *q) 1911{ 1912 int i; 1913 struct tx_sw_desc *txsd; 1914 1915 for (i = 0; i < q->size; i++) { 1916 txsd = &q->sdesc[i]; 1917 if (txsd->mi.mi_base != NULL) { 1918 if (txsd->flags & TX_SW_DESC_MAPPED) { 1919 bus_dmamap_unload(q->entry_tag, txsd->map); 1920 txsd->flags &= ~TX_SW_DESC_MAPPED; 1921 } 1922 m_freem_iovec(&txsd->mi); 1923 bzero(&txsd->mi, sizeof(txsd->mi)); 1924 } 1925 } 1926} 1927 1928/** 1929 * is_new_response - check if a response is newly written 1930 * @r: the response descriptor 1931 * @q: the response queue 1932 * 1933 * Returns true if a response descriptor contains a yet unprocessed 1934 * response. 1935 */ 1936static __inline int 1937is_new_response(const struct rsp_desc *r, 1938 const struct sge_rspq *q) 1939{ 1940 return (r->intr_gen & F_RSPD_GEN2) == q->gen; 1941} 1942 1943#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) 1944#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ 1945 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ 1946 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \ 1947 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR)) 1948 1949/* How long to delay the next interrupt in case of memory shortage, in 0.1us. */ 1950#define NOMEM_INTR_DELAY 2500 1951 1952/** 1953 * write_ofld_wr - write an offload work request 1954 * @adap: the adapter 1955 * @m: the packet to send 1956 * @q: the Tx queue 1957 * @pidx: index of the first Tx descriptor to write 1958 * @gen: the generation value to use 1959 * @ndesc: number of descriptors the packet will occupy 1960 * 1961 * Write an offload work request to send the supplied packet. The packet 1962 * data already carry the work request with most fields populated. 1963 */ 1964static void 1965write_ofld_wr(adapter_t *adap, struct mbuf *m, 1966 struct sge_txq *q, unsigned int pidx, 1967 unsigned int gen, unsigned int ndesc, 1968 bus_dma_segment_t *segs, unsigned int nsegs) 1969{ 1970 unsigned int sgl_flits, flits; 1971 struct work_request_hdr *from; 1972 struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; 1973 struct tx_desc *d = &q->desc[pidx]; 1974 struct txq_state txqs; 1975 1976 if (immediate(m) && nsegs == 0) { 1977 write_imm(d, m, m->m_len, gen); 1978 return; 1979 } 1980 1981 /* Only TX_DATA builds SGLs */ 1982 from = mtod(m, struct work_request_hdr *); 1983 memcpy(&d->flit[1], &from[1], m->m_len - sizeof(*from)); 1984 1985 flits = m->m_len / 8; 1986 sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : sgl; 1987 1988 make_sgl(sgp, segs, nsegs); 1989 sgl_flits = sgl_len(nsegs); 1990 1991 txqs.gen = gen; 1992 txqs.pidx = pidx; 1993 txqs.compl = 0; 1994 1995 write_wr_hdr_sgl(ndesc, d, &txqs, q, sgl, flits, sgl_flits, 1996 from->wr_hi, from->wr_lo); 1997} 1998 1999/** 2000 * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet 2001 * @m: the packet 2002 * 2003 * Returns the number of Tx descriptors needed for the given offload 2004 * packet. These packets are already fully constructed. 2005 */ 2006static __inline unsigned int 2007calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs) 2008{ 2009 unsigned int flits, cnt = 0; 2010 int ndescs; 2011 2012 if (m->m_len <= WR_LEN && nsegs == 0) 2013 return (1); /* packet fits as immediate data */ 2014 2015 if (m->m_flags & M_IOVEC) 2016 cnt = mtomv(m)->mv_count; 2017 else 2018 cnt = nsegs; 2019 2020 /* headers */ 2021 flits = m->m_len / 8; 2022 2023 ndescs = flits_to_desc(flits + sgl_len(cnt)); 2024 2025 CTR4(KTR_CXGB, "flits=%d sgl_len=%d nsegs=%d ndescs=%d", 2026 flits, sgl_len(cnt), nsegs, ndescs); 2027 2028 return (ndescs); 2029} 2030 2031/** 2032 * ofld_xmit - send a packet through an offload queue 2033 * @adap: the adapter 2034 * @q: the Tx offload queue 2035 * @m: the packet 2036 * 2037 * Send an offload packet through an SGE offload queue. 2038 */ 2039static int 2040ofld_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) 2041{ 2042 int ret, nsegs; 2043 unsigned int ndesc; 2044 unsigned int pidx, gen; 2045 bus_dma_segment_t segs[TX_MAX_SEGS], *vsegs; 2046 struct tx_sw_desc *stx; 2047 2048 nsegs = m_get_sgllen(m); 2049 vsegs = m_get_sgl(m); 2050 ndesc = calc_tx_descs_ofld(m, nsegs); 2051 busdma_map_sgl(vsegs, segs, nsegs); 2052 2053 stx = &q->sdesc[q->pidx]; 2054 KASSERT(stx->mi.mi_base == NULL, ("mi_base set")); 2055 2056 mtx_lock(&q->lock); 2057again: reclaim_completed_tx_(q, 16); 2058 ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD); 2059 if (__predict_false(ret)) { 2060 if (ret == 1) { 2061 printf("no ofld desc avail\n"); 2062 2063 m_set_priority(m, ndesc); /* save for restart */ 2064 mtx_unlock(&q->lock); 2065 return (EINTR); 2066 } 2067 goto again; 2068 } 2069 2070 gen = q->gen; 2071 q->in_use += ndesc; 2072 pidx = q->pidx; 2073 q->pidx += ndesc; 2074 if (q->pidx >= q->size) { 2075 q->pidx -= q->size; 2076 q->gen ^= 1; 2077 } 2078#ifdef T3_TRACE 2079 T3_TRACE5(adap->tb[q->cntxt_id & 7], 2080 "ofld_xmit: ndesc %u, pidx %u, len %u, main %u, frags %u", 2081 ndesc, pidx, skb->len, skb->len - skb->data_len, 2082 skb_shinfo(skb)->nr_frags); 2083#endif 2084 mtx_unlock(&q->lock); 2085 2086 write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 2087 check_ring_tx_db(adap, q); 2088 return (0); 2089} 2090 2091/** 2092 * restart_offloadq - restart a suspended offload queue 2093 * @qs: the queue set cotaining the offload queue 2094 * 2095 * Resumes transmission on a suspended Tx offload queue. 2096 */ 2097static void 2098restart_offloadq(void *data, int npending) 2099{ 2100 struct mbuf *m; 2101 struct sge_qset *qs = data; 2102 struct sge_txq *q = &qs->txq[TXQ_OFLD]; 2103 adapter_t *adap = qs->port->adapter; 2104 bus_dma_segment_t segs[TX_MAX_SEGS]; 2105 struct tx_sw_desc *stx = &q->sdesc[q->pidx]; 2106 int nsegs, cleaned; 2107 2108 mtx_lock(&q->lock); 2109again: cleaned = reclaim_completed_tx_(q, 16); 2110 2111 while ((m = mbufq_peek(&q->sendq)) != NULL) { 2112 unsigned int gen, pidx; 2113 unsigned int ndesc = m_get_priority(m); 2114 2115 if (__predict_false(q->size - q->in_use < ndesc)) { 2116 setbit(&qs->txq_stopped, TXQ_OFLD); 2117 smp_mb(); 2118 2119 if (should_restart_tx(q) && 2120 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) 2121 goto again; 2122 q->stops++; 2123 break; 2124 } 2125 2126 gen = q->gen; 2127 q->in_use += ndesc; 2128 pidx = q->pidx; 2129 q->pidx += ndesc; 2130 if (q->pidx >= q->size) { 2131 q->pidx -= q->size; 2132 q->gen ^= 1; 2133 } 2134 2135 (void)mbufq_dequeue(&q->sendq); 2136 busdma_map_mbufs(&m, q, stx, segs, &nsegs); 2137 mtx_unlock(&q->lock); 2138 write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 2139 mtx_lock(&q->lock); 2140 } 2141 mtx_unlock(&q->lock); 2142 2143#if USE_GTS 2144 set_bit(TXQ_RUNNING, &q->flags); 2145 set_bit(TXQ_LAST_PKT_DB, &q->flags); 2146#endif 2147 wmb(); 2148 t3_write_reg(adap, A_SG_KDOORBELL, 2149 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 2150} 2151 2152/** 2153 * queue_set - return the queue set a packet should use 2154 * @m: the packet 2155 * 2156 * Maps a packet to the SGE queue set it should use. The desired queue 2157 * set is carried in bits 1-3 in the packet's priority. 2158 */ 2159static __inline int 2160queue_set(const struct mbuf *m) 2161{ 2162 return m_get_priority(m) >> 1; 2163} 2164 2165/** 2166 * is_ctrl_pkt - return whether an offload packet is a control packet 2167 * @m: the packet 2168 * 2169 * Determines whether an offload packet should use an OFLD or a CTRL 2170 * Tx queue. This is indicated by bit 0 in the packet's priority. 2171 */ 2172static __inline int 2173is_ctrl_pkt(const struct mbuf *m) 2174{ 2175 return m_get_priority(m) & 1; 2176} 2177 2178/** 2179 * t3_offload_tx - send an offload packet 2180 * @tdev: the offload device to send to 2181 * @m: the packet 2182 * 2183 * Sends an offload packet. We use the packet priority to select the 2184 * appropriate Tx queue as follows: bit 0 indicates whether the packet 2185 * should be sent as regular or control, bits 1-3 select the queue set. 2186 */ 2187int 2188t3_offload_tx(struct t3cdev *tdev, struct mbuf *m) 2189{ 2190 adapter_t *adap = tdev2adap(tdev); 2191 struct sge_qset *qs = &adap->sge.qs[queue_set(m)]; 2192 2193 if (__predict_false(is_ctrl_pkt(m))) 2194 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], m); 2195 2196 return ofld_xmit(adap, &qs->txq[TXQ_OFLD], m); 2197} 2198 2199/** 2200 * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts 2201 * @tdev: the offload device that will be receiving the packets 2202 * @q: the SGE response queue that assembled the bundle 2203 * @m: the partial bundle 2204 * @n: the number of packets in the bundle 2205 * 2206 * Delivers a (partial) bundle of Rx offload packets to an offload device. 2207 */ 2208static __inline void 2209deliver_partial_bundle(struct t3cdev *tdev, 2210 struct sge_rspq *q, 2211 struct mbuf *mbufs[], int n) 2212{ 2213 if (n) { 2214 q->offload_bundles++; 2215 cxgb_ofld_recv(tdev, mbufs, n); 2216 } 2217} 2218 2219static __inline int 2220rx_offload(struct t3cdev *tdev, struct sge_rspq *rq, 2221 struct mbuf *m, struct mbuf *rx_gather[], 2222 unsigned int gather_idx) 2223{ 2224 2225 rq->offload_pkts++; 2226 m->m_pkthdr.header = mtod(m, void *); 2227 rx_gather[gather_idx++] = m; 2228 if (gather_idx == RX_BUNDLE_SIZE) { 2229 cxgb_ofld_recv(tdev, rx_gather, RX_BUNDLE_SIZE); 2230 gather_idx = 0; 2231 rq->offload_bundles++; 2232 } 2233 return (gather_idx); 2234} 2235 2236static void 2237restart_tx(struct sge_qset *qs) 2238{ 2239 struct adapter *sc = qs->port->adapter; 2240 2241 2242 if (isset(&qs->txq_stopped, TXQ_OFLD) && 2243 should_restart_tx(&qs->txq[TXQ_OFLD]) && 2244 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { 2245 qs->txq[TXQ_OFLD].restarts++; 2246 DPRINTF("restarting TXQ_OFLD\n"); 2247 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); 2248 } 2249 DPRINTF("stopped=0x%x restart=%d processed=%d cleaned=%d in_use=%d\n", 2250 qs->txq_stopped, should_restart_tx(&qs->txq[TXQ_CTRL]), 2251 qs->txq[TXQ_CTRL].processed, qs->txq[TXQ_CTRL].cleaned, 2252 qs->txq[TXQ_CTRL].in_use); 2253 2254 if (isset(&qs->txq_stopped, TXQ_CTRL) && 2255 should_restart_tx(&qs->txq[TXQ_CTRL]) && 2256 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { 2257 qs->txq[TXQ_CTRL].restarts++; 2258 DPRINTF("restarting TXQ_CTRL\n"); 2259 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); 2260 } 2261} 2262 2263/** 2264 * t3_sge_alloc_qset - initialize an SGE queue set 2265 * @sc: the controller softc 2266 * @id: the queue set id 2267 * @nports: how many Ethernet ports will be using this queue set 2268 * @irq_vec_idx: the IRQ vector index for response queue interrupts 2269 * @p: configuration parameters for this queue set 2270 * @ntxq: number of Tx queues for the queue set 2271 * @pi: port info for queue set 2272 * 2273 * Allocate resources and initialize an SGE queue set. A queue set 2274 * comprises a response queue, two Rx free-buffer queues, and up to 3 2275 * Tx queues. The Tx queues are assigned roles in the order Ethernet 2276 * queue, offload queue, and control queue. 2277 */ 2278int 2279t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, 2280 const struct qset_params *p, int ntxq, struct port_info *pi) 2281{ 2282 struct sge_qset *q = &sc->sge.qs[id]; 2283 int i, header_size, ret = 0; 2284 2285 for (i = 0; i < SGE_TXQ_PER_SET; i++) { 2286 if ((q->txq[i].txq_mr.br_ring = malloc(cxgb_txq_buf_ring_size*sizeof(struct mbuf *), 2287 M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { 2288 device_printf(sc->dev, "failed to allocate mbuf ring\n"); 2289 goto err; 2290 } 2291 q->txq[i].txq_mr.br_prod = q->txq[i].txq_mr.br_cons = 0; 2292 q->txq[i].txq_mr.br_size = cxgb_txq_buf_ring_size; 2293 mtx_init(&q->txq[i].txq_mr.br_lock, "txq mbuf ring", NULL, MTX_DEF); 2294 } 2295 2296 init_qset_cntxt(q, id); 2297 q->idx = id; 2298 2299 if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), 2300 sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, 2301 &q->fl[0].desc, &q->fl[0].sdesc, 2302 &q->fl[0].desc_tag, &q->fl[0].desc_map, 2303 sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { 2304 printf("error %d from alloc ring fl0\n", ret); 2305 goto err; 2306 } 2307 2308 if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), 2309 sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, 2310 &q->fl[1].desc, &q->fl[1].sdesc, 2311 &q->fl[1].desc_tag, &q->fl[1].desc_map, 2312 sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { 2313 printf("error %d from alloc ring fl1\n", ret); 2314 goto err; 2315 } 2316 2317 if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, 2318 &q->rspq.phys_addr, &q->rspq.desc, NULL, 2319 &q->rspq.desc_tag, &q->rspq.desc_map, 2320 NULL, NULL)) != 0) { 2321 printf("error %d from alloc ring rspq\n", ret); 2322 goto err; 2323 } 2324 2325 for (i = 0; i < ntxq; ++i) { 2326 /* 2327 * The control queue always uses immediate data so does not 2328 * need to keep track of any mbufs. 2329 * XXX Placeholder for future TOE support. 2330 */ 2331 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); 2332 2333 if ((ret = alloc_ring(sc, p->txq_size[i], 2334 sizeof(struct tx_desc), sz, 2335 &q->txq[i].phys_addr, &q->txq[i].desc, 2336 &q->txq[i].sdesc, &q->txq[i].desc_tag, 2337 &q->txq[i].desc_map, 2338 sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { 2339 printf("error %d from alloc ring tx %i\n", ret, i); 2340 goto err; 2341 } 2342 mbufq_init(&q->txq[i].sendq); 2343 q->txq[i].gen = 1; 2344 q->txq[i].size = p->txq_size[i]; 2345 snprintf(q->txq[i].lockbuf, TXQ_NAME_LEN, "t3 txq lock %d:%d:%d", 2346 device_get_unit(sc->dev), irq_vec_idx, i); 2347 MTX_INIT(&q->txq[i].lock, q->txq[i].lockbuf, NULL, MTX_DEF); 2348 } 2349 2350 q->txq[TXQ_ETH].port = pi; 2351 2352 TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q); 2353 TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q); 2354 TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, &q->txq[TXQ_ETH]); 2355 TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, &q->txq[TXQ_OFLD]); 2356 2357 q->fl[0].gen = q->fl[1].gen = 1; 2358 q->fl[0].size = p->fl_size; 2359 q->fl[1].size = p->jumbo_size; 2360 2361 q->rspq.gen = 1; 2362 q->rspq.cidx = 0; 2363 q->rspq.size = p->rspq_size; 2364 2365 2366 header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t); 2367 q->txq[TXQ_ETH].stop_thres = nports * 2368 flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); 2369 2370 q->fl[0].buf_size = (MCLBYTES - header_size); 2371 q->fl[0].zone = zone_clust; 2372 q->fl[0].type = EXT_CLUSTER; 2373#if __FreeBSD_version > 800000 2374 if (cxgb_use_16k_clusters) { 2375 q->fl[1].buf_size = MJUM16BYTES - header_size; 2376 q->fl[1].zone = zone_jumbo16; 2377 q->fl[1].type = EXT_JUMBO16; 2378 } else { 2379 q->fl[1].buf_size = MJUM9BYTES - header_size; 2380 q->fl[1].zone = zone_jumbo9; 2381 q->fl[1].type = EXT_JUMBO9; 2382 } 2383#else 2384 q->fl[1].buf_size = MJUMPAGESIZE - header_size; 2385 q->fl[1].zone = zone_jumbop; 2386 q->fl[1].type = EXT_JUMBOP; 2387#endif 2388 2389 /* 2390 * We allocate and setup the lro_ctrl structure irrespective of whether 2391 * lro is available and/or enabled. 2392 */ 2393 q->lro.enabled = !!(pi->ifp->if_capenable & IFCAP_LRO); 2394 ret = tcp_lro_init(&q->lro.ctrl); 2395 if (ret) { 2396 printf("error %d from tcp_lro_init\n", ret); 2397 goto err; 2398 } 2399 q->lro.ctrl.ifp = pi->ifp; 2400 2401 mtx_lock_spin(&sc->sge.reg_lock); 2402 ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, 2403 q->rspq.phys_addr, q->rspq.size, 2404 q->fl[0].buf_size, 1, 0); 2405 if (ret) { 2406 printf("error %d from t3_sge_init_rspcntxt\n", ret); 2407 goto err_unlock; 2408 } 2409 2410 for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 2411 ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, 2412 q->fl[i].phys_addr, q->fl[i].size, 2413 q->fl[i].buf_size, p->cong_thres, 1, 2414 0); 2415 if (ret) { 2416 printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i); 2417 goto err_unlock; 2418 } 2419 } 2420 2421 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, 2422 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, 2423 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, 2424 1, 0); 2425 if (ret) { 2426 printf("error %d from t3_sge_init_ecntxt\n", ret); 2427 goto err_unlock; 2428 } 2429 2430 if (ntxq > 1) { 2431 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, 2432 USE_GTS, SGE_CNTXT_OFLD, id, 2433 q->txq[TXQ_OFLD].phys_addr, 2434 q->txq[TXQ_OFLD].size, 0, 1, 0); 2435 if (ret) { 2436 printf("error %d from t3_sge_init_ecntxt\n", ret); 2437 goto err_unlock; 2438 } 2439 } 2440 2441 if (ntxq > 2) { 2442 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, 2443 SGE_CNTXT_CTRL, id, 2444 q->txq[TXQ_CTRL].phys_addr, 2445 q->txq[TXQ_CTRL].size, 2446 q->txq[TXQ_CTRL].token, 1, 0); 2447 if (ret) { 2448 printf("error %d from t3_sge_init_ecntxt\n", ret); 2449 goto err_unlock; 2450 } 2451 } 2452 2453 snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d", 2454 device_get_unit(sc->dev), irq_vec_idx); 2455 MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF); 2456 2457 mtx_unlock_spin(&sc->sge.reg_lock); 2458 t3_update_qset_coalesce(q, p); 2459 q->port = pi; 2460 2461 refill_fl(sc, &q->fl[0], q->fl[0].size); 2462 refill_fl(sc, &q->fl[1], q->fl[1].size); 2463 refill_rspq(sc, &q->rspq, q->rspq.size - 1); 2464 2465 t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | 2466 V_NEWTIMER(q->rspq.holdoff_tmr)); 2467 2468 return (0); 2469 2470err_unlock: 2471 mtx_unlock_spin(&sc->sge.reg_lock); 2472err: 2473 t3_free_qset(sc, q); 2474 2475 return (ret); 2476} 2477 2478/* 2479 * Remove CPL_RX_PKT headers from the mbuf and reduce it to a regular mbuf with 2480 * ethernet data. Hardware assistance with various checksums and any vlan tag 2481 * will also be taken into account here. 2482 */ 2483void 2484t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad) 2485{ 2486 struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad); 2487 struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]]; 2488 struct ifnet *ifp = pi->ifp; 2489 2490 DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, mtod(m, uint8_t *), cpl->iff); 2491 2492 if ((ifp->if_capenable & IFCAP_RXCSUM) && !cpl->fragment && 2493 cpl->csum_valid && cpl->csum == 0xffff) { 2494 m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); 2495 rspq_to_qset(rq)->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; 2496 m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID|CSUM_DATA_VALID|CSUM_PSEUDO_HDR); 2497 m->m_pkthdr.csum_data = 0xffff; 2498 } 2499 /* 2500 * XXX need to add VLAN support for 6.x 2501 */ 2502#ifdef VLAN_SUPPORTED 2503 if (__predict_false(cpl->vlan_valid)) { 2504 m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); 2505 m->m_flags |= M_VLANTAG; 2506 } 2507#endif 2508 2509 m->m_pkthdr.rcvif = ifp; 2510 m->m_pkthdr.header = mtod(m, uint8_t *) + sizeof(*cpl) + ethpad; 2511#ifndef DISABLE_MBUF_IOVEC 2512 m_explode(m); 2513#endif 2514 /* 2515 * adjust after conversion to mbuf chain 2516 */ 2517 m->m_pkthdr.len -= (sizeof(*cpl) + ethpad); 2518 m->m_len -= (sizeof(*cpl) + ethpad); 2519 m->m_data += (sizeof(*cpl) + ethpad); 2520} 2521 2522static void 2523ext_free_handler(void *arg1, void * arg2) 2524{ 2525 uintptr_t type = (uintptr_t)arg2; 2526 uma_zone_t zone; 2527 struct mbuf *m; 2528 2529 m = arg1; 2530 zone = m_getzonefromtype(type); 2531 m->m_ext.ext_type = (int)type; 2532 cxgb_ext_freed++; 2533 cxgb_cache_put(zone, m); 2534} 2535 2536static void 2537init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone) 2538{ 2539 struct mbuf *m; 2540 int header_size; 2541 2542 header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + 2543 sizeof(struct m_ext_) + sizeof(uint32_t); 2544 2545 bzero(cl, header_size); 2546 m = (struct mbuf *)cl; 2547 2548 cxgb_ext_inited++; 2549 SLIST_INIT(&m->m_pkthdr.tags); 2550 m->m_type = MT_DATA; 2551 m->m_flags = flags | M_NOFREE | M_EXT; 2552 m->m_data = cl + header_size; 2553 m->m_ext.ext_buf = cl; 2554 m->m_ext.ref_cnt = (uint32_t *)(cl + header_size - sizeof(uint32_t)); 2555 m->m_ext.ext_size = m_getsizefromtype(type); 2556 m->m_ext.ext_free = ext_free_handler; 2557 m->m_ext.ext_arg1 = cl; 2558 m->m_ext.ext_arg2 = (void *)(uintptr_t)type; 2559 m->m_ext.ext_type = EXT_EXTREF; 2560 *(m->m_ext.ref_cnt) = 1; 2561 DPRINTF("data=%p ref_cnt=%p\n", m->m_data, m->m_ext.ref_cnt); 2562} 2563 2564 2565/** 2566 * get_packet - return the next ingress packet buffer from a free list 2567 * @adap: the adapter that received the packet 2568 * @drop_thres: # of remaining buffers before we start dropping packets 2569 * @qs: the qset that the SGE free list holding the packet belongs to 2570 * @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain 2571 * @r: response descriptor 2572 * 2573 * Get the next packet from a free list and complete setup of the 2574 * sk_buff. If the packet is small we make a copy and recycle the 2575 * original buffer, otherwise we use the original buffer itself. If a 2576 * positive drop threshold is supplied packets are dropped and their 2577 * buffers recycled if (a) the number of remaining buffers is under the 2578 * threshold and the packet is too big to copy, or (b) the packet should 2579 * be copied but there is no memory for the copy. 2580 */ 2581#ifdef DISABLE_MBUF_IOVEC 2582 2583static int 2584get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, 2585 struct t3_mbuf_hdr *mh, struct rsp_desc *r) 2586{ 2587 2588 unsigned int len_cq = ntohl(r->len_cq); 2589 struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; 2590 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; 2591 uint32_t len = G_RSPD_LEN(len_cq); 2592 uint32_t flags = ntohl(r->flags); 2593 uint8_t sopeop = G_RSPD_SOP_EOP(flags); 2594 caddr_t cl; 2595 struct mbuf *m, *m0; 2596 int ret = 0; 2597 2598 prefetch(sd->rxsd_cl); 2599 2600 fl->credits--; 2601 bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); 2602 2603 if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { 2604 if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) 2605 goto skip_recycle; 2606 cl = mtod(m0, void *); 2607 memcpy(cl, sd->data, len); 2608 recycle_rx_buf(adap, fl, fl->cidx); 2609 m = m0; 2610 m0->m_len = len; 2611 } else { 2612 skip_recycle: 2613 2614 bus_dmamap_unload(fl->entry_tag, sd->map); 2615 cl = sd->rxsd_cl; 2616 m = m0 = (struct mbuf *)cl; 2617 2618 if ((sopeop == RSPQ_SOP_EOP) || 2619 (sopeop == RSPQ_SOP)) 2620 flags = M_PKTHDR; 2621 init_cluster_mbuf(cl, flags, fl->type, fl->zone); 2622 m0->m_len = len; 2623 } 2624 switch(sopeop) { 2625 case RSPQ_SOP_EOP: 2626 DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); 2627 mh->mh_head = mh->mh_tail = m; 2628 m->m_pkthdr.len = len; 2629 ret = 1; 2630 break; 2631 case RSPQ_NSOP_NEOP: 2632 DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); 2633 if (mh->mh_tail == NULL) { 2634 log(LOG_ERR, "discarding intermediate descriptor entry\n"); 2635 m_freem(m); 2636 break; 2637 } 2638 mh->mh_tail->m_next = m; 2639 mh->mh_tail = m; 2640 mh->mh_head->m_pkthdr.len += len; 2641 ret = 0; 2642 break; 2643 case RSPQ_SOP: 2644 DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); 2645 m->m_pkthdr.len = len; 2646 mh->mh_head = mh->mh_tail = m; 2647 ret = 0; 2648 break; 2649 case RSPQ_EOP: 2650 DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); 2651 mh->mh_head->m_pkthdr.len += len; 2652 mh->mh_tail->m_next = m; 2653 mh->mh_tail = m; 2654 ret = 1; 2655 break; 2656 } 2657 if (++fl->cidx == fl->size) 2658 fl->cidx = 0; 2659 2660 return (ret); 2661} 2662 2663#else 2664 2665static int 2666get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, 2667 struct mbuf **m, struct rsp_desc *r) 2668{ 2669 2670 unsigned int len_cq = ntohl(r->len_cq); 2671 struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; 2672 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; 2673 uint32_t len = G_RSPD_LEN(len_cq); 2674 uint32_t flags = ntohl(r->flags); 2675 uint8_t sopeop = G_RSPD_SOP_EOP(flags); 2676 void *cl; 2677 int ret = 0; 2678 struct mbuf *m0; 2679#if 0 2680 if ((sd + 1 )->rxsd_cl) 2681 prefetch((sd + 1)->rxsd_cl); 2682 if ((sd + 2)->rxsd_cl) 2683 prefetch((sd + 2)->rxsd_cl); 2684#endif 2685 DPRINTF("rx cpu=%d\n", curcpu); 2686 fl->credits--; 2687 bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); 2688 2689 if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { 2690 if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) 2691 goto skip_recycle; 2692 cl = mtod(m0, void *); 2693 memcpy(cl, sd->data, len); 2694 recycle_rx_buf(adap, fl, fl->cidx); 2695 *m = m0; 2696 } else { 2697 skip_recycle: 2698 bus_dmamap_unload(fl->entry_tag, sd->map); 2699 cl = sd->rxsd_cl; 2700 *m = m0 = (struct mbuf *)cl; 2701 } 2702 2703 switch(sopeop) { 2704 case RSPQ_SOP_EOP: 2705 DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); 2706 if (cl == sd->rxsd_cl) 2707 init_cluster_mbuf(cl, M_PKTHDR, fl->type, fl->zone); 2708 m0->m_len = m0->m_pkthdr.len = len; 2709 ret = 1; 2710 goto done; 2711 break; 2712 case RSPQ_NSOP_NEOP: 2713 DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); 2714 panic("chaining unsupported"); 2715 ret = 0; 2716 break; 2717 case RSPQ_SOP: 2718 DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); 2719 panic("chaining unsupported"); 2720 m_iovinit(m0); 2721 ret = 0; 2722 break; 2723 case RSPQ_EOP: 2724 DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); 2725 panic("chaining unsupported"); 2726 ret = 1; 2727 break; 2728 } 2729 panic("append not supported"); 2730#if 0 2731 m_iovappend(m0, cl, fl->buf_size, len, sizeof(uint32_t), sd->rxsd_ref); 2732#endif 2733done: 2734 if (++fl->cidx == fl->size) 2735 fl->cidx = 0; 2736 2737 return (ret); 2738} 2739#endif 2740/** 2741 * handle_rsp_cntrl_info - handles control information in a response 2742 * @qs: the queue set corresponding to the response 2743 * @flags: the response control flags 2744 * 2745 * Handles the control information of an SGE response, such as GTS 2746 * indications and completion credits for the queue set's Tx queues. 2747 * HW coalesces credits, we don't do any extra SW coalescing. 2748 */ 2749static __inline void 2750handle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags) 2751{ 2752 unsigned int credits; 2753 2754#if USE_GTS 2755 if (flags & F_RSPD_TXQ0_GTS) 2756 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); 2757#endif 2758 credits = G_RSPD_TXQ0_CR(flags); 2759 if (credits) 2760 qs->txq[TXQ_ETH].processed += credits; 2761 2762 credits = G_RSPD_TXQ2_CR(flags); 2763 if (credits) 2764 qs->txq[TXQ_CTRL].processed += credits; 2765 2766# if USE_GTS 2767 if (flags & F_RSPD_TXQ1_GTS) 2768 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); 2769# endif 2770 credits = G_RSPD_TXQ1_CR(flags); 2771 if (credits) 2772 qs->txq[TXQ_OFLD].processed += credits; 2773 2774} 2775 2776static void 2777check_ring_db(adapter_t *adap, struct sge_qset *qs, 2778 unsigned int sleeping) 2779{ 2780 ; 2781} 2782 2783/** 2784 * process_responses - process responses from an SGE response queue 2785 * @adap: the adapter 2786 * @qs: the queue set to which the response queue belongs 2787 * @budget: how many responses can be processed in this round 2788 * 2789 * Process responses from an SGE response queue up to the supplied budget. 2790 * Responses include received packets as well as credits and other events 2791 * for the queues that belong to the response queue's queue set. 2792 * A negative budget is effectively unlimited. 2793 * 2794 * Additionally choose the interrupt holdoff time for the next interrupt 2795 * on this queue. If the system is under memory shortage use a fairly 2796 * long delay to help recovery. 2797 */ 2798int 2799process_responses(adapter_t *adap, struct sge_qset *qs, int budget) 2800{ 2801 struct sge_rspq *rspq = &qs->rspq; 2802 struct rsp_desc *r = &rspq->desc[rspq->cidx]; 2803 int budget_left = budget; 2804 unsigned int sleeping = 0; 2805 int lro_enabled = qs->lro.enabled; 2806 struct lro_ctrl *lro_ctrl = &qs->lro.ctrl; 2807 struct mbuf *offload_mbufs[RX_BUNDLE_SIZE]; 2808 int ngathered = 0; 2809#ifdef DEBUG 2810 static int last_holdoff = 0; 2811 if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) { 2812 printf("next_holdoff=%d\n", rspq->holdoff_tmr); 2813 last_holdoff = rspq->holdoff_tmr; 2814 } 2815#endif 2816 rspq->next_holdoff = rspq->holdoff_tmr; 2817 2818 while (__predict_true(budget_left && is_new_response(r, rspq))) { 2819 int eth, eop = 0, ethpad = 0; 2820 uint32_t flags = ntohl(r->flags); 2821 uint32_t rss_csum = *(const uint32_t *)r; 2822 uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val); 2823 2824 eth = (r->rss_hdr.opcode == CPL_RX_PKT); 2825 2826 if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) { 2827 struct mbuf *m; 2828 2829 if (cxgb_debug) 2830 printf("async notification\n"); 2831 2832 if (rspq->rspq_mh.mh_head == NULL) { 2833 rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); 2834 m = rspq->rspq_mh.mh_head; 2835 } else { 2836 m = m_gethdr(M_DONTWAIT, MT_DATA); 2837 } 2838 2839 /* XXX m is lost here if rspq->rspq_mbuf is not NULL */ 2840 2841 if (m == NULL) 2842 goto no_mem; 2843 2844 memcpy(mtod(m, char *), r, AN_PKT_SIZE); 2845 m->m_len = m->m_pkthdr.len = AN_PKT_SIZE; 2846 *mtod(m, char *) = CPL_ASYNC_NOTIF; 2847 rss_csum = htonl(CPL_ASYNC_NOTIF << 24); 2848 eop = 1; 2849 rspq->async_notif++; 2850 goto skip; 2851 } else if (flags & F_RSPD_IMM_DATA_VALID) { 2852 struct mbuf *m = NULL; 2853 2854 DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", 2855 r->rss_hdr.opcode, rspq->cidx); 2856 if (rspq->rspq_mh.mh_head == NULL) 2857 rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); 2858 else 2859 m = m_gethdr(M_DONTWAIT, MT_DATA); 2860 2861 if (rspq->rspq_mh.mh_head == NULL && m == NULL) { 2862 no_mem: 2863 rspq->next_holdoff = NOMEM_INTR_DELAY; 2864 budget_left--; 2865 break; 2866 } 2867 get_imm_packet(adap, r, rspq->rspq_mh.mh_head); 2868 eop = 1; 2869 rspq->imm_data++; 2870 } else if (r->len_cq) { 2871 int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; 2872 2873#ifdef DISABLE_MBUF_IOVEC 2874 eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r); 2875#else 2876 eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r); 2877#endif 2878#ifdef IFNET_MULTIQUEUE 2879 rspq->rspq_mh.mh_head->m_pkthdr.rss_hash = rss_hash; 2880#endif 2881 ethpad = 2; 2882 } else { 2883 DPRINTF("pure response\n"); 2884 rspq->pure_rsps++; 2885 } 2886 skip: 2887 if (flags & RSPD_CTRL_MASK) { 2888 sleeping |= flags & RSPD_GTS_MASK; 2889 handle_rsp_cntrl_info(qs, flags); 2890 } 2891 2892 r++; 2893 if (__predict_false(++rspq->cidx == rspq->size)) { 2894 rspq->cidx = 0; 2895 rspq->gen ^= 1; 2896 r = rspq->desc; 2897 } 2898 prefetch(r); 2899 if (++rspq->credits >= (rspq->size / 4)) { 2900 refill_rspq(adap, rspq, rspq->credits); 2901 rspq->credits = 0; 2902 } 2903 DPRINTF("eth=%d eop=%d flags=0x%x\n", eth, eop, flags); 2904 2905 if (!eth && eop) { 2906 rspq->rspq_mh.mh_head->m_pkthdr.csum_data = rss_csum; 2907 /* 2908 * XXX size mismatch 2909 */ 2910 m_set_priority(rspq->rspq_mh.mh_head, rss_hash); 2911 2912 2913 ngathered = rx_offload(&adap->tdev, rspq, 2914 rspq->rspq_mh.mh_head, offload_mbufs, ngathered); 2915 rspq->rspq_mh.mh_head = NULL; 2916 DPRINTF("received offload packet\n"); 2917 2918 } else if (eth && eop) { 2919 struct mbuf *m = rspq->rspq_mh.mh_head; 2920 prefetch(mtod(m, uint8_t *)); 2921 prefetch(mtod(m, uint8_t *) + L1_CACHE_BYTES); 2922 2923 t3_rx_eth(adap, rspq, m, ethpad); 2924 if (lro_enabled && lro_ctrl->lro_cnt && 2925 (tcp_lro_rx(lro_ctrl, m, 0) == 0)) { 2926 /* successfully queue'd for LRO */ 2927 } else { 2928 /* 2929 * LRO not enabled, packet unsuitable for LRO, 2930 * or unable to queue. Pass it up right now in 2931 * either case. 2932 */ 2933 struct ifnet *ifp = m->m_pkthdr.rcvif; 2934 (*ifp->if_input)(ifp, m); 2935 } 2936 DPRINTF("received tunnel packet\n"); 2937 rspq->rspq_mh.mh_head = NULL; 2938 2939 } 2940 __refill_fl_lt(adap, &qs->fl[0], 32); 2941 __refill_fl_lt(adap, &qs->fl[1], 32); 2942 --budget_left; 2943 } 2944 2945 deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered); 2946 2947 /* Flush LRO */ 2948 while (!SLIST_EMPTY(&lro_ctrl->lro_active)) { 2949 struct lro_entry *queued = SLIST_FIRST(&lro_ctrl->lro_active); 2950 SLIST_REMOVE_HEAD(&lro_ctrl->lro_active, next); 2951 tcp_lro_flush(lro_ctrl, queued); 2952 } 2953 2954 if (sleeping) 2955 check_ring_db(adap, qs, sleeping); 2956 2957 smp_mb(); /* commit Tx queue processed updates */ 2958 if (__predict_false(qs->txq_stopped > 1)) { 2959 printf("restarting tx on %p\n", qs); 2960 2961 restart_tx(qs); 2962 } 2963 2964 __refill_fl_lt(adap, &qs->fl[0], 512); 2965 __refill_fl_lt(adap, &qs->fl[1], 512); 2966 budget -= budget_left; 2967 return (budget); 2968} 2969 2970/* 2971 * A helper function that processes responses and issues GTS. 2972 */ 2973static __inline int 2974process_responses_gts(adapter_t *adap, struct sge_rspq *rq) 2975{ 2976 int work; 2977 static int last_holdoff = 0; 2978 2979 work = process_responses(adap, rspq_to_qset(rq), -1); 2980 2981 if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { 2982 printf("next_holdoff=%d\n", rq->next_holdoff); 2983 last_holdoff = rq->next_holdoff; 2984 } 2985 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | 2986 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); 2987 2988 return (work); 2989} 2990 2991 2992/* 2993 * Interrupt handler for legacy INTx interrupts for T3B-based cards. 2994 * Handles data events from SGE response queues as well as error and other 2995 * async events as they all use the same interrupt pin. We use one SGE 2996 * response queue per port in this mode and protect all response queues with 2997 * queue 0's lock. 2998 */ 2999void 3000t3b_intr(void *data) 3001{ 3002 uint32_t i, map; 3003 adapter_t *adap = data; 3004 struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 3005 3006 t3_write_reg(adap, A_PL_CLI, 0); 3007 map = t3_read_reg(adap, A_SG_DATA_INTR); 3008 3009 if (!map) 3010 return; 3011 3012 if (__predict_false(map & F_ERRINTR)) 3013 taskqueue_enqueue(adap->tq, &adap->slow_intr_task); 3014 3015 mtx_lock(&q0->lock); 3016 for_each_port(adap, i) 3017 if (map & (1 << i)) 3018 process_responses_gts(adap, &adap->sge.qs[i].rspq); 3019 mtx_unlock(&q0->lock); 3020} 3021 3022/* 3023 * The MSI interrupt handler. This needs to handle data events from SGE 3024 * response queues as well as error and other async events as they all use 3025 * the same MSI vector. We use one SGE response queue per port in this mode 3026 * and protect all response queues with queue 0's lock. 3027 */ 3028void 3029t3_intr_msi(void *data) 3030{ 3031 adapter_t *adap = data; 3032 struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 3033 int i, new_packets = 0; 3034 3035 mtx_lock(&q0->lock); 3036 3037 for_each_port(adap, i) 3038 if (process_responses_gts(adap, &adap->sge.qs[i].rspq)) 3039 new_packets = 1; 3040 mtx_unlock(&q0->lock); 3041 if (new_packets == 0) 3042 taskqueue_enqueue(adap->tq, &adap->slow_intr_task); 3043} 3044 3045void 3046t3_intr_msix(void *data) 3047{ 3048 struct sge_qset *qs = data; 3049 adapter_t *adap = qs->port->adapter; 3050 struct sge_rspq *rspq = &qs->rspq; 3051#ifndef IFNET_MULTIQUEUE 3052 mtx_lock(&rspq->lock); 3053#else 3054 if (mtx_trylock(&rspq->lock)) 3055#endif 3056 { 3057 3058 if (process_responses_gts(adap, rspq) == 0) 3059 rspq->unhandled_irqs++; 3060 mtx_unlock(&rspq->lock); 3061 } 3062} 3063 3064#define QDUMP_SBUF_SIZE 32 * 400 3065static int 3066t3_dump_rspq(SYSCTL_HANDLER_ARGS) 3067{ 3068 struct sge_rspq *rspq; 3069 struct sge_qset *qs; 3070 int i, err, dump_end, idx; 3071 static int multiplier = 1; 3072 struct sbuf *sb; 3073 struct rsp_desc *rspd; 3074 uint32_t data[4]; 3075 3076 rspq = arg1; 3077 qs = rspq_to_qset(rspq); 3078 if (rspq->rspq_dump_count == 0) 3079 return (0); 3080 if (rspq->rspq_dump_count > RSPQ_Q_SIZE) { 3081 log(LOG_WARNING, 3082 "dump count is too large %d\n", rspq->rspq_dump_count); 3083 rspq->rspq_dump_count = 0; 3084 return (EINVAL); 3085 } 3086 if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) { 3087 log(LOG_WARNING, 3088 "dump start of %d is greater than queue size\n", 3089 rspq->rspq_dump_start); 3090 rspq->rspq_dump_start = 0; 3091 return (EINVAL); 3092 } 3093 err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data); 3094 if (err) 3095 return (err); 3096retry_sbufops: 3097 sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3098 3099 sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n", 3100 (data[0] & 0xffff), data[0] >> 16, ((data[2] >> 20) & 0x3f), 3101 ((data[2] >> 26) & 1), ((data[2] >> 27) & 1)); 3102 sbuf_printf(sb, " generation=%u CQ mode=%u FL threshold=%u\n", 3103 ((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]); 3104 3105 sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start, 3106 (rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1)); 3107 3108 dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count; 3109 for (i = rspq->rspq_dump_start; i < dump_end; i++) { 3110 idx = i & (RSPQ_Q_SIZE-1); 3111 3112 rspd = &rspq->desc[idx]; 3113 sbuf_printf(sb, "\tidx=%04d opcode=%02x cpu_idx=%x hash_type=%x cq_idx=%x\n", 3114 idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx, 3115 rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx)); 3116 sbuf_printf(sb, "\trss_hash_val=%x flags=%08x len_cq=%x intr_gen=%x\n", 3117 rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags), 3118 be32toh(rspd->len_cq), rspd->intr_gen); 3119 } 3120 if (sbuf_overflowed(sb)) { 3121 sbuf_delete(sb); 3122 multiplier++; 3123 goto retry_sbufops; 3124 } 3125 sbuf_finish(sb); 3126 err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3127 sbuf_delete(sb); 3128 return (err); 3129} 3130 3131static int 3132t3_dump_txq_eth(SYSCTL_HANDLER_ARGS) 3133{ 3134 struct sge_txq *txq; 3135 struct sge_qset *qs; 3136 int i, j, err, dump_end; 3137 static int multiplier = 1; 3138 struct sbuf *sb; 3139 struct tx_desc *txd; 3140 uint32_t *WR, wr_hi, wr_lo, gen; 3141 uint32_t data[4]; 3142 3143 txq = arg1; 3144 qs = txq_to_qset(txq, TXQ_ETH); 3145 if (txq->txq_dump_count == 0) { 3146 return (0); 3147 } 3148 if (txq->txq_dump_count > TX_ETH_Q_SIZE) { 3149 log(LOG_WARNING, 3150 "dump count is too large %d\n", txq->txq_dump_count); 3151 txq->txq_dump_count = 1; 3152 return (EINVAL); 3153 } 3154 if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) { 3155 log(LOG_WARNING, 3156 "dump start of %d is greater than queue size\n", 3157 txq->txq_dump_start); 3158 txq->txq_dump_start = 0; 3159 return (EINVAL); 3160 } 3161 err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data); 3162 if (err) 3163 return (err); 3164 3165 3166retry_sbufops: 3167 sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3168 3169 sbuf_printf(sb, " \n credits=%u GTS=%u index=%u size=%u rspq#=%u cmdq#=%u\n", 3170 (data[0] & 0x7fff), ((data[0] >> 15) & 1), (data[0] >> 16), 3171 (data[1] & 0xffff), ((data[3] >> 4) & 7), ((data[3] >> 7) & 1)); 3172 sbuf_printf(sb, " TUN=%u TOE=%u generation%u uP token=%u valid=%u\n", 3173 ((data[3] >> 8) & 1), ((data[3] >> 9) & 1), ((data[3] >> 10) & 1), 3174 ((data[3] >> 11) & 0xfffff), ((data[3] >> 31) & 1)); 3175 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, 3176 txq->txq_dump_start, 3177 (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1)); 3178 3179 dump_end = txq->txq_dump_start + txq->txq_dump_count; 3180 for (i = txq->txq_dump_start; i < dump_end; i++) { 3181 txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)]; 3182 WR = (uint32_t *)txd->flit; 3183 wr_hi = ntohl(WR[0]); 3184 wr_lo = ntohl(WR[1]); 3185 gen = G_WR_GEN(wr_lo); 3186 3187 sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", 3188 wr_hi, wr_lo, gen); 3189 for (j = 2; j < 30; j += 4) 3190 sbuf_printf(sb, "\t%08x %08x %08x %08x \n", 3191 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); 3192 3193 } 3194 if (sbuf_overflowed(sb)) { 3195 sbuf_delete(sb); 3196 multiplier++; 3197 goto retry_sbufops; 3198 } 3199 sbuf_finish(sb); 3200 err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3201 sbuf_delete(sb); 3202 return (err); 3203} 3204 3205static int 3206t3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS) 3207{ 3208 struct sge_txq *txq; 3209 struct sge_qset *qs; 3210 int i, j, err, dump_end; 3211 static int multiplier = 1; 3212 struct sbuf *sb; 3213 struct tx_desc *txd; 3214 uint32_t *WR, wr_hi, wr_lo, gen; 3215 3216 txq = arg1; 3217 qs = txq_to_qset(txq, TXQ_CTRL); 3218 if (txq->txq_dump_count == 0) { 3219 return (0); 3220 } 3221 if (txq->txq_dump_count > 256) { 3222 log(LOG_WARNING, 3223 "dump count is too large %d\n", txq->txq_dump_count); 3224 txq->txq_dump_count = 1; 3225 return (EINVAL); 3226 } 3227 if (txq->txq_dump_start > 255) { 3228 log(LOG_WARNING, 3229 "dump start of %d is greater than queue size\n", 3230 txq->txq_dump_start); 3231 txq->txq_dump_start = 0; 3232 return (EINVAL); 3233 } 3234 3235retry_sbufops: 3236 sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); 3237 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, 3238 txq->txq_dump_start, 3239 (txq->txq_dump_start + txq->txq_dump_count) & 255); 3240 3241 dump_end = txq->txq_dump_start + txq->txq_dump_count; 3242 for (i = txq->txq_dump_start; i < dump_end; i++) { 3243 txd = &txq->desc[i & (255)]; 3244 WR = (uint32_t *)txd->flit; 3245 wr_hi = ntohl(WR[0]); 3246 wr_lo = ntohl(WR[1]); 3247 gen = G_WR_GEN(wr_lo); 3248 3249 sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", 3250 wr_hi, wr_lo, gen); 3251 for (j = 2; j < 30; j += 4) 3252 sbuf_printf(sb, "\t%08x %08x %08x %08x \n", 3253 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); 3254 3255 } 3256 if (sbuf_overflowed(sb)) { 3257 sbuf_delete(sb); 3258 multiplier++; 3259 goto retry_sbufops; 3260 } 3261 sbuf_finish(sb); 3262 err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 3263 sbuf_delete(sb); 3264 return (err); 3265} 3266 3267static int 3268t3_set_coalesce_usecs(SYSCTL_HANDLER_ARGS) 3269{ 3270 adapter_t *sc = arg1; 3271 struct qset_params *qsp = &sc->params.sge.qset[0]; 3272 int coalesce_usecs; 3273 struct sge_qset *qs; 3274 int i, j, err, nqsets = 0; 3275 struct mtx *lock; 3276 3277 if ((sc->flags & FULL_INIT_DONE) == 0) 3278 return (ENXIO); 3279 3280 coalesce_usecs = qsp->coalesce_usecs; 3281 err = sysctl_handle_int(oidp, &coalesce_usecs, arg2, req); 3282 3283 if (err != 0) { 3284 return (err); 3285 } 3286 if (coalesce_usecs == qsp->coalesce_usecs) 3287 return (0); 3288 3289 for (i = 0; i < sc->params.nports; i++) 3290 for (j = 0; j < sc->port[i].nqsets; j++) 3291 nqsets++; 3292 3293 coalesce_usecs = max(1, coalesce_usecs); 3294 3295 for (i = 0; i < nqsets; i++) { 3296 qs = &sc->sge.qs[i]; 3297 qsp = &sc->params.sge.qset[i]; 3298 qsp->coalesce_usecs = coalesce_usecs; 3299 3300 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : 3301 &sc->sge.qs[0].rspq.lock; 3302 3303 mtx_lock(lock); 3304 t3_update_qset_coalesce(qs, qsp); 3305 t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | 3306 V_NEWTIMER(qs->rspq.holdoff_tmr)); 3307 mtx_unlock(lock); 3308 } 3309 3310 return (0); 3311} 3312 3313 3314void 3315t3_add_attach_sysctls(adapter_t *sc) 3316{ 3317 struct sysctl_ctx_list *ctx; 3318 struct sysctl_oid_list *children; 3319 3320 ctx = device_get_sysctl_ctx(sc->dev); 3321 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 3322 3323 /* random information */ 3324 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, 3325 "firmware_version", 3326 CTLFLAG_RD, &sc->fw_version, 3327 0, "firmware version"); 3328 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3329 "hw_revision", 3330 CTLFLAG_RD, &sc->params.rev, 3331 0, "chip model"); 3332 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3333 "enable_debug", 3334 CTLFLAG_RW, &cxgb_debug, 3335 0, "enable verbose debugging output"); 3336 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tunq_coalesce", 3337 CTLFLAG_RD, &sc->tunq_coalesce, 3338 "#tunneled packets freed"); 3339 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3340 "txq_overrun", 3341 CTLFLAG_RD, &txq_fills, 3342 0, "#times txq overrun"); 3343 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3344 "pcpu_cache_enable", 3345 CTLFLAG_RW, &cxgb_pcpu_cache_enable, 3346 0, "#enable driver local pcpu caches"); 3347 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3348 "cache_alloc", 3349 CTLFLAG_RD, &cxgb_cached_allocations, 3350 0, "#times a cluster was allocated from cache"); 3351 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3352 "cached", 3353 CTLFLAG_RD, &cxgb_cached, 3354 0, "#times a cluster was cached"); 3355 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3356 "ext_freed", 3357 CTLFLAG_RD, &cxgb_ext_freed, 3358 0, "#times a cluster was freed through ext_free"); 3359 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3360 "ext_inited", 3361 CTLFLAG_RD, &cxgb_ext_inited, 3362 0, "#times a cluster was initialized for ext_free"); 3363 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3364 "mbufs_outstanding", 3365 CTLFLAG_RD, &cxgb_mbufs_outstanding, 3366 0, "#mbufs in flight in the driver"); 3367 SYSCTL_ADD_INT(ctx, children, OID_AUTO, 3368 "pack_outstanding", 3369 CTLFLAG_RD, &cxgb_pack_outstanding, 3370 0, "#packet in flight in the driver"); 3371} 3372 3373 3374static const char *rspq_name = "rspq"; 3375static const char *txq_names[] = 3376{ 3377 "txq_eth", 3378 "txq_ofld", 3379 "txq_ctrl" 3380}; 3381 3382static int 3383sysctl_handle_macstat(SYSCTL_HANDLER_ARGS) 3384{ 3385 struct port_info *p = arg1; 3386 uint64_t *parg; 3387 3388 if (!p) 3389 return (EINVAL); 3390 3391 parg = (uint64_t *) ((uint8_t *)&p->mac.stats + arg2); 3392 3393 PORT_LOCK(p); 3394 t3_mac_update_stats(&p->mac); 3395 PORT_UNLOCK(p); 3396 3397 return (sysctl_handle_quad(oidp, parg, 0, req)); 3398} 3399 3400void 3401t3_add_configured_sysctls(adapter_t *sc) 3402{ 3403 struct sysctl_ctx_list *ctx; 3404 struct sysctl_oid_list *children; 3405 int i, j; 3406 3407 ctx = device_get_sysctl_ctx(sc->dev); 3408 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 3409 3410 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, 3411 "intr_coal", 3412 CTLTYPE_INT|CTLFLAG_RW, sc, 3413 0, t3_set_coalesce_usecs, 3414 "I", "interrupt coalescing timer (us)"); 3415 3416 for (i = 0; i < sc->params.nports; i++) { 3417 struct port_info *pi = &sc->port[i]; 3418 struct sysctl_oid *poid; 3419 struct sysctl_oid_list *poidlist; 3420 struct mac_stats *mstats = &pi->mac.stats; 3421 3422 snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i); 3423 poid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, 3424 pi->namebuf, CTLFLAG_RD, NULL, "port statistics"); 3425 poidlist = SYSCTL_CHILDREN(poid); 3426 SYSCTL_ADD_INT(ctx, poidlist, OID_AUTO, 3427 "nqsets", CTLFLAG_RD, &pi->nqsets, 3428 0, "#queue sets"); 3429 3430 for (j = 0; j < pi->nqsets; j++) { 3431 struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j]; 3432 struct sysctl_oid *qspoid, *rspqpoid, *txqpoid, *ctrlqpoid, *lropoid; 3433 struct sysctl_oid_list *qspoidlist, *rspqpoidlist, *txqpoidlist, *ctrlqpoidlist, *lropoidlist; 3434 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 3435 3436 snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j); 3437 3438 qspoid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, 3439 qs->namebuf, CTLFLAG_RD, NULL, "qset statistics"); 3440 qspoidlist = SYSCTL_CHILDREN(qspoid); 3441 3442 rspqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3443 rspq_name, CTLFLAG_RD, NULL, "rspq statistics"); 3444 rspqpoidlist = SYSCTL_CHILDREN(rspqpoid); 3445 3446 txqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3447 txq_names[0], CTLFLAG_RD, NULL, "txq statistics"); 3448 txqpoidlist = SYSCTL_CHILDREN(txqpoid); 3449 3450 ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3451 txq_names[2], CTLFLAG_RD, NULL, "ctrlq statistics"); 3452 ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid); 3453 3454 lropoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, 3455 "lro_stats", CTLFLAG_RD, NULL, "LRO statistics"); 3456 lropoidlist = SYSCTL_CHILDREN(lropoid); 3457 3458 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size", 3459 CTLFLAG_RD, &qs->rspq.size, 3460 0, "#entries in response queue"); 3461 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "cidx", 3462 CTLFLAG_RD, &qs->rspq.cidx, 3463 0, "consumer index"); 3464 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "credits", 3465 CTLFLAG_RD, &qs->rspq.credits, 3466 0, "#credits"); 3467 SYSCTL_ADD_XLONG(ctx, rspqpoidlist, OID_AUTO, "phys_addr", 3468 CTLFLAG_RD, &qs->rspq.phys_addr, 3469 "physical_address_of the queue"); 3470 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_start", 3471 CTLFLAG_RW, &qs->rspq.rspq_dump_start, 3472 0, "start rspq dump entry"); 3473 SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_count", 3474 CTLFLAG_RW, &qs->rspq.rspq_dump_count, 3475 0, "#rspq entries to dump"); 3476 SYSCTL_ADD_PROC(ctx, rspqpoidlist, OID_AUTO, "qdump", 3477 CTLTYPE_STRING | CTLFLAG_RD, &qs->rspq, 3478 0, t3_dump_rspq, "A", "dump of the response queue"); 3479 3480 3481 SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "dropped", 3482 CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_drops, 3483 0, "#tunneled packets dropped"); 3484 SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "sendqlen", 3485 CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.qlen, 3486 0, "#tunneled packets waiting to be sent"); 3487 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_pidx", 3488 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod, 3489 0, "#tunneled packets queue producer index"); 3490 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_cidx", 3491 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons, 3492 0, "#tunneled packets queue consumer index"); 3493 SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "processed", 3494 CTLFLAG_RD, &qs->txq[TXQ_ETH].processed, 3495 0, "#tunneled packets processed by the card"); 3496 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "cleaned", 3497 CTLFLAG_RD, &txq->cleaned, 3498 0, "#tunneled packets cleaned"); 3499 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "in_use", 3500 CTLFLAG_RD, &txq->in_use, 3501 0, "#tunneled packet slots in use"); 3502 SYSCTL_ADD_ULONG(ctx, txqpoidlist, OID_AUTO, "frees", 3503 CTLFLAG_RD, &txq->txq_frees, 3504 "#tunneled packets freed"); 3505 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "skipped", 3506 CTLFLAG_RD, &txq->txq_skipped, 3507 0, "#tunneled packet descriptors skipped"); 3508 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "coalesced", 3509 CTLFLAG_RD, &txq->txq_coalesced, 3510 0, "#tunneled packets coalesced"); 3511 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "enqueued", 3512 CTLFLAG_RD, &txq->txq_enqueued, 3513 0, "#tunneled packets enqueued to hardware"); 3514 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "stopped_flags", 3515 CTLFLAG_RD, &qs->txq_stopped, 3516 0, "tx queues stopped"); 3517 SYSCTL_ADD_XLONG(ctx, txqpoidlist, OID_AUTO, "phys_addr", 3518 CTLFLAG_RD, &txq->phys_addr, 3519 "physical_address_of the queue"); 3520 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "qgen", 3521 CTLFLAG_RW, &qs->txq[TXQ_ETH].gen, 3522 0, "txq generation"); 3523 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_cidx", 3524 CTLFLAG_RD, &txq->cidx, 3525 0, "hardware queue cidx"); 3526 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_pidx", 3527 CTLFLAG_RD, &txq->pidx, 3528 0, "hardware queue pidx"); 3529 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_start", 3530 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start, 3531 0, "txq start idx for dump"); 3532 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_count", 3533 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count, 3534 0, "txq #entries to dump"); 3535 SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump", 3536 CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH], 3537 0, t3_dump_txq_eth, "A", "dump of the transmit queue"); 3538 3539 SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start", 3540 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start, 3541 0, "ctrlq start idx for dump"); 3542 SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count", 3543 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count, 3544 0, "ctrl #entries to dump"); 3545 SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump", 3546 CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_CTRL], 3547 0, t3_dump_txq_ctrl, "A", "dump of the transmit queue"); 3548 3549 SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_queued", 3550 CTLFLAG_RD, &qs->lro.ctrl.lro_queued, 0, NULL); 3551 SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_flushed", 3552 CTLFLAG_RD, &qs->lro.ctrl.lro_flushed, 0, NULL); 3553 SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_bad_csum", 3554 CTLFLAG_RD, &qs->lro.ctrl.lro_bad_csum, 0, NULL); 3555 SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_cnt", 3556 CTLFLAG_RD, &qs->lro.ctrl.lro_cnt, 0, NULL); 3557 } 3558 3559 /* Now add a node for mac stats. */ 3560 poid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, "mac_stats", 3561 CTLFLAG_RD, NULL, "MAC statistics"); 3562 poidlist = SYSCTL_CHILDREN(poid); 3563 3564 /* 3565 * We (ab)use the length argument (arg2) to pass on the offset 3566 * of the data that we are interested in. This is only required 3567 * for the quad counters that are updated from the hardware (we 3568 * make sure that we return the latest value). 3569 * sysctl_handle_macstat first updates *all* the counters from 3570 * the hardware, and then returns the latest value of the 3571 * requested counter. Best would be to update only the 3572 * requested counter from hardware, but t3_mac_update_stats() 3573 * hides all the register details and we don't want to dive into 3574 * all that here. 3575 */ 3576#define CXGB_SYSCTL_ADD_QUAD(a) SYSCTL_ADD_OID(ctx, poidlist, OID_AUTO, #a, \ 3577 (CTLTYPE_QUAD | CTLFLAG_RD), pi, offsetof(struct mac_stats, a), \ 3578 sysctl_handle_macstat, "QU", 0) 3579 CXGB_SYSCTL_ADD_QUAD(tx_octets); 3580 CXGB_SYSCTL_ADD_QUAD(tx_octets_bad); 3581 CXGB_SYSCTL_ADD_QUAD(tx_frames); 3582 CXGB_SYSCTL_ADD_QUAD(tx_mcast_frames); 3583 CXGB_SYSCTL_ADD_QUAD(tx_bcast_frames); 3584 CXGB_SYSCTL_ADD_QUAD(tx_pause); 3585 CXGB_SYSCTL_ADD_QUAD(tx_deferred); 3586 CXGB_SYSCTL_ADD_QUAD(tx_late_collisions); 3587 CXGB_SYSCTL_ADD_QUAD(tx_total_collisions); 3588 CXGB_SYSCTL_ADD_QUAD(tx_excess_collisions); 3589 CXGB_SYSCTL_ADD_QUAD(tx_underrun); 3590 CXGB_SYSCTL_ADD_QUAD(tx_len_errs); 3591 CXGB_SYSCTL_ADD_QUAD(tx_mac_internal_errs); 3592 CXGB_SYSCTL_ADD_QUAD(tx_excess_deferral); 3593 CXGB_SYSCTL_ADD_QUAD(tx_fcs_errs); 3594 CXGB_SYSCTL_ADD_QUAD(tx_frames_64); 3595 CXGB_SYSCTL_ADD_QUAD(tx_frames_65_127); 3596 CXGB_SYSCTL_ADD_QUAD(tx_frames_128_255); 3597 CXGB_SYSCTL_ADD_QUAD(tx_frames_256_511); 3598 CXGB_SYSCTL_ADD_QUAD(tx_frames_512_1023); 3599 CXGB_SYSCTL_ADD_QUAD(tx_frames_1024_1518); 3600 CXGB_SYSCTL_ADD_QUAD(tx_frames_1519_max); 3601 CXGB_SYSCTL_ADD_QUAD(rx_octets); 3602 CXGB_SYSCTL_ADD_QUAD(rx_octets_bad); 3603 CXGB_SYSCTL_ADD_QUAD(rx_frames); 3604 CXGB_SYSCTL_ADD_QUAD(rx_mcast_frames); 3605 CXGB_SYSCTL_ADD_QUAD(rx_bcast_frames); 3606 CXGB_SYSCTL_ADD_QUAD(rx_pause); 3607 CXGB_SYSCTL_ADD_QUAD(rx_align_errs); 3608 CXGB_SYSCTL_ADD_QUAD(rx_symbol_errs); 3609 CXGB_SYSCTL_ADD_QUAD(rx_data_errs); 3610 CXGB_SYSCTL_ADD_QUAD(rx_sequence_errs); 3611 CXGB_SYSCTL_ADD_QUAD(rx_runt); 3612 CXGB_SYSCTL_ADD_QUAD(rx_jabber); 3613 CXGB_SYSCTL_ADD_QUAD(rx_short); 3614 CXGB_SYSCTL_ADD_QUAD(rx_too_long); 3615 CXGB_SYSCTL_ADD_QUAD(rx_mac_internal_errs); 3616 CXGB_SYSCTL_ADD_QUAD(rx_cong_drops); 3617 CXGB_SYSCTL_ADD_QUAD(rx_frames_64); 3618 CXGB_SYSCTL_ADD_QUAD(rx_frames_65_127); 3619 CXGB_SYSCTL_ADD_QUAD(rx_frames_128_255); 3620 CXGB_SYSCTL_ADD_QUAD(rx_frames_256_511); 3621 CXGB_SYSCTL_ADD_QUAD(rx_frames_512_1023); 3622 CXGB_SYSCTL_ADD_QUAD(rx_frames_1024_1518); 3623 CXGB_SYSCTL_ADD_QUAD(rx_frames_1519_max); 3624#undef CXGB_SYSCTL_ADD_QUAD 3625 3626#define CXGB_SYSCTL_ADD_ULONG(a) SYSCTL_ADD_ULONG(ctx, poidlist, OID_AUTO, #a, \ 3627 CTLFLAG_RD, &mstats->a, 0) 3628 CXGB_SYSCTL_ADD_ULONG(tx_fifo_parity_err); 3629 CXGB_SYSCTL_ADD_ULONG(rx_fifo_parity_err); 3630 CXGB_SYSCTL_ADD_ULONG(tx_fifo_urun); 3631 CXGB_SYSCTL_ADD_ULONG(rx_fifo_ovfl); 3632 CXGB_SYSCTL_ADD_ULONG(serdes_signal_loss); 3633 CXGB_SYSCTL_ADD_ULONG(xaui_pcs_ctc_err); 3634 CXGB_SYSCTL_ADD_ULONG(xaui_pcs_align_change); 3635 CXGB_SYSCTL_ADD_ULONG(num_toggled); 3636 CXGB_SYSCTL_ADD_ULONG(num_resets); 3637#undef CXGB_SYSCTL_ADD_ULONG 3638 } 3639} 3640 3641/** 3642 * t3_get_desc - dump an SGE descriptor for debugging purposes 3643 * @qs: the queue set 3644 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx) 3645 * @idx: the descriptor index in the queue 3646 * @data: where to dump the descriptor contents 3647 * 3648 * Dumps the contents of a HW descriptor of an SGE queue. Returns the 3649 * size of the descriptor. 3650 */ 3651int 3652t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, 3653 unsigned char *data) 3654{ 3655 if (qnum >= 6) 3656 return (EINVAL); 3657 3658 if (qnum < 3) { 3659 if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) 3660 return -EINVAL; 3661 memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); 3662 return sizeof(struct tx_desc); 3663 } 3664 3665 if (qnum == 3) { 3666 if (!qs->rspq.desc || idx >= qs->rspq.size) 3667 return (EINVAL); 3668 memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); 3669 return sizeof(struct rsp_desc); 3670 } 3671 3672 qnum -= 4; 3673 if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) 3674 return (EINVAL); 3675 memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); 3676 return sizeof(struct rx_desc); 3677}
|