1290650Shselasky/*- 2290650Shselasky * Copyright (c) 2015 Mellanox Technologies. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c 362313 2020-06-18 10:41:51Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include "en.h" 29290650Shselasky#include <machine/atomic.h> 30290650Shselasky 31301258Shselaskystatic inline bool 32301258Shselaskymlx5e_do_send_cqe(struct mlx5e_sq *sq) 33301258Shselasky{ 34301258Shselasky sq->cev_counter++; 35301258Shselasky /* interleave the CQEs */ 36301258Shselasky if (sq->cev_counter >= sq->cev_factor) { 37301258Shselasky sq->cev_counter = 0; 38301258Shselasky return (1); 39301258Shselasky } 40301258Shselasky return (0); 41301258Shselasky} 42301258Shselasky 43290650Shselaskyvoid 44301259Shselaskymlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt) 45290650Shselasky{ 46290650Shselasky u16 pi = sq->pc & sq->wq.sz_m1; 47290650Shselasky struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 48290650Shselasky 49290650Shselasky memset(&wqe->ctrl, 0, sizeof(wqe->ctrl)); 50290650Shselasky 51290650Shselasky wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP); 52290650Shselasky wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 53301258Shselasky if (mlx5e_do_send_cqe(sq)) 54301258Shselasky wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 55301258Shselasky else 56301258Shselasky wqe->ctrl.fm_ce_se = 0; 57290650Shselasky 58301259Shselasky /* Copy data for doorbell */ 59301259Shselasky memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 60301259Shselasky 61290650Shselasky sq->mbuf[pi].mbuf = NULL; 62290650Shselasky sq->mbuf[pi].num_bytes = 0; 63290650Shselasky sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 64290650Shselasky sq->pc += sq->mbuf[pi].num_wqebbs; 65290650Shselasky} 66290650Shselasky 67290650Shselasky#if (__FreeBSD_version >= 1100000) 68290650Shselaskystatic uint32_t mlx5e_hash_value; 69290650Shselasky 70290650Shselaskystatic void 71290650Shselaskymlx5e_hash_init(void *arg) 72290650Shselasky{ 73290650Shselasky mlx5e_hash_value = m_ether_tcpip_hash_init(); 74290650Shselasky} 75290650Shselasky 76290650Shselasky/* Make kernel call mlx5e_hash_init after the random stack finished initializing */ 77290650ShselaskySYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL); 78290650Shselasky#endif 79290650Shselasky 80290650Shselaskystatic struct mlx5e_sq * 81290650Shselaskymlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb) 82290650Shselasky{ 83290650Shselasky struct mlx5e_priv *priv = ifp->if_softc; 84321998Shselasky struct mlx5e_channel * volatile *ppch; 85321998Shselasky struct mlx5e_channel *pch; 86290650Shselasky u32 ch; 87290650Shselasky u32 tc; 88290650Shselasky 89321998Shselasky ppch = priv->channel; 90321998Shselasky 91290650Shselasky /* check if channels are successfully opened */ 92321998Shselasky if (unlikely(ppch == NULL)) 93290650Shselasky return (NULL); 94290650Shselasky 95290650Shselasky /* obtain VLAN information if present */ 96290650Shselasky if (mb->m_flags & M_VLANTAG) { 97290650Shselasky tc = (mb->m_pkthdr.ether_vtag >> 13); 98290650Shselasky if (tc >= priv->num_tc) 99290650Shselasky tc = priv->default_vlan_prio; 100290650Shselasky } else { 101290650Shselasky tc = priv->default_vlan_prio; 102290650Shselasky } 103290650Shselasky 104290650Shselasky ch = priv->params.num_channels; 105290650Shselasky 106290650Shselasky /* check if flowid is set */ 107290650Shselasky if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) { 108292195Shselasky#ifdef RSS 109292195Shselasky u32 temp; 110292195Shselasky 111292195Shselasky if (rss_hash2bucket(mb->m_pkthdr.flowid, 112292195Shselasky M_HASHTYPE_GET(mb), &temp) == 0) 113292195Shselasky ch = temp % ch; 114292195Shselasky else 115292195Shselasky#endif 116292195Shselasky ch = (mb->m_pkthdr.flowid % 128) % ch; 117290650Shselasky } else { 118290650Shselasky#if (__FreeBSD_version >= 1100000) 119290650Shselasky ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | 120290650Shselasky MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch; 121290650Shselasky#else 122290650Shselasky /* 123290650Shselasky * m_ether_tcpip_hash not present in stable, so just 124290650Shselasky * throw unhashed mbufs on queue 0 125290650Shselasky */ 126290650Shselasky ch = 0; 127290650Shselasky#endif 128290650Shselasky } 129290650Shselasky 130321998Shselasky /* check if channel is allocated and not stopped */ 131321998Shselasky pch = ppch[ch]; 132321998Shselasky if (likely(pch != NULL && pch->sq[tc].stopped == 0)) 133321998Shselasky return (&pch->sq[tc]); 134321998Shselasky return (NULL); 135290650Shselasky} 136290650Shselasky 137290650Shselaskystatic inline u16 138290650Shselaskymlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb) 139290650Shselasky{ 140337742Shselasky 141337742Shselasky switch(sq->min_inline_mode) { 142337742Shselasky case MLX5_INLINE_MODE_NONE: 143337742Shselasky /* 144337742Shselasky * When inline mode is NONE, we do not need to copy 145337742Shselasky * headers into WQEs, except when vlan tag framing is 146337742Shselasky * requested. Hardware might offload vlan tagging on 147337742Shselasky * transmit. This is a separate capability, which is 148337742Shselasky * known to be disabled on ConnectX-5 due to a hardware 149337742Shselasky * bug RM 931383. If vlan_inline_cap is not present and 150337742Shselasky * the packet has vlan tag, fall back to inlining. 151337742Shselasky */ 152337742Shselasky if ((mb->m_flags & M_VLANTAG) != 0 && 153337742Shselasky sq->vlan_inline_cap == 0) 154337742Shselasky break; 155337742Shselasky return (0); 156337742Shselasky case MLX5_INLINE_MODE_L2: 157337742Shselasky /* 158337742Shselasky * Due to hardware limitations, when trust mode is 159337742Shselasky * DSCP, the hardware may request MLX5_INLINE_MODE_L2 160337742Shselasky * while it really needs all L2 headers and the 4 first 161337742Shselasky * bytes of the IP header (which include the 162337742Shselasky * TOS/traffic-class). 163337742Shselasky * 164337742Shselasky * To avoid doing a firmware command for querying the 165337742Shselasky * trust state and parsing the mbuf for doing 166337742Shselasky * unnecessary checks (VLAN/eth_type) in the fast path, 167337742Shselasky * we are going for the worth case (22 Bytes) if 168337742Shselasky * the mb->m_pkthdr.len allows it. 169337742Shselasky */ 170337742Shselasky if (mb->m_pkthdr.len > ETHER_HDR_LEN + 171337742Shselasky ETHER_VLAN_ENCAP_LEN + 4) 172337742Shselasky return (MIN(sq->max_inline, ETHER_HDR_LEN + 173337742Shselasky ETHER_VLAN_ENCAP_LEN + 4)); 174337742Shselasky break; 175337742Shselasky } 176337742Shselasky return (MIN(sq->max_inline, mb->m_pkthdr.len)); 177290650Shselasky} 178290650Shselasky 179362307Shselasky/* 180362307Shselasky * This function parse IPv4 and IPv6 packets looking for TCP and UDP 181362307Shselasky * headers. 182362307Shselasky * 183362307Shselasky * The return value indicates the number of bytes from the beginning 184362307Shselasky * of the packet until the first byte after the TCP or UDP header. If 185362307Shselasky * this function returns zero, the parsing failed. 186362307Shselasky */ 187290650Shselaskystatic int 188362307Shselaskymlx5e_get_header_size(const struct mbuf *mb) 189290650Shselasky{ 190362307Shselasky const struct ether_vlan_header *eh; 191362307Shselasky const struct tcphdr *th; 192362307Shselasky const struct ip *ip; 193290650Shselasky int ip_hlen, tcp_hlen; 194362307Shselasky const struct ip6_hdr *ip6; 195290650Shselasky uint16_t eth_type; 196290650Shselasky int eth_hdr_len; 197290650Shselasky 198362307Shselasky eh = mtod(mb, const struct ether_vlan_header *); 199362310Shselasky if (unlikely(mb->m_len < ETHER_HDR_LEN)) 200290650Shselasky return (0); 201290650Shselasky if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 202362310Shselasky if (unlikely(mb->m_len < (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN))) 203362310Shselasky return (0); 204290650Shselasky eth_type = ntohs(eh->evl_proto); 205290650Shselasky eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 206290650Shselasky } else { 207290650Shselasky eth_type = ntohs(eh->evl_encap_proto); 208290650Shselasky eth_hdr_len = ETHER_HDR_LEN; 209290650Shselasky } 210290650Shselasky switch (eth_type) { 211290650Shselasky case ETHERTYPE_IP: 212362307Shselasky ip = (const struct ip *)(mb->m_data + eth_hdr_len); 213362310Shselasky if (unlikely(mb->m_len < eth_hdr_len + sizeof(*ip))) 214290650Shselasky return (0); 215290650Shselasky if (ip->ip_p != IPPROTO_TCP) 216290650Shselasky return (0); 217290650Shselasky ip_hlen = ip->ip_hl << 2; 218290650Shselasky eth_hdr_len += ip_hlen; 219290650Shselasky break; 220290650Shselasky case ETHERTYPE_IPV6: 221362307Shselasky ip6 = (const struct ip6_hdr *)(mb->m_data + eth_hdr_len); 222362310Shselasky if (unlikely(mb->m_len < eth_hdr_len + sizeof(*ip6))) 223290650Shselasky return (0); 224290650Shselasky if (ip6->ip6_nxt != IPPROTO_TCP) 225290650Shselasky return (0); 226290650Shselasky eth_hdr_len += sizeof(*ip6); 227290650Shselasky break; 228290650Shselasky default: 229290650Shselasky return (0); 230290650Shselasky } 231362313Shselasky if (unlikely(mb->m_len < eth_hdr_len + sizeof(*th))) { 232362313Shselasky const struct mbuf *m_th = mb->m_next; 233362313Shselasky if (unlikely(mb->m_len != eth_hdr_len || 234362313Shselasky m_th == NULL || m_th->m_len < sizeof(*th))) 235362313Shselasky return (0); 236362313Shselasky th = (const struct tcphdr *)(m_th->m_data); 237362313Shselasky } else { 238362313Shselasky th = (const struct tcphdr *)(mb->m_data + eth_hdr_len); 239362313Shselasky } 240290650Shselasky tcp_hlen = th->th_off << 2; 241290650Shselasky eth_hdr_len += tcp_hlen; 242362310Shselasky /* 243362310Shselasky * m_copydata() will be used on the remaining header which 244362310Shselasky * does not need to reside within the first m_len bytes of 245362310Shselasky * data: 246362310Shselasky */ 247362310Shselasky if (unlikely(mb->m_pkthdr.len < eth_hdr_len)) 248290650Shselasky return (0); 249290650Shselasky return (eth_hdr_len); 250290650Shselasky} 251290650Shselasky 252291184Shselasky/* 253291184Shselasky * The return value is not going back to the stack because of 254291184Shselasky * the drbr 255291184Shselasky */ 256290650Shselaskystatic int 257290650Shselaskymlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp) 258290650Shselasky{ 259290650Shselasky bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS]; 260290650Shselasky struct mlx5_wqe_data_seg *dseg; 261290650Shselasky struct mlx5e_tx_wqe *wqe; 262290650Shselasky struct ifnet *ifp; 263290650Shselasky int nsegs; 264290650Shselasky int err; 265290650Shselasky int x; 266290650Shselasky struct mbuf *mb = *mbp; 267290650Shselasky u16 ds_cnt; 268290650Shselasky u16 ihs; 269290650Shselasky u16 pi; 270290650Shselasky u8 opcode; 271290650Shselasky 272291184Shselasky /* 273291184Shselasky * Return ENOBUFS if the queue is full, this may trigger reinsertion 274291184Shselasky * of the mbuf into the drbr (see mlx5e_xmit_locked) 275291184Shselasky */ 276290650Shselasky if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) { 277359855Shselasky sq->stats.enobuf++; 278290650Shselasky return (ENOBUFS); 279290650Shselasky } 280290650Shselasky 281290650Shselasky /* Align SQ edge with NOPs to avoid WQE wrap around */ 282290650Shselasky pi = ((~sq->pc) & sq->wq.sz_m1); 283290650Shselasky if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) { 284291184Shselasky /* Send one multi NOP message instead of many */ 285301259Shselasky mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS); 286290650Shselasky pi = ((~sq->pc) & sq->wq.sz_m1); 287359855Shselasky if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) { 288359855Shselasky sq->stats.enobuf++; 289290650Shselasky return (ENOMEM); 290359855Shselasky } 291290650Shselasky } 292290650Shselasky 293290650Shselasky /* Setup local variables */ 294290650Shselasky pi = sq->pc & sq->wq.sz_m1; 295290650Shselasky wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 296306245Shselasky ifp = sq->ifp; 297290650Shselasky 298290650Shselasky memset(wqe, 0, sizeof(*wqe)); 299290650Shselasky 300291184Shselasky /* Send a copy of the frame to the BPF listener, if any */ 301290650Shselasky if (ifp != NULL && ifp->if_bpf != NULL) 302290650Shselasky ETHER_BPF_MTAP(ifp, mb); 303290650Shselasky 304290650Shselasky if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) { 305290650Shselasky wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM; 306290650Shselasky } 307290650Shselasky if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) { 308290650Shselasky wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM; 309290650Shselasky } 310291184Shselasky if (wqe->eth.cs_flags == 0) { 311290650Shselasky sq->stats.csum_offload_none++; 312290650Shselasky } 313290650Shselasky if (mb->m_pkthdr.csum_flags & CSUM_TSO) { 314290650Shselasky u32 payload_len; 315290650Shselasky u32 mss = mb->m_pkthdr.tso_segsz; 316290650Shselasky u32 num_pkts; 317290650Shselasky 318290650Shselasky wqe->eth.mss = cpu_to_be16(mss); 319290650Shselasky opcode = MLX5_OPCODE_LSO; 320290650Shselasky ihs = mlx5e_get_header_size(mb); 321290650Shselasky payload_len = mb->m_pkthdr.len - ihs; 322290650Shselasky if (payload_len == 0) 323290650Shselasky num_pkts = 1; 324290650Shselasky else 325290650Shselasky num_pkts = DIV_ROUND_UP(payload_len, mss); 326290650Shselasky sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs); 327290650Shselasky 328290650Shselasky sq->stats.tso_packets++; 329290650Shselasky sq->stats.tso_bytes += payload_len; 330290650Shselasky } else { 331290650Shselasky opcode = MLX5_OPCODE_SEND; 332290650Shselasky ihs = mlx5e_get_inline_hdr_size(sq, mb); 333290650Shselasky sq->mbuf[pi].num_bytes = max_t (unsigned int, 334290650Shselasky mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); 335290650Shselasky } 336337742Shselasky if (ihs == 0) { 337337742Shselasky if ((mb->m_flags & M_VLANTAG) != 0) { 338337742Shselasky wqe->eth.vlan_cmd = htons(0x8000); /* bit 0 CVLAN */ 339337742Shselasky wqe->eth.vlan_hdr = htons(mb->m_pkthdr.ether_vtag); 340337742Shselasky } else { 341337742Shselasky wqe->eth.inline_hdr_sz = 0; 342337742Shselasky } 343337742Shselasky } else { 344337742Shselasky if ((mb->m_flags & M_VLANTAG) != 0) { 345337742Shselasky struct ether_vlan_header *eh = (struct ether_vlan_header 346337742Shselasky *)wqe->eth.inline_hdr_start; 347291184Shselasky 348337742Shselasky /* Range checks */ 349337742Shselasky if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN)) 350337742Shselasky ihs = (MLX5E_MAX_TX_INLINE - 351337742Shselasky ETHER_VLAN_ENCAP_LEN); 352337742Shselasky else if (ihs < ETHER_HDR_LEN) { 353337742Shselasky err = EINVAL; 354337742Shselasky goto tx_drop; 355337742Shselasky } 356337742Shselasky m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); 357337742Shselasky m_adj(mb, ETHER_HDR_LEN); 358337742Shselasky /* Insert 4 bytes VLAN tag into data stream */ 359337742Shselasky eh->evl_proto = eh->evl_encap_proto; 360337742Shselasky eh->evl_encap_proto = htons(ETHERTYPE_VLAN); 361337742Shselasky eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); 362337742Shselasky /* Copy rest of header data, if any */ 363337742Shselasky m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 364337742Shselasky 1)); 365337742Shselasky m_adj(mb, ihs - ETHER_HDR_LEN); 366337742Shselasky /* Extend header by 4 bytes */ 367337742Shselasky ihs += ETHER_VLAN_ENCAP_LEN; 368337742Shselasky } else { 369337742Shselasky m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); 370337742Shselasky m_adj(mb, ihs); 371290650Shselasky } 372337742Shselasky wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); 373290650Shselasky } 374290650Shselasky 375290650Shselasky ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; 376337742Shselasky if (ihs > sizeof(wqe->eth.inline_hdr_start)) { 377290650Shselasky ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start), 378290650Shselasky MLX5_SEND_WQE_DS); 379290650Shselasky } 380290650Shselasky dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt; 381290650Shselasky 382291184Shselasky /* Trim off empty mbufs */ 383290650Shselasky while (mb->m_len == 0) { 384290650Shselasky mb = m_free(mb); 385291184Shselasky /* Check if all data has been inlined */ 386290650Shselasky if (mb == NULL) 387290650Shselasky goto skip_dma; 388290650Shselasky } 389290650Shselasky 390290650Shselasky err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, 391290650Shselasky mb, segs, &nsegs, BUS_DMA_NOWAIT); 392290650Shselasky if (err == EFBIG) { 393291184Shselasky /* 394291184Shselasky * Update *mbp before defrag in case it was trimmed in the 395291184Shselasky * loop above 396291184Shselasky */ 397290650Shselasky *mbp = mb; 398290650Shselasky /* Update statistics */ 399290650Shselasky sq->stats.defragged++; 400290650Shselasky /* Too many mbuf fragments */ 401290650Shselasky mb = m_defrag(*mbp, M_NOWAIT); 402290650Shselasky if (mb == NULL) { 403290650Shselasky mb = *mbp; 404290650Shselasky goto tx_drop; 405290650Shselasky } 406290650Shselasky /* Try again */ 407290650Shselasky err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, 408290650Shselasky mb, segs, &nsegs, BUS_DMA_NOWAIT); 409290650Shselasky } 410291184Shselasky /* Catch errors */ 411306254Shselasky if (err != 0) 412290650Shselasky goto tx_drop; 413290650Shselasky 414290650Shselasky for (x = 0; x != nsegs; x++) { 415290650Shselasky if (segs[x].ds_len == 0) 416290650Shselasky continue; 417290650Shselasky dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr); 418290650Shselasky dseg->lkey = sq->mkey_be; 419290650Shselasky dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len); 420290650Shselasky dseg++; 421290650Shselasky } 422290650Shselaskyskip_dma: 423290650Shselasky ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl)); 424290650Shselasky 425290650Shselasky wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); 426290650Shselasky wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 427301258Shselasky if (mlx5e_do_send_cqe(sq)) 428301258Shselasky wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 429301258Shselasky else 430301258Shselasky wqe->ctrl.fm_ce_se = 0; 431290650Shselasky 432301259Shselasky /* Copy data for doorbell */ 433301259Shselasky memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 434301259Shselasky 435291184Shselasky /* Store pointer to mbuf */ 436290650Shselasky sq->mbuf[pi].mbuf = mb; 437290650Shselasky sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 438290650Shselasky sq->pc += sq->mbuf[pi].num_wqebbs; 439290650Shselasky 440291184Shselasky /* Make sure all mbuf data is written to RAM */ 441290650Shselasky if (mb != NULL) 442290650Shselasky bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map, BUS_DMASYNC_PREWRITE); 443290650Shselasky 444290650Shselasky sq->stats.packets++; 445306254Shselasky *mbp = NULL; /* safety clear */ 446290650Shselasky return (0); 447290650Shselasky 448290650Shselaskytx_drop: 449290650Shselasky sq->stats.dropped++; 450290650Shselasky *mbp = NULL; 451290650Shselasky m_freem(mb); 452290650Shselasky return err; 453290650Shselasky} 454290650Shselasky 455290650Shselaskystatic void 456290650Shselaskymlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget) 457290650Shselasky{ 458290650Shselasky u16 sqcc; 459290650Shselasky 460290650Shselasky /* 461290650Shselasky * sq->cc must be updated only after mlx5_cqwq_update_db_record(), 462290650Shselasky * otherwise a cq overrun may occur 463290650Shselasky */ 464290650Shselasky sqcc = sq->cc; 465290650Shselasky 466301258Shselasky while (budget > 0) { 467290650Shselasky struct mlx5_cqe64 *cqe; 468290650Shselasky struct mbuf *mb; 469301258Shselasky u16 x; 470290650Shselasky u16 ci; 471290650Shselasky 472290650Shselasky cqe = mlx5e_get_cqe(&sq->cq); 473290650Shselasky if (!cqe) 474290650Shselasky break; 475290650Shselasky 476293155Shselasky mlx5_cqwq_pop(&sq->cq.wq); 477293155Shselasky 478301258Shselasky /* update budget according to the event factor */ 479301258Shselasky budget -= sq->cev_factor; 480290650Shselasky 481301258Shselasky for (x = 0; x != sq->cev_factor; x++) { 482301258Shselasky ci = sqcc & sq->wq.sz_m1; 483301258Shselasky mb = sq->mbuf[ci].mbuf; 484301258Shselasky sq->mbuf[ci].mbuf = NULL; /* Safety clear */ 485301258Shselasky 486301258Shselasky if (mb == NULL) { 487301258Shselasky if (sq->mbuf[ci].num_bytes == 0) { 488301258Shselasky /* NOP */ 489301258Shselasky sq->stats.nop++; 490301258Shselasky } 491301258Shselasky } else { 492301258Shselasky bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map, 493301258Shselasky BUS_DMASYNC_POSTWRITE); 494301258Shselasky bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map); 495301258Shselasky 496301258Shselasky /* Free transmitted mbuf */ 497301258Shselasky m_freem(mb); 498290650Shselasky } 499301258Shselasky sqcc += sq->mbuf[ci].num_wqebbs; 500290650Shselasky } 501290650Shselasky } 502290650Shselasky 503290650Shselasky mlx5_cqwq_update_db_record(&sq->cq.wq); 504290650Shselasky 505291184Shselasky /* Ensure cq space is freed before enabling more cqes */ 506290650Shselasky wmb(); 507290650Shselasky 508290650Shselasky sq->cc = sqcc; 509290650Shselasky 510322000Shselasky if (sq->sq_tq != NULL && 511322000Shselasky atomic_cmpset_int(&sq->queue_state, MLX5E_SQ_FULL, MLX5E_SQ_READY)) 512290650Shselasky taskqueue_enqueue(sq->sq_tq, &sq->sq_task); 513290650Shselasky} 514290650Shselasky 515290650Shselaskystatic int 516290650Shselaskymlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb) 517290650Shselasky{ 518291184Shselasky struct mbuf *next; 519290650Shselasky int err = 0; 520290650Shselasky 521321998Shselasky if (likely(mb != NULL)) { 522291184Shselasky /* 523291184Shselasky * If we can't insert mbuf into drbr, try to xmit anyway. 524291184Shselasky * We keep the error we got so we could return that after xmit. 525291184Shselasky */ 526291184Shselasky err = drbr_enqueue(ifp, sq->br, mb); 527321998Shselasky } 528290650Shselasky 529321998Shselasky /* 530321998Shselasky * Check if the network interface is closed or if the SQ is 531321998Shselasky * being stopped: 532321998Shselasky */ 533321998Shselasky if (unlikely((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 534321998Shselasky sq->stopped != 0)) 535321998Shselasky return (err); 536321998Shselasky 537291184Shselasky /* Process the queue */ 538291184Shselasky while ((next = drbr_peek(ifp, sq->br)) != NULL) { 539291184Shselasky if (mlx5e_sq_xmit(sq, &next) != 0) { 540338551Shselasky if (next != NULL) { 541291184Shselasky drbr_putback(ifp, sq->br, next); 542291184Shselasky atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_FULL); 543338551Shselasky break; 544291184Shselasky } 545291184Shselasky } 546291184Shselasky drbr_advance(ifp, sq->br); 547291184Shselasky } 548301259Shselasky /* Check if we need to write the doorbell */ 549301259Shselasky if (likely(sq->doorbell.d64 != 0)) { 550301259Shselasky mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0); 551301259Shselasky sq->doorbell.d64 = 0; 552301259Shselasky } 553301258Shselasky /* 554301258Shselasky * Check if we need to start the event timer which flushes the 555301258Shselasky * transmit ring on timeout: 556301258Shselasky */ 557301258Shselasky if (unlikely(sq->cev_next_state == MLX5E_CEV_STATE_INITIAL && 558301258Shselasky sq->cev_factor != 1)) { 559301258Shselasky /* start the timer */ 560301258Shselasky mlx5e_sq_cev_timeout(sq); 561301258Shselasky } else { 562301258Shselasky /* don't send NOPs yet */ 563301258Shselasky sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS; 564301258Shselasky } 565291184Shselasky return (err); 566290650Shselasky} 567290650Shselasky 568322000Shselaskystatic int 569322000Shselaskymlx5e_xmit_locked_no_br(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb) 570322000Shselasky{ 571322000Shselasky int err = 0; 572322000Shselasky 573322000Shselasky if (unlikely((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 574322000Shselasky sq->stopped != 0)) { 575322000Shselasky m_freem(mb); 576322000Shselasky return (ENETDOWN); 577322000Shselasky } 578322000Shselasky 579322000Shselasky /* Do transmit */ 580322000Shselasky if (mlx5e_sq_xmit(sq, &mb) != 0) { 581322000Shselasky /* NOTE: m_freem() is NULL safe */ 582322000Shselasky m_freem(mb); 583322000Shselasky err = ENOBUFS; 584322000Shselasky } 585322000Shselasky 586322000Shselasky /* Check if we need to write the doorbell */ 587322000Shselasky if (likely(sq->doorbell.d64 != 0)) { 588322000Shselasky mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0); 589322000Shselasky sq->doorbell.d64 = 0; 590322000Shselasky } 591322000Shselasky 592322000Shselasky /* 593322000Shselasky * Check if we need to start the event timer which flushes the 594322000Shselasky * transmit ring on timeout: 595322000Shselasky */ 596322000Shselasky if (unlikely(sq->cev_next_state == MLX5E_CEV_STATE_INITIAL && 597322000Shselasky sq->cev_factor != 1)) { 598322000Shselasky /* start the timer */ 599322000Shselasky mlx5e_sq_cev_timeout(sq); 600322000Shselasky } else { 601322000Shselasky /* don't send NOPs yet */ 602322000Shselasky sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS; 603322000Shselasky } 604322000Shselasky return (err); 605322000Shselasky} 606322000Shselasky 607290650Shselaskyint 608290650Shselaskymlx5e_xmit(struct ifnet *ifp, struct mbuf *mb) 609290650Shselasky{ 610290650Shselasky struct mlx5e_sq *sq; 611290650Shselasky int ret; 612290650Shselasky 613290650Shselasky sq = mlx5e_select_queue(ifp, mb); 614290650Shselasky if (unlikely(sq == NULL)) { 615291184Shselasky /* Invalid send queue */ 616290650Shselasky m_freem(mb); 617290650Shselasky return (ENXIO); 618290650Shselasky } 619322000Shselasky 620322000Shselasky if (unlikely(sq->br == NULL)) { 621322000Shselasky /* rate limited traffic */ 622322000Shselasky mtx_lock(&sq->lock); 623322000Shselasky ret = mlx5e_xmit_locked_no_br(ifp, sq, mb); 624322000Shselasky mtx_unlock(&sq->lock); 625322000Shselasky } else if (mtx_trylock(&sq->lock)) { 626290650Shselasky ret = mlx5e_xmit_locked(ifp, sq, mb); 627290650Shselasky mtx_unlock(&sq->lock); 628290650Shselasky } else { 629290650Shselasky ret = drbr_enqueue(ifp, sq->br, mb); 630291184Shselasky taskqueue_enqueue(sq->sq_tq, &sq->sq_task); 631290650Shselasky } 632290650Shselasky 633290650Shselasky return (ret); 634290650Shselasky} 635290650Shselasky 636290650Shselaskyvoid 637290650Shselaskymlx5e_tx_cq_comp(struct mlx5_core_cq *mcq) 638290650Shselasky{ 639290650Shselasky struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq); 640290650Shselasky 641290650Shselasky mtx_lock(&sq->comp_lock); 642290650Shselasky mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX); 643324523Shselasky mlx5e_cq_arm(&sq->cq, MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock)); 644290650Shselasky mtx_unlock(&sq->comp_lock); 645290650Shselasky} 646290650Shselasky 647290650Shselaskyvoid 648290650Shselaskymlx5e_tx_que(void *context, int pending) 649290650Shselasky{ 650290650Shselasky struct mlx5e_sq *sq = context; 651306245Shselasky struct ifnet *ifp = sq->ifp; 652290650Shselasky 653290650Shselasky if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 654290650Shselasky mtx_lock(&sq->lock); 655290650Shselasky if (!drbr_empty(ifp, sq->br)) 656290650Shselasky mlx5e_xmit_locked(ifp, sq, NULL); 657290650Shselasky mtx_unlock(&sq->lock); 658290650Shselasky } 659290650Shselasky} 660