mlx4_en_tx.c revision 297966
1219820Sjeff/* 2272027Shselasky * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff * 32219820Sjeff */ 33219820Sjeff 34273135Shselasky#include <linux/page.h> 35219820Sjeff#include <linux/mlx4/cq.h> 36272027Shselasky#include <linux/slab.h> 37219820Sjeff#include <linux/mlx4/qp.h> 38272027Shselasky#include <linux/if_vlan.h> 39219820Sjeff#include <linux/vmalloc.h> 40272027Shselasky#include <linux/moduleparam.h> 41219820Sjeff 42219820Sjeff#include <netinet/in_systm.h> 43219820Sjeff#include <netinet/in.h> 44219820Sjeff#include <netinet/if_ether.h> 45219820Sjeff#include <netinet/ip.h> 46219820Sjeff#include <netinet/ip6.h> 47219820Sjeff#include <netinet/tcp.h> 48219820Sjeff#include <netinet/tcp_lro.h> 49219820Sjeff#include <netinet/udp.h> 50219820Sjeff 51272027Shselasky#include "mlx4_en.h" 52272027Shselasky 53219820Sjeffenum { 54219820Sjeff MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */ 55219820Sjeff MAX_BF = 256, 56272027Shselasky MIN_PKT_LEN = 17, 57219820Sjeff}; 58219820Sjeff 59272027Shselaskystatic int inline_thold __read_mostly = MAX_INLINE; 60219820Sjeff 61272027Shselaskymodule_param_named(inline_thold, inline_thold, uint, 0444); 62272027ShselaskyMODULE_PARM_DESC(inline_thold, "threshold for using inline data"); 63219820Sjeff 64219820Sjeffint mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, 65272027Shselasky struct mlx4_en_tx_ring **pring, u32 size, 66272027Shselasky u16 stride, int node, int queue_idx) 67219820Sjeff{ 68219820Sjeff struct mlx4_en_dev *mdev = priv->mdev; 69272027Shselasky struct mlx4_en_tx_ring *ring; 70291699Shselasky uint32_t x; 71219820Sjeff int tmp; 72219820Sjeff int err; 73219820Sjeff 74272027Shselasky ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node); 75272027Shselasky if (!ring) { 76272027Shselasky ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL); 77272027Shselasky if (!ring) { 78272027Shselasky en_err(priv, "Failed allocating TX ring\n"); 79272027Shselasky return -ENOMEM; 80272027Shselasky } 81272027Shselasky } 82272027Shselasky 83291699Shselasky /* Create DMA descriptor TAG */ 84291699Shselasky if ((err = -bus_dma_tag_create( 85291699Shselasky bus_get_dma_tag(mdev->pdev->dev.bsddev), 86291699Shselasky 1, /* any alignment */ 87291699Shselasky 0, /* no boundary */ 88291699Shselasky BUS_SPACE_MAXADDR, /* lowaddr */ 89291699Shselasky BUS_SPACE_MAXADDR, /* highaddr */ 90291699Shselasky NULL, NULL, /* filter, filterarg */ 91291699Shselasky MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */ 92291699Shselasky MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */ 93291699Shselasky MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */ 94291699Shselasky 0, /* flags */ 95291699Shselasky NULL, NULL, /* lockfunc, lockfuncarg */ 96291699Shselasky &ring->dma_tag))) 97291699Shselasky goto done; 98291699Shselasky 99219820Sjeff ring->size = size; 100219820Sjeff ring->size_mask = size - 1; 101219820Sjeff ring->stride = stride; 102291699Shselasky ring->inline_thold = MAX(MIN_PKT_LEN, MIN(inline_thold, MAX_INLINE)); 103219820Sjeff mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF); 104219820Sjeff mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF); 105219820Sjeff 106219820Sjeff /* Allocate the buf ring */ 107219820Sjeff ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF, 108272027Shselasky M_WAITOK, &ring->tx_lock.m); 109219820Sjeff if (ring->br == NULL) { 110219820Sjeff en_err(priv, "Failed allocating tx_info ring\n"); 111291699Shselasky err = -ENOMEM; 112291699Shselasky goto err_free_dma_tag; 113219820Sjeff } 114219820Sjeff 115219820Sjeff tmp = size * sizeof(struct mlx4_en_tx_info); 116291699Shselasky ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node); 117219820Sjeff if (!ring->tx_info) { 118291699Shselasky ring->tx_info = kzalloc(tmp, GFP_KERNEL); 119272027Shselasky if (!ring->tx_info) { 120272027Shselasky err = -ENOMEM; 121272027Shselasky goto err_ring; 122272027Shselasky } 123219820Sjeff } 124272027Shselasky 125291699Shselasky /* Create DMA descriptor MAPs */ 126291699Shselasky for (x = 0; x != size; x++) { 127291699Shselasky err = -bus_dmamap_create(ring->dma_tag, 0, 128291699Shselasky &ring->tx_info[x].dma_map); 129291699Shselasky if (err != 0) { 130291699Shselasky while (x--) { 131291699Shselasky bus_dmamap_destroy(ring->dma_tag, 132291699Shselasky ring->tx_info[x].dma_map); 133291699Shselasky } 134291699Shselasky goto err_info; 135291699Shselasky } 136291699Shselasky } 137291699Shselasky 138219820Sjeff en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", 139219820Sjeff ring->tx_info, tmp); 140219820Sjeff 141219820Sjeff ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); 142219820Sjeff 143272027Shselasky /* Allocate HW buffers on provided NUMA node */ 144219820Sjeff err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 145219820Sjeff 2 * PAGE_SIZE); 146219820Sjeff if (err) { 147219820Sjeff en_err(priv, "Failed allocating hwq resources\n"); 148291699Shselasky goto err_dma_map; 149219820Sjeff } 150219820Sjeff 151219820Sjeff err = mlx4_en_map_buffer(&ring->wqres.buf); 152219820Sjeff if (err) { 153219820Sjeff en_err(priv, "Failed to map TX buffer\n"); 154219820Sjeff goto err_hwq_res; 155219820Sjeff } 156219820Sjeff 157219820Sjeff ring->buf = ring->wqres.buf.direct.buf; 158219820Sjeff 159219820Sjeff en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d " 160219820Sjeff "buf_size:%d dma:%llx\n", ring, ring->buf, ring->size, 161219820Sjeff ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map); 162219820Sjeff 163272027Shselasky err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn, 164272027Shselasky MLX4_RESERVE_BF_QP); 165219820Sjeff if (err) { 166272027Shselasky en_err(priv, "failed reserving qp for TX ring\n"); 167219820Sjeff goto err_map; 168219820Sjeff } 169219820Sjeff 170219820Sjeff err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp); 171219820Sjeff if (err) { 172219820Sjeff en_err(priv, "Failed allocating qp %d\n", ring->qpn); 173219820Sjeff goto err_reserve; 174219820Sjeff } 175219820Sjeff ring->qp.event = mlx4_en_sqp_event; 176219820Sjeff 177272027Shselasky err = mlx4_bf_alloc(mdev->dev, &ring->bf, node); 178219820Sjeff if (err) { 179272027Shselasky en_dbg(DRV, priv, "working without blueflame (%d)", err); 180219820Sjeff ring->bf.uar = &mdev->priv_uar; 181219820Sjeff ring->bf.uar->map = mdev->uar_map; 182219820Sjeff ring->bf_enabled = false; 183219820Sjeff } else 184219820Sjeff ring->bf_enabled = true; 185272027Shselasky ring->queue_index = queue_idx; 186272027Shselasky if (queue_idx < priv->num_tx_rings_p_up ) 187272027Shselasky CPU_SET(queue_idx, &ring->affinity_mask); 188219820Sjeff 189272027Shselasky *pring = ring; 190219820Sjeff return 0; 191219820Sjeff 192219820Sjefferr_reserve: 193219820Sjeff mlx4_qp_release_range(mdev->dev, ring->qpn, 1); 194219820Sjefferr_map: 195219820Sjeff mlx4_en_unmap_buffer(&ring->wqres.buf); 196219820Sjefferr_hwq_res: 197219820Sjeff mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 198291699Shselaskyerr_dma_map: 199291699Shselasky for (x = 0; x != size; x++) 200291699Shselasky bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); 201272027Shselaskyerr_info: 202272027Shselasky vfree(ring->tx_info); 203272027Shselaskyerr_ring: 204219820Sjeff buf_ring_free(ring->br, M_DEVBUF); 205291699Shselaskyerr_free_dma_tag: 206291699Shselasky bus_dma_tag_destroy(ring->dma_tag); 207291699Shselaskydone: 208272027Shselasky kfree(ring); 209219820Sjeff return err; 210219820Sjeff} 211219820Sjeff 212219820Sjeffvoid mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, 213272027Shselasky struct mlx4_en_tx_ring **pring) 214219820Sjeff{ 215219820Sjeff struct mlx4_en_dev *mdev = priv->mdev; 216272027Shselasky struct mlx4_en_tx_ring *ring = *pring; 217291699Shselasky uint32_t x; 218219820Sjeff en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); 219219820Sjeff 220219820Sjeff buf_ring_free(ring->br, M_DEVBUF); 221219820Sjeff if (ring->bf_enabled) 222219820Sjeff mlx4_bf_free(mdev->dev, &ring->bf); 223219820Sjeff mlx4_qp_remove(mdev->dev, &ring->qp); 224219820Sjeff mlx4_qp_free(mdev->dev, &ring->qp); 225272027Shselasky mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1); 226219820Sjeff mlx4_en_unmap_buffer(&ring->wqres.buf); 227219820Sjeff mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 228291699Shselasky for (x = 0; x != ring->size; x++) 229291699Shselasky bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); 230272027Shselasky vfree(ring->tx_info); 231219820Sjeff mtx_destroy(&ring->tx_lock.m); 232219820Sjeff mtx_destroy(&ring->comp_lock.m); 233291699Shselasky bus_dma_tag_destroy(ring->dma_tag); 234272027Shselasky kfree(ring); 235272027Shselasky *pring = NULL; 236219820Sjeff} 237219820Sjeff 238219820Sjeffint mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, 239219820Sjeff struct mlx4_en_tx_ring *ring, 240272027Shselasky int cq, int user_prio) 241219820Sjeff{ 242219820Sjeff struct mlx4_en_dev *mdev = priv->mdev; 243219820Sjeff int err; 244219820Sjeff 245219820Sjeff ring->cqn = cq; 246219820Sjeff ring->prod = 0; 247219820Sjeff ring->cons = 0xffffffff; 248219820Sjeff ring->last_nr_txbb = 1; 249219820Sjeff ring->poll_cnt = 0; 250219820Sjeff ring->blocked = 0; 251219820Sjeff memset(ring->buf, 0, ring->buf_size); 252219820Sjeff 253219820Sjeff ring->qp_state = MLX4_QP_STATE_RST; 254272027Shselasky ring->doorbell_qpn = ring->qp.qpn << 8; 255219820Sjeff 256219820Sjeff mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn, 257272027Shselasky ring->cqn, user_prio, &ring->context); 258219820Sjeff if (ring->bf_enabled) 259219820Sjeff ring->context.usr_page = cpu_to_be32(ring->bf.uar->index); 260219820Sjeff 261219820Sjeff err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context, 262219820Sjeff &ring->qp, &ring->qp_state); 263219820Sjeff return err; 264219820Sjeff} 265219820Sjeff 266219820Sjeffvoid mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, 267219820Sjeff struct mlx4_en_tx_ring *ring) 268219820Sjeff{ 269219820Sjeff struct mlx4_en_dev *mdev = priv->mdev; 270219820Sjeff 271219820Sjeff mlx4_qp_modify(mdev->dev, NULL, ring->qp_state, 272219820Sjeff MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); 273219820Sjeff} 274219820Sjeff 275291699Shselaskystatic volatile struct mlx4_wqe_data_seg * 276291699Shselaskymlx4_en_store_inline_lso_data(volatile struct mlx4_wqe_data_seg *dseg, 277291699Shselasky struct mbuf *mb, int len, __be32 owner_bit) 278272027Shselasky{ 279291699Shselasky uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); 280219820Sjeff 281291699Shselasky /* copy data into place */ 282291699Shselasky m_copydata(mb, 0, len, inl + 4); 283291699Shselasky dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); 284291699Shselasky return (dseg); 285272027Shselasky} 286272027Shselasky 287291699Shselaskystatic void 288291699Shselaskymlx4_en_store_inline_lso_header(volatile struct mlx4_wqe_data_seg *dseg, 289291699Shselasky int len, __be32 owner_bit) 290219820Sjeff{ 291291699Shselasky} 292291699Shselasky 293291699Shselaskystatic void 294291699Shselaskymlx4_en_stamp_wqe(struct mlx4_en_priv *priv, 295291699Shselasky struct mlx4_en_tx_ring *ring, u32 index, u8 owner) 296291699Shselasky{ 297219820Sjeff struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; 298291694Shselasky struct mlx4_en_tx_desc *tx_desc = (struct mlx4_en_tx_desc *) 299291699Shselasky (ring->buf + (index * TXBB_SIZE)); 300291699Shselasky volatile __be32 *ptr = (__be32 *)tx_desc; 301291699Shselasky const __be32 stamp = cpu_to_be32(STAMP_VAL | 302291699Shselasky ((u32)owner << STAMP_SHIFT)); 303291699Shselasky u32 i; 304219820Sjeff 305291699Shselasky /* Stamp the freed descriptor */ 306291699Shselasky for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { 307291699Shselasky *ptr = stamp; 308291699Shselasky ptr += STAMP_DWORDS; 309291699Shselasky } 310291699Shselasky} 311272027Shselasky 312291699Shselaskystatic u32 313291699Shselaskymlx4_en_free_tx_desc(struct mlx4_en_priv *priv, 314291699Shselasky struct mlx4_en_tx_ring *ring, u32 index) 315291699Shselasky{ 316291699Shselasky struct mlx4_en_tx_info *tx_info; 317291699Shselasky struct mbuf *mb; 318272027Shselasky 319291699Shselasky tx_info = &ring->tx_info[index]; 320291699Shselasky mb = tx_info->mb; 321272027Shselasky 322291699Shselasky if (mb == NULL) 323291699Shselasky goto done; 324291699Shselasky 325291699Shselasky bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, 326291699Shselasky BUS_DMASYNC_POSTWRITE); 327291699Shselasky bus_dmamap_unload(ring->dma_tag, tx_info->dma_map); 328291699Shselasky 329272027Shselasky m_freem(mb); 330291699Shselaskydone: 331291699Shselasky return (tx_info->nr_txbb); 332219820Sjeff} 333219820Sjeff 334219820Sjeffint mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) 335219820Sjeff{ 336219820Sjeff struct mlx4_en_priv *priv = netdev_priv(dev); 337219820Sjeff int cnt = 0; 338219820Sjeff 339219820Sjeff /* Skip last polled descriptor */ 340219820Sjeff ring->cons += ring->last_nr_txbb; 341219820Sjeff en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n", 342219820Sjeff ring->cons, ring->prod); 343219820Sjeff 344219820Sjeff if ((u32) (ring->prod - ring->cons) > ring->size) { 345272027Shselasky en_warn(priv, "Tx consumer passed producer!\n"); 346219820Sjeff return 0; 347219820Sjeff } 348219820Sjeff 349219820Sjeff while (ring->cons != ring->prod) { 350219820Sjeff ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, 351291699Shselasky ring->cons & ring->size_mask); 352219820Sjeff ring->cons += ring->last_nr_txbb; 353219820Sjeff cnt++; 354219820Sjeff } 355219820Sjeff 356219820Sjeff if (cnt) 357219820Sjeff en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt); 358219820Sjeff 359219820Sjeff return cnt; 360219820Sjeff} 361219820Sjeff 362291699Shselaskystatic bool 363291699Shselaskymlx4_en_tx_ring_is_full(struct mlx4_en_tx_ring *ring) 364291699Shselasky{ 365291699Shselasky int wqs; 366291699Shselasky wqs = ring->size - (ring->prod - ring->cons); 367291699Shselasky return (wqs < (HEADROOM + (2 * MLX4_EN_TX_WQE_MAX_WQEBBS))); 368291699Shselasky} 369291699Shselasky 370272027Shselaskystatic int mlx4_en_process_tx_cq(struct net_device *dev, 371272027Shselasky struct mlx4_en_cq *cq) 372219820Sjeff{ 373219820Sjeff struct mlx4_en_priv *priv = netdev_priv(dev); 374219820Sjeff struct mlx4_cq *mcq = &cq->mcq; 375272027Shselasky struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; 376272027Shselasky struct mlx4_cqe *cqe; 377219820Sjeff u16 index; 378272027Shselasky u16 new_index, ring_index, stamp_index; 379219820Sjeff u32 txbbs_skipped = 0; 380272027Shselasky u32 txbbs_stamp = 0; 381272027Shselasky u32 cons_index = mcq->cons_index; 382272027Shselasky int size = cq->size; 383272027Shselasky u32 size_mask = ring->size_mask; 384272027Shselasky struct mlx4_cqe *buf = cq->buf; 385272027Shselasky int factor = priv->cqe_factor; 386219820Sjeff 387219820Sjeff if (!priv->port_up) 388272027Shselasky return 0; 389219820Sjeff 390272027Shselasky index = cons_index & size_mask; 391272027Shselasky cqe = &buf[(index << factor) + factor]; 392272027Shselasky ring_index = ring->cons & size_mask; 393272027Shselasky stamp_index = ring_index; 394272027Shselasky 395272027Shselasky /* Process all completed CQEs */ 396272027Shselasky while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, 397272027Shselasky cons_index & size)) { 398272027Shselasky /* 399272027Shselasky * make sure we read the CQE after we read the 400272027Shselasky * ownership bit 401272027Shselasky */ 402272027Shselasky rmb(); 403272027Shselasky 404272027Shselasky if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == 405272027Shselasky MLX4_CQE_OPCODE_ERROR)) { 406272027Shselasky en_err(priv, "CQE completed in error - vendor syndrom: 0x%x syndrom: 0x%x\n", 407272027Shselasky ((struct mlx4_err_cqe *)cqe)-> 408272027Shselasky vendor_err_syndrome, 409272027Shselasky ((struct mlx4_err_cqe *)cqe)->syndrome); 410272027Shselasky } 411272027Shselasky 412272027Shselasky /* Skip over last polled CQE */ 413272027Shselasky new_index = be16_to_cpu(cqe->wqe_index) & size_mask; 414272027Shselasky 415219820Sjeff do { 416219820Sjeff txbbs_skipped += ring->last_nr_txbb; 417272027Shselasky ring_index = (ring_index + ring->last_nr_txbb) & size_mask; 418272027Shselasky /* free next descriptor */ 419219820Sjeff ring->last_nr_txbb = mlx4_en_free_tx_desc( 420291699Shselasky priv, ring, ring_index); 421272027Shselasky mlx4_en_stamp_wqe(priv, ring, stamp_index, 422272027Shselasky !!((ring->cons + txbbs_stamp) & 423272027Shselasky ring->size)); 424272027Shselasky stamp_index = ring_index; 425272027Shselasky txbbs_stamp = txbbs_skipped; 426272027Shselasky } while (ring_index != new_index); 427219820Sjeff 428272027Shselasky ++cons_index; 429272027Shselasky index = cons_index & size_mask; 430272027Shselasky cqe = &buf[(index << factor) + factor]; 431272027Shselasky } 432219820Sjeff 433219820Sjeff 434219820Sjeff /* 435219820Sjeff * To prevent CQ overflow we first update CQ consumer and only then 436219820Sjeff * the ring consumer. 437219820Sjeff */ 438272027Shselasky mcq->cons_index = cons_index; 439219820Sjeff mlx4_cq_set_ci(mcq); 440219820Sjeff wmb(); 441219820Sjeff ring->cons += txbbs_skipped; 442219820Sjeff 443272027Shselasky /* Wakeup Tx queue if it was stopped and ring is not full */ 444291699Shselasky if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) { 445272027Shselasky ring->blocked = 0; 446272027Shselasky if (atomic_fetchadd_int(&priv->blocked, -1) == 1) 447272027Shselasky atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE); 448272027Shselasky ring->wake_queue++; 449272027Shselasky priv->port_stats.wake_queue++; 450219820Sjeff } 451291699Shselasky return (0); 452219820Sjeff} 453219820Sjeff 454219820Sjeffvoid mlx4_en_tx_irq(struct mlx4_cq *mcq) 455219820Sjeff{ 456219820Sjeff struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); 457219820Sjeff struct mlx4_en_priv *priv = netdev_priv(cq->dev); 458272027Shselasky struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; 459219820Sjeff 460297966Shselasky if (priv->port_up == 0 || !spin_trylock(&ring->comp_lock)) 461219820Sjeff return; 462219820Sjeff mlx4_en_process_tx_cq(cq->dev, cq); 463219820Sjeff mod_timer(&cq->timer, jiffies + 1); 464219820Sjeff spin_unlock(&ring->comp_lock); 465219820Sjeff} 466219820Sjeff 467219820Sjeffvoid mlx4_en_poll_tx_cq(unsigned long data) 468219820Sjeff{ 469219820Sjeff struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data; 470219820Sjeff struct mlx4_en_priv *priv = netdev_priv(cq->dev); 471272027Shselasky struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; 472219820Sjeff u32 inflight; 473219820Sjeff 474219820Sjeff INC_PERF_COUNTER(priv->pstats.tx_poll); 475219820Sjeff 476297966Shselasky if (priv->port_up == 0) 477297966Shselasky return; 478219820Sjeff if (!spin_trylock(&ring->comp_lock)) { 479219820Sjeff mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); 480219820Sjeff return; 481219820Sjeff } 482219820Sjeff mlx4_en_process_tx_cq(cq->dev, cq); 483219820Sjeff inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb); 484219820Sjeff 485219820Sjeff /* If there are still packets in flight and the timer has not already 486219820Sjeff * been scheduled by the Tx routine then schedule it here to guarantee 487219820Sjeff * completion processing of these packets */ 488219820Sjeff if (inflight && priv->port_up) 489219820Sjeff mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); 490219820Sjeff 491219820Sjeff spin_unlock(&ring->comp_lock); 492219820Sjeff} 493219820Sjeff 494219820Sjeffstatic inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) 495219820Sjeff{ 496272027Shselasky struct mlx4_en_cq *cq = priv->tx_cq[tx_ind]; 497272027Shselasky struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; 498219820Sjeff 499297966Shselasky if (priv->port_up == 0) 500297966Shselasky return; 501297966Shselasky 502219820Sjeff /* If we don't have a pending timer, set one up to catch our recent 503219820Sjeff post in case the interface becomes idle */ 504219820Sjeff if (!timer_pending(&cq->timer)) 505219820Sjeff mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); 506219820Sjeff 507219820Sjeff /* Poll the CQ every mlx4_en_TX_MODER_POLL packets */ 508219820Sjeff if ((++ring->poll_cnt & (MLX4_EN_TX_POLL_MODER - 1)) == 0) 509219820Sjeff if (spin_trylock(&ring->comp_lock)) { 510219820Sjeff mlx4_en_process_tx_cq(priv->dev, cq); 511219820Sjeff spin_unlock(&ring->comp_lock); 512219820Sjeff } 513219820Sjeff} 514219820Sjeff 515291699Shselaskystatic u16 516291699Shselaskymlx4_en_get_inline_hdr_size(struct mlx4_en_tx_ring *ring, struct mbuf *mb) 517219820Sjeff{ 518291699Shselasky u16 retval; 519219820Sjeff 520291699Shselasky /* only copy from first fragment, if possible */ 521291699Shselasky retval = MIN(ring->inline_thold, mb->m_len); 522219820Sjeff 523291699Shselasky /* check for too little data */ 524291699Shselasky if (unlikely(retval < MIN_PKT_LEN)) 525291699Shselasky retval = MIN(ring->inline_thold, mb->m_pkthdr.len); 526291699Shselasky return (retval); 527219820Sjeff} 528219820Sjeff 529291699Shselaskystatic int 530291699Shselaskymlx4_en_get_header_size(struct mbuf *mb) 531219820Sjeff{ 532272027Shselasky struct ether_vlan_header *eh; 533272027Shselasky struct tcphdr *th; 534272027Shselasky struct ip *ip; 535272027Shselasky int ip_hlen, tcp_hlen; 536272027Shselasky struct ip6_hdr *ip6; 537272027Shselasky uint16_t eth_type; 538272027Shselasky int eth_hdr_len; 539219820Sjeff 540272027Shselasky eh = mtod(mb, struct ether_vlan_header *); 541272027Shselasky if (mb->m_len < ETHER_HDR_LEN) 542219820Sjeff return (0); 543272027Shselasky if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 544272027Shselasky eth_type = ntohs(eh->evl_proto); 545272027Shselasky eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 546272027Shselasky } else { 547272027Shselasky eth_type = ntohs(eh->evl_encap_proto); 548272027Shselasky eth_hdr_len = ETHER_HDR_LEN; 549272027Shselasky } 550272027Shselasky if (mb->m_len < eth_hdr_len) 551219820Sjeff return (0); 552272027Shselasky switch (eth_type) { 553272027Shselasky case ETHERTYPE_IP: 554272027Shselasky ip = (struct ip *)(mb->m_data + eth_hdr_len); 555272027Shselasky if (mb->m_len < eth_hdr_len + sizeof(*ip)) 556272027Shselasky return (0); 557272027Shselasky if (ip->ip_p != IPPROTO_TCP) 558272027Shselasky return (0); 559272027Shselasky ip_hlen = ip->ip_hl << 2; 560272027Shselasky eth_hdr_len += ip_hlen; 561272027Shselasky break; 562272027Shselasky case ETHERTYPE_IPV6: 563272027Shselasky ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); 564272027Shselasky if (mb->m_len < eth_hdr_len + sizeof(*ip6)) 565272027Shselasky return (0); 566272027Shselasky if (ip6->ip6_nxt != IPPROTO_TCP) 567272027Shselasky return (0); 568272027Shselasky eth_hdr_len += sizeof(*ip6); 569272027Shselasky break; 570272027Shselasky default: 571219820Sjeff return (0); 572272027Shselasky } 573272027Shselasky if (mb->m_len < eth_hdr_len + sizeof(*th)) 574272027Shselasky return (0); 575272027Shselasky th = (struct tcphdr *)(mb->m_data + eth_hdr_len); 576219820Sjeff tcp_hlen = th->th_off << 2; 577272027Shselasky eth_hdr_len += tcp_hlen; 578272027Shselasky if (mb->m_len < eth_hdr_len) 579219820Sjeff return (0); 580272027Shselasky return (eth_hdr_len); 581219820Sjeff} 582219820Sjeff 583291699Shselaskystatic volatile struct mlx4_wqe_data_seg * 584291699Shselaskymlx4_en_store_inline_data(volatile struct mlx4_wqe_data_seg *dseg, 585291699Shselasky struct mbuf *mb, int len, __be32 owner_bit) 586219820Sjeff{ 587291699Shselasky uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); 588291699Shselasky const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; 589219820Sjeff 590291699Shselasky if (unlikely(len < MIN_PKT_LEN)) { 591291699Shselasky m_copydata(mb, 0, len, inl + 4); 592291699Shselasky memset(inl + 4 + len, 0, MIN_PKT_LEN - len); 593291699Shselasky dseg += DIV_ROUND_UP(4 + MIN_PKT_LEN, DS_SIZE_ALIGNMENT); 594291699Shselasky } else if (len <= spc) { 595291699Shselasky m_copydata(mb, 0, len, inl + 4); 596291699Shselasky dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); 597291699Shselasky } else { 598291699Shselasky m_copydata(mb, 0, spc, inl + 4); 599291699Shselasky m_copydata(mb, spc, len - spc, inl + 8 + spc); 600291699Shselasky dseg += DIV_ROUND_UP(8 + len, DS_SIZE_ALIGNMENT); 601291699Shselasky } 602291699Shselasky return (dseg); 603219820Sjeff} 604219820Sjeff 605291699Shselaskystatic void 606291699Shselaskymlx4_en_store_inline_header(volatile struct mlx4_wqe_data_seg *dseg, 607291699Shselasky int len, __be32 owner_bit) 608219820Sjeff{ 609291699Shselasky uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); 610291699Shselasky const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; 611219820Sjeff 612291699Shselasky if (unlikely(len < MIN_PKT_LEN)) { 613291699Shselasky *(volatile uint32_t *)inl = 614291699Shselasky SET_BYTE_COUNT((1 << 31) | MIN_PKT_LEN); 615291699Shselasky } else if (len <= spc) { 616291699Shselasky *(volatile uint32_t *)inl = 617291699Shselasky SET_BYTE_COUNT((1 << 31) | len); 618219820Sjeff } else { 619291699Shselasky *(volatile uint32_t *)(inl + 4 + spc) = 620291699Shselasky SET_BYTE_COUNT((1 << 31) | (len - spc)); 621219820Sjeff wmb(); 622291699Shselasky *(volatile uint32_t *)inl = 623291699Shselasky SET_BYTE_COUNT((1 << 31) | spc); 624219820Sjeff } 625219820Sjeff} 626219820Sjeff 627279891Shselaskystatic uint32_t hashrandom; 628272027Shselaskystatic void hashrandom_init(void *arg) 629272027Shselasky{ 630286282Shselasky /* 631286282Shselasky * It is assumed that the random subsystem has been 632286282Shselasky * initialized when this function is called: 633286282Shselasky */ 634279891Shselasky hashrandom = m_ether_tcpip_hash_init(); 635272027Shselasky} 636286282ShselaskySYSINIT(hashrandom_init, SI_SUB_RANDOM, SI_ORDER_ANY, &hashrandom_init, NULL); 637272027Shselasky 638219820Sjeffu16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb) 639219820Sjeff{ 640219820Sjeff struct mlx4_en_priv *priv = netdev_priv(dev); 641272027Shselasky u32 rings_p_up = priv->num_tx_rings_p_up; 642272027Shselasky u32 up = 0; 643272027Shselasky u32 queue_index; 644219820Sjeff 645279584Shselasky#if (MLX4_EN_NUM_UP > 1) 646219820Sjeff /* Obtain VLAN information if present */ 647219820Sjeff if (mb->m_flags & M_VLANTAG) { 648279584Shselasky u32 vlan_tag = mb->m_pkthdr.ether_vtag; 649279584Shselasky up = (vlan_tag >> 13) % MLX4_EN_NUM_UP; 650219820Sjeff } 651279584Shselasky#endif 652291694Shselasky queue_index = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | MBUF_HASHFLAG_L4, mb, hashrandom); 653219820Sjeff 654272027Shselasky return ((queue_index % rings_p_up) + (up * rings_p_up)); 655219820Sjeff} 656219820Sjeff 657291694Shselaskystatic void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigned bytecnt) 658219820Sjeff{ 659291694Shselasky __iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8); 660219820Sjeff} 661219820Sjeff 662272027Shselaskystatic u64 mlx4_en_mac_to_u64(u8 *addr) 663272027Shselasky{ 664272027Shselasky u64 mac = 0; 665272027Shselasky int i; 666272027Shselasky 667272027Shselasky for (i = 0; i < ETHER_ADDR_LEN; i++) { 668272027Shselasky mac <<= 8; 669272027Shselasky mac |= addr[i]; 670272027Shselasky } 671272027Shselasky return mac; 672272027Shselasky} 673272027Shselasky 674291699Shselaskystatic int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp) 675219820Sjeff{ 676291699Shselasky enum { 677291699Shselasky DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT, 678291699Shselasky CTRL_FLAGS = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | 679291699Shselasky MLX4_WQE_CTRL_SOLICITED), 680291699Shselasky }; 681291699Shselasky bus_dma_segment_t segs[MLX4_EN_TX_MAX_MBUF_FRAGS]; 682291699Shselasky volatile struct mlx4_wqe_data_seg *dseg; 683291699Shselasky volatile struct mlx4_wqe_data_seg *dseg_inline; 684291699Shselasky volatile struct mlx4_en_tx_desc *tx_desc; 685291699Shselasky struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; 686291699Shselasky struct ifnet *ifp = priv->dev; 687219820Sjeff struct mlx4_en_tx_info *tx_info; 688291699Shselasky struct mbuf *mb = *mbp; 689219820Sjeff struct mbuf *m; 690291699Shselasky __be32 owner_bit; 691219820Sjeff int nr_segs; 692291699Shselasky int pad; 693291699Shselasky int err; 694291699Shselasky u32 bf_size; 695291699Shselasky u32 bf_prod; 696291699Shselasky u32 opcode; 697291699Shselasky u16 index; 698291699Shselasky u16 ds_cnt; 699291699Shselasky u16 ihs; 700219820Sjeff 701291699Shselasky if (unlikely(!priv->port_up)) { 702291699Shselasky err = EINVAL; 703219820Sjeff goto tx_drop; 704219820Sjeff } 705219820Sjeff 706291699Shselasky /* check if TX ring is full */ 707291699Shselasky if (unlikely(mlx4_en_tx_ring_is_full(ring))) { 708291699Shselasky /* every full native Tx ring stops queue */ 709291699Shselasky if (ring->blocked == 0) 710291699Shselasky atomic_add_int(&priv->blocked, 1); 711291699Shselasky /* Set HW-queue-is-full flag */ 712291699Shselasky atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); 713291699Shselasky priv->port_stats.queue_stopped++; 714219820Sjeff ring->blocked = 1; 715219820Sjeff priv->port_stats.queue_stopped++; 716272027Shselasky ring->queue_stopped++; 717219820Sjeff 718219820Sjeff /* Use interrupts to find out when queue opened */ 719291699Shselasky mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]); 720291699Shselasky return (ENOBUFS); 721272027Shselasky } 722219820Sjeff 723291699Shselasky /* sanity check we are not wrapping around */ 724291699Shselasky KASSERT(((~ring->prod) & ring->size_mask) >= 725291699Shselasky (MLX4_EN_TX_WQE_MAX_WQEBBS - 1), ("Wrapping around TX ring")); 726291699Shselasky 727219820Sjeff /* Track current inflight packets for performance analysis */ 728219820Sjeff AVG_PERF_COUNTER(priv->pstats.inflight_avg, 729219820Sjeff (u32) (ring->prod - ring->cons - 1)); 730219820Sjeff 731291699Shselasky /* Track current mbuf packet header length */ 732291699Shselasky AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); 733291699Shselasky 734291699Shselasky /* Grab an index and try to transmit packet */ 735291699Shselasky owner_bit = (ring->prod & ring->size) ? 736291699Shselasky cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0; 737219820Sjeff index = ring->prod & ring->size_mask; 738291699Shselasky tx_desc = (volatile struct mlx4_en_tx_desc *) 739291699Shselasky (ring->buf + index * TXBB_SIZE); 740291699Shselasky tx_info = &ring->tx_info[index]; 741291699Shselasky dseg = &tx_desc->data; 742219820Sjeff 743291699Shselasky /* send a copy of the frame to the BPF listener, if any */ 744291699Shselasky if (ifp != NULL && ifp->if_bpf != NULL) 745291699Shselasky ETHER_BPF_MTAP(ifp, mb); 746291699Shselasky 747291699Shselasky /* get default flags */ 748291699Shselasky tx_desc->ctrl.srcrb_flags = CTRL_FLAGS; 749291699Shselasky 750291699Shselasky if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) 751291699Shselasky tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); 752291699Shselasky 753291699Shselasky if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | 754291699Shselasky CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) 755291699Shselasky tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); 756291699Shselasky 757291699Shselasky /* do statistics */ 758291699Shselasky if (likely(tx_desc->ctrl.srcrb_flags != CTRL_FLAGS)) { 759291699Shselasky priv->port_stats.tx_chksum_offload++; 760291699Shselasky ring->tx_csum++; 761219820Sjeff } 762219820Sjeff 763291699Shselasky /* check for VLAN tag */ 764291699Shselasky if (mb->m_flags & M_VLANTAG) { 765291699Shselasky tx_desc->ctrl.vlan_tag = cpu_to_be16(mb->m_pkthdr.ether_vtag); 766291699Shselasky tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN; 767291699Shselasky } else { 768291699Shselasky tx_desc->ctrl.vlan_tag = 0; 769291699Shselasky tx_desc->ctrl.ins_vlan = 0; 770291699Shselasky } 771272027Shselasky 772291699Shselasky /* clear immediate field */ 773291699Shselasky tx_desc->ctrl.imm = 0; 774291699Shselasky 775291699Shselasky /* Handle LSO (TSO) packets */ 776291699Shselasky if (mb->m_pkthdr.csum_flags & CSUM_TSO) { 777291699Shselasky u32 payload_len; 778291699Shselasky u32 mss = mb->m_pkthdr.tso_segsz; 779291699Shselasky u32 num_pkts; 780291699Shselasky 781291699Shselasky opcode = cpu_to_be32(MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR) | 782291699Shselasky owner_bit; 783291699Shselasky ihs = mlx4_en_get_header_size(mb); 784291699Shselasky if (unlikely(ihs > MAX_INLINE)) { 785291699Shselasky ring->oversized_packets++; 786291699Shselasky err = EINVAL; 787291699Shselasky goto tx_drop; 788291699Shselasky } 789291699Shselasky tx_desc->lso.mss_hdr_size = cpu_to_be32((mss << 16) | ihs); 790291699Shselasky payload_len = mb->m_pkthdr.len - ihs; 791291699Shselasky if (unlikely(payload_len == 0)) 792291699Shselasky num_pkts = 1; 793291699Shselasky else 794291699Shselasky num_pkts = DIV_ROUND_UP(payload_len, mss); 795291699Shselasky ring->bytes += payload_len + (num_pkts * ihs); 796291699Shselasky ring->packets += num_pkts; 797291699Shselasky priv->port_stats.tso_packets++; 798291699Shselasky /* store pointer to inline header */ 799291699Shselasky dseg_inline = dseg; 800291699Shselasky /* copy data inline */ 801291699Shselasky dseg = mlx4_en_store_inline_lso_data(dseg, 802291699Shselasky mb, ihs, owner_bit); 803291699Shselasky } else { 804291699Shselasky opcode = cpu_to_be32(MLX4_OPCODE_SEND) | 805291699Shselasky owner_bit; 806291699Shselasky ihs = mlx4_en_get_inline_hdr_size(ring, mb); 807291699Shselasky ring->bytes += max_t (unsigned int, 808291699Shselasky mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); 809291699Shselasky ring->packets++; 810291699Shselasky /* store pointer to inline header */ 811291699Shselasky dseg_inline = dseg; 812291699Shselasky /* copy data inline */ 813291699Shselasky dseg = mlx4_en_store_inline_data(dseg, 814291699Shselasky mb, ihs, owner_bit); 815272027Shselasky } 816291699Shselasky m_adj(mb, ihs); 817272027Shselasky 818291699Shselasky /* trim off empty mbufs */ 819291699Shselasky while (mb->m_len == 0) { 820291699Shselasky mb = m_free(mb); 821291699Shselasky /* check if all data has been inlined */ 822291699Shselasky if (mb == NULL) { 823291699Shselasky nr_segs = 0; 824291699Shselasky goto skip_dma; 825291699Shselasky } 826291699Shselasky } 827272027Shselasky 828291699Shselasky err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, 829291699Shselasky mb, segs, &nr_segs, BUS_DMA_NOWAIT); 830291699Shselasky if (unlikely(err == EFBIG)) { 831291699Shselasky /* Too many mbuf fragments */ 832291699Shselasky m = m_defrag(mb, M_NOWAIT); 833291699Shselasky if (m == NULL) { 834291699Shselasky ring->oversized_packets++; 835291699Shselasky goto tx_drop; 836291699Shselasky } 837291699Shselasky mb = m; 838291699Shselasky /* Try again */ 839291699Shselasky err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, 840291699Shselasky mb, segs, &nr_segs, BUS_DMA_NOWAIT); 841291699Shselasky } 842291699Shselasky /* catch errors */ 843291699Shselasky if (unlikely(err != 0)) { 844291699Shselasky ring->oversized_packets++; 845291699Shselasky goto tx_drop; 846291699Shselasky } 847291699Shselasky /* make sure all mbuf data is written to RAM */ 848291699Shselasky bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, 849291699Shselasky BUS_DMASYNC_PREWRITE); 850291699Shselasky 851291699Shselaskyskip_dma: 852291699Shselasky /* compute number of DS needed */ 853291699Shselasky ds_cnt = (dseg - ((volatile struct mlx4_wqe_data_seg *)tx_desc)) + nr_segs; 854291699Shselasky 855291699Shselasky /* 856291699Shselasky * Check if the next request can wrap around and fill the end 857291699Shselasky * of the current request with zero immediate data: 858291699Shselasky */ 859291699Shselasky pad = DIV_ROUND_UP(ds_cnt, DS_FACT); 860291699Shselasky pad = (~(ring->prod + pad)) & ring->size_mask; 861291699Shselasky 862291699Shselasky if (unlikely(pad < (MLX4_EN_TX_WQE_MAX_WQEBBS - 1))) { 863291699Shselasky /* 864291699Shselasky * Compute the least number of DS blocks we need to 865291699Shselasky * pad in order to achieve a TX ring wraparound: 866291699Shselasky */ 867291699Shselasky pad = (DS_FACT * (pad + 1)); 868272027Shselasky } else { 869291699Shselasky /* 870291699Shselasky * The hardware will automatically jump to the next 871291699Shselasky * TXBB. No need for padding. 872291699Shselasky */ 873291699Shselasky pad = 0; 874272027Shselasky } 875272027Shselasky 876291699Shselasky /* compute total number of DS blocks */ 877291699Shselasky ds_cnt += pad; 878291699Shselasky /* 879291699Shselasky * When modifying this code, please ensure that the following 880291699Shselasky * computation is always less than or equal to 0x3F: 881291699Shselasky * 882291699Shselasky * ((MLX4_EN_TX_WQE_MAX_WQEBBS - 1) * DS_FACT) + 883291699Shselasky * (MLX4_EN_TX_WQE_MAX_WQEBBS * DS_FACT) 884291699Shselasky * 885291699Shselasky * Else the "ds_cnt" variable can become too big. 886291699Shselasky */ 887291699Shselasky tx_desc->ctrl.fence_size = (ds_cnt & 0x3f); 888272027Shselasky 889291699Shselasky /* store pointer to mbuf */ 890291699Shselasky tx_info->mb = mb; 891291699Shselasky tx_info->nr_txbb = DIV_ROUND_UP(ds_cnt, DS_FACT); 892291699Shselasky bf_size = ds_cnt * DS_SIZE_ALIGNMENT; 893291699Shselasky bf_prod = ring->prod; 894219820Sjeff 895291699Shselasky /* compute end of "dseg" array */ 896291699Shselasky dseg += nr_segs + pad; 897291699Shselasky 898291699Shselasky /* pad using zero immediate dseg */ 899291699Shselasky while (pad--) { 900291699Shselasky dseg--; 901291699Shselasky dseg->addr = 0; 902291699Shselasky dseg->lkey = 0; 903291699Shselasky wmb(); 904291699Shselasky dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); 905291699Shselasky } 906291699Shselasky 907291699Shselasky /* fill segment list */ 908291699Shselasky while (nr_segs--) { 909291699Shselasky if (unlikely(segs[nr_segs].ds_len == 0)) { 910291699Shselasky dseg--; 911291699Shselasky dseg->addr = 0; 912291699Shselasky dseg->lkey = 0; 913291699Shselasky wmb(); 914291699Shselasky dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); 915291699Shselasky } else { 916291699Shselasky dseg--; 917291699Shselasky dseg->addr = cpu_to_be64((uint64_t)segs[nr_segs].ds_addr); 918291699Shselasky dseg->lkey = cpu_to_be32(priv->mdev->mr.key); 919291699Shselasky wmb(); 920291699Shselasky dseg->byte_count = SET_BYTE_COUNT((uint32_t)segs[nr_segs].ds_len); 921291699Shselasky } 922291699Shselasky } 923291699Shselasky 924291699Shselasky wmb(); 925291699Shselasky 926291699Shselasky /* write owner bits in reverse order */ 927291699Shselasky if ((opcode & cpu_to_be32(0x1F)) == cpu_to_be32(MLX4_OPCODE_LSO)) 928291699Shselasky mlx4_en_store_inline_lso_header(dseg_inline, ihs, owner_bit); 929291699Shselasky else 930291699Shselasky mlx4_en_store_inline_header(dseg_inline, ihs, owner_bit); 931291699Shselasky 932219820Sjeff if (unlikely(priv->validate_loopback)) { 933219820Sjeff /* Copy dst mac address to wqe */ 934272027Shselasky struct ether_header *ethh; 935272027Shselasky u64 mac; 936272027Shselasky u32 mac_l, mac_h; 937219820Sjeff 938272027Shselasky ethh = mtod(mb, struct ether_header *); 939272027Shselasky mac = mlx4_en_mac_to_u64(ethh->ether_dhost); 940272027Shselasky if (mac) { 941272027Shselasky mac_h = (u32) ((mac & 0xffff00000000ULL) >> 16); 942272027Shselasky mac_l = (u32) (mac & 0xffffffff); 943272027Shselasky tx_desc->ctrl.srcrb_flags |= cpu_to_be32(mac_h); 944272027Shselasky tx_desc->ctrl.imm = cpu_to_be32(mac_l); 945272027Shselasky } 946219820Sjeff } 947219820Sjeff 948291699Shselasky /* update producer counter */ 949291699Shselasky ring->prod += tx_info->nr_txbb; 950219820Sjeff 951291699Shselasky if (ring->bf_enabled && bf_size <= MAX_BF && 952291699Shselasky (tx_desc->ctrl.ins_vlan != MLX4_WQE_CTRL_INS_VLAN)) { 953219820Sjeff 954291699Shselasky /* store doorbell number */ 955291699Shselasky *(volatile __be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); 956219820Sjeff 957291699Shselasky /* or in producer number for this WQE */ 958291699Shselasky opcode |= cpu_to_be32((bf_prod & 0xffff) << 8); 959219820Sjeff 960291699Shselasky /* 961291699Shselasky * Ensure the new descriptor hits memory before 962291699Shselasky * setting ownership of this descriptor to HW: 963291699Shselasky */ 964219820Sjeff wmb(); 965291699Shselasky tx_desc->ctrl.owner_opcode = opcode; 966219820Sjeff wmb(); 967291699Shselasky mlx4_bf_copy(((u8 *)ring->bf.reg) + ring->bf.offset, 968291699Shselasky (volatile unsigned long *) &tx_desc->ctrl, bf_size); 969219820Sjeff wmb(); 970219820Sjeff ring->bf.offset ^= ring->bf.buf_size; 971219820Sjeff } else { 972291699Shselasky /* 973291699Shselasky * Ensure the new descriptor hits memory before 974291699Shselasky * setting ownership of this descriptor to HW: 975291699Shselasky */ 976219820Sjeff wmb(); 977291699Shselasky tx_desc->ctrl.owner_opcode = opcode; 978219820Sjeff wmb(); 979291699Shselasky writel(cpu_to_be32(ring->doorbell_qpn), 980291699Shselasky ((u8 *)ring->bf.uar->map) + MLX4_SEND_DOORBELL); 981219820Sjeff } 982219820Sjeff 983291699Shselasky return (0); 984219820Sjefftx_drop: 985219820Sjeff *mbp = NULL; 986219820Sjeff m_freem(mb); 987291699Shselasky return (err); 988219820Sjeff} 989219820Sjeff 990219820Sjeffstatic int 991219820Sjeffmlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m) 992219820Sjeff{ 993219820Sjeff struct mlx4_en_priv *priv = netdev_priv(dev); 994219820Sjeff struct mlx4_en_tx_ring *ring; 995219820Sjeff struct mbuf *next; 996219820Sjeff int enqueued, err = 0; 997219820Sjeff 998272027Shselasky ring = priv->tx_ring[tx_ind]; 999219820Sjeff if ((dev->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1000219820Sjeff IFF_DRV_RUNNING || priv->port_up == 0) { 1001219820Sjeff if (m != NULL) 1002219820Sjeff err = drbr_enqueue(dev, ring->br, m); 1003272027Shselasky return (err); 1004219820Sjeff } 1005219820Sjeff 1006219820Sjeff enqueued = 0; 1007291694Shselasky if (m != NULL) 1008291694Shselasky /* 1009291694Shselasky * If we can't insert mbuf into drbr, try to xmit anyway. 1010291694Shselasky * We keep the error we got so we could return that after xmit. 1011291694Shselasky */ 1012291694Shselasky err = drbr_enqueue(dev, ring->br, m); 1013291694Shselasky 1014219820Sjeff /* Process the queue */ 1015246581Sdelphij while ((next = drbr_peek(dev, ring->br)) != NULL) { 1016291699Shselasky if (mlx4_en_xmit(priv, tx_ind, &next) != 0) { 1017246482Srrs if (next == NULL) { 1018246581Sdelphij drbr_advance(dev, ring->br); 1019246482Srrs } else { 1020246581Sdelphij drbr_putback(dev, ring->br, next); 1021246482Srrs } 1022219820Sjeff break; 1023219820Sjeff } 1024246581Sdelphij drbr_advance(dev, ring->br); 1025219820Sjeff enqueued++; 1026219820Sjeff if ((dev->if_drv_flags & IFF_DRV_RUNNING) == 0) 1027219820Sjeff break; 1028219820Sjeff } 1029219820Sjeff 1030219820Sjeff if (enqueued > 0) 1031219820Sjeff ring->watchdog_time = ticks; 1032219820Sjeff 1033219820Sjeff return (err); 1034219820Sjeff} 1035219820Sjeff 1036219820Sjeffvoid 1037219820Sjeffmlx4_en_tx_que(void *context, int pending) 1038219820Sjeff{ 1039219820Sjeff struct mlx4_en_tx_ring *ring; 1040219820Sjeff struct mlx4_en_priv *priv; 1041219820Sjeff struct net_device *dev; 1042219820Sjeff struct mlx4_en_cq *cq; 1043219820Sjeff int tx_ind; 1044219820Sjeff cq = context; 1045219820Sjeff dev = cq->dev; 1046219820Sjeff priv = dev->if_softc; 1047219820Sjeff tx_ind = cq->ring; 1048272027Shselasky ring = priv->tx_ring[tx_ind]; 1049297966Shselasky 1050297966Shselasky if (priv->port_up != 0 && 1051297966Shselasky (dev->if_drv_flags & IFF_DRV_RUNNING) != 0) { 1052219820Sjeff mlx4_en_xmit_poll(priv, tx_ind); 1053219820Sjeff spin_lock(&ring->tx_lock); 1054219820Sjeff if (!drbr_empty(dev, ring->br)) 1055219820Sjeff mlx4_en_transmit_locked(dev, tx_ind, NULL); 1056219820Sjeff spin_unlock(&ring->tx_lock); 1057219820Sjeff } 1058219820Sjeff} 1059219820Sjeff 1060219820Sjeffint 1061219820Sjeffmlx4_en_transmit(struct ifnet *dev, struct mbuf *m) 1062219820Sjeff{ 1063219820Sjeff struct mlx4_en_priv *priv = netdev_priv(dev); 1064219820Sjeff struct mlx4_en_tx_ring *ring; 1065219820Sjeff struct mlx4_en_cq *cq; 1066275358Shselasky int i, err = 0; 1067219820Sjeff 1068297966Shselasky if (priv->port_up == 0) { 1069297966Shselasky m_freem(m); 1070297966Shselasky return (ENETDOWN); 1071297966Shselasky } 1072297966Shselasky 1073275358Shselasky /* Compute which queue to use */ 1074291694Shselasky if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 1075296910Shselasky i = (m->m_pkthdr.flowid % 128) % priv->tx_ring_num; 1076291694Shselasky } 1077291694Shselasky else { 1078291694Shselasky i = mlx4_en_select_queue(dev, m); 1079291694Shselasky } 1080275358Shselasky 1081272027Shselasky ring = priv->tx_ring[i]; 1082219820Sjeff if (spin_trylock(&ring->tx_lock)) { 1083219820Sjeff err = mlx4_en_transmit_locked(dev, i, m); 1084219820Sjeff spin_unlock(&ring->tx_lock); 1085219820Sjeff /* Poll CQ here */ 1086219820Sjeff mlx4_en_xmit_poll(priv, i); 1087219820Sjeff } else { 1088219820Sjeff err = drbr_enqueue(dev, ring->br, m); 1089272027Shselasky cq = priv->tx_cq[i]; 1090219820Sjeff taskqueue_enqueue(cq->tq, &cq->cq_task); 1091219820Sjeff } 1092219820Sjeff 1093219820Sjeff return (err); 1094219820Sjeff} 1095219820Sjeff 1096219820Sjeff/* 1097219820Sjeff * Flush ring buffers. 1098219820Sjeff */ 1099219820Sjeffvoid 1100219820Sjeffmlx4_en_qflush(struct ifnet *dev) 1101219820Sjeff{ 1102219820Sjeff struct mlx4_en_priv *priv = netdev_priv(dev); 1103272027Shselasky struct mlx4_en_tx_ring *ring; 1104219820Sjeff struct mbuf *m; 1105219820Sjeff 1106297966Shselasky if (priv->port_up == 0) 1107297966Shselasky return; 1108297966Shselasky 1109272027Shselasky for (int i = 0; i < priv->tx_ring_num; i++) { 1110272027Shselasky ring = priv->tx_ring[i]; 1111219820Sjeff spin_lock(&ring->tx_lock); 1112219820Sjeff while ((m = buf_ring_dequeue_sc(ring->br)) != NULL) 1113219820Sjeff m_freem(m); 1114219820Sjeff spin_unlock(&ring->tx_lock); 1115219820Sjeff } 1116219820Sjeff if_qflush(dev); 1117219820Sjeff} 1118