1/* 2 * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33#include "opt_inet.h" 34#include <dev/mlx4/cq.h> 35#include <linux/slab.h> 36#include <dev/mlx4/qp.h> 37#include <linux/if_ether.h> 38#include <linux/if_vlan.h> 39#include <linux/vmalloc.h> 40#include <dev/mlx4/driver.h> 41#ifdef CONFIG_NET_RX_BUSY_POLL 42#include <net/busy_poll.h> 43#endif 44 45#include "en.h" 46 47#if (MLX4_EN_MAX_RX_SEGS == 1) 48static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, 49 struct mlx4_en_rx_ring *ring, 50 int index) 51{ 52 struct mlx4_en_rx_desc *rx_desc = 53 ((struct mlx4_en_rx_desc *)ring->buf) + index; 54 int i; 55 56 /* Set size and memtype fields */ 57 rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size - MLX4_NET_IP_ALIGN); 58 rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); 59 60 /* 61 * If the number of used fragments does not fill up the ring 62 * stride, remaining (unused) fragments must be padded with 63 * null address/size and a special memory key: 64 */ 65 for (i = 1; i < MLX4_EN_MAX_RX_SEGS; i++) { 66 rx_desc->data[i].byte_count = 0; 67 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); 68 rx_desc->data[i].addr = 0; 69 } 70} 71#endif 72 73static inline struct mbuf * 74mlx4_en_alloc_mbuf(struct mlx4_en_rx_ring *ring) 75{ 76 struct mbuf *mb; 77 78#if (MLX4_EN_MAX_RX_SEGS == 1) 79 mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); 80 if (likely(mb != NULL)) 81 mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; 82#else 83 mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MLX4_EN_MAX_RX_BYTES); 84 if (likely(mb != NULL)) { 85 struct mbuf *mb_head = mb; 86 int i; 87 88 mb->m_len = MLX4_EN_MAX_RX_BYTES; 89 mb->m_pkthdr.len = MLX4_EN_MAX_RX_BYTES; 90 91 for (i = 1; i != MLX4_EN_MAX_RX_SEGS; i++) { 92 if (mb_head->m_pkthdr.len >= ring->rx_mb_size) 93 break; 94 mb = (mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, MLX4_EN_MAX_RX_BYTES)); 95 if (unlikely(mb == NULL)) { 96 m_freem(mb_head); 97 return (NULL); 98 } 99 mb->m_len = MLX4_EN_MAX_RX_BYTES; 100 mb_head->m_pkthdr.len += MLX4_EN_MAX_RX_BYTES; 101 } 102 /* rewind to first mbuf in chain */ 103 mb = mb_head; 104 } 105#endif 106 return (mb); 107} 108 109static int 110mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, 111 struct mlx4_en_rx_mbuf *mb_list) 112{ 113 bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS]; 114 bus_dmamap_t map; 115 struct mbuf *mb; 116 int nsegs; 117 int err; 118#if (MLX4_EN_MAX_RX_SEGS != 1) 119 int i; 120#endif 121 122 /* try to allocate a new spare mbuf */ 123 if (unlikely(ring->spare.mbuf == NULL)) { 124 mb = mlx4_en_alloc_mbuf(ring); 125 if (unlikely(mb == NULL)) 126 return (-ENOMEM); 127 128 /* make sure IP header gets aligned */ 129 m_adj(mb, MLX4_NET_IP_ALIGN); 130 131 /* load spare mbuf into BUSDMA */ 132 err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, 133 mb, ring->spare.segs, &nsegs, BUS_DMA_NOWAIT); 134 if (unlikely(err != 0)) { 135 m_freem(mb); 136 return (err); 137 } 138 139 /* store spare info */ 140 ring->spare.mbuf = mb; 141 142#if (MLX4_EN_MAX_RX_SEGS != 1) 143 /* zero remaining segs */ 144 for (i = nsegs; i != MLX4_EN_MAX_RX_SEGS; i++) { 145 ring->spare.segs[i].ds_addr = 0; 146 ring->spare.segs[i].ds_len = 0; 147 } 148#endif 149 bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, 150 BUS_DMASYNC_PREREAD); 151 } 152 153 /* synchronize and unload the current mbuf, if any */ 154 if (likely(mb_list->mbuf != NULL)) { 155 bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, 156 BUS_DMASYNC_POSTREAD); 157 bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); 158 } 159 160 mb = mlx4_en_alloc_mbuf(ring); 161 if (unlikely(mb == NULL)) 162 goto use_spare; 163 164 /* make sure IP header gets aligned */ 165 m_adj(mb, MLX4_NET_IP_ALIGN); 166 167 err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, 168 mb, segs, &nsegs, BUS_DMA_NOWAIT); 169 if (unlikely(err != 0)) { 170 m_freem(mb); 171 goto use_spare; 172 } 173 174#if (MLX4_EN_MAX_RX_SEGS == 1) 175 rx_desc->data[0].addr = cpu_to_be64(segs[0].ds_addr); 176#else 177 for (i = 0; i != nsegs; i++) { 178 rx_desc->data[i].byte_count = cpu_to_be32(segs[i].ds_len); 179 rx_desc->data[i].lkey = ring->rx_mr_key_be; 180 rx_desc->data[i].addr = cpu_to_be64(segs[i].ds_addr); 181 } 182 for (; i != MLX4_EN_MAX_RX_SEGS; i++) { 183 rx_desc->data[i].byte_count = 0; 184 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); 185 rx_desc->data[i].addr = 0; 186 } 187#endif 188 mb_list->mbuf = mb; 189 190 bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); 191 return (0); 192 193use_spare: 194 /* swap DMA maps */ 195 map = mb_list->dma_map; 196 mb_list->dma_map = ring->spare.dma_map; 197 ring->spare.dma_map = map; 198 199 /* swap MBUFs */ 200 mb_list->mbuf = ring->spare.mbuf; 201 ring->spare.mbuf = NULL; 202 203 /* store physical address */ 204#if (MLX4_EN_MAX_RX_SEGS == 1) 205 rx_desc->data[0].addr = cpu_to_be64(ring->spare.segs[0].ds_addr); 206#else 207 for (i = 0; i != MLX4_EN_MAX_RX_SEGS; i++) { 208 if (ring->spare.segs[i].ds_len != 0) { 209 rx_desc->data[i].byte_count = cpu_to_be32(ring->spare.segs[i].ds_len); 210 rx_desc->data[i].lkey = ring->rx_mr_key_be; 211 rx_desc->data[i].addr = cpu_to_be64(ring->spare.segs[i].ds_addr); 212 } else { 213 rx_desc->data[i].byte_count = 0; 214 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); 215 rx_desc->data[i].addr = 0; 216 } 217 } 218#endif 219 return (0); 220} 221 222static void 223mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) 224{ 225 bus_dmamap_t map = mb_list->dma_map; 226 bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); 227 bus_dmamap_unload(ring->dma_tag, map); 228 m_freem(mb_list->mbuf); 229 mb_list->mbuf = NULL; /* safety clearing */ 230} 231 232static int 233mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, 234 struct mlx4_en_rx_ring *ring, int index) 235{ 236 struct mlx4_en_rx_desc *rx_desc = 237 ((struct mlx4_en_rx_desc *)ring->buf) + index; 238 struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; 239 240 mb_list->mbuf = NULL; 241 242 if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) { 243 priv->port_stats.rx_alloc_failed++; 244 return (-ENOMEM); 245 } 246 return (0); 247} 248 249static inline void 250mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) 251{ 252 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); 253} 254 255static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) 256{ 257 struct mlx4_en_rx_ring *ring; 258 int ring_ind; 259 int buf_ind; 260 int new_size; 261 int err; 262 263 for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { 264 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 265 ring = priv->rx_ring[ring_ind]; 266 267 err = mlx4_en_prepare_rx_desc(priv, ring, 268 ring->actual_size); 269 if (err) { 270 if (ring->actual_size == 0) { 271 en_err(priv, "Failed to allocate " 272 "enough rx buffers\n"); 273 return -ENOMEM; 274 } else { 275 new_size = 276 rounddown_pow_of_two(ring->actual_size); 277 en_warn(priv, "Only %d buffers allocated " 278 "reducing ring size to %d\n", 279 ring->actual_size, new_size); 280 goto reduce_rings; 281 } 282 } 283 ring->actual_size++; 284 ring->prod++; 285 } 286 } 287 return 0; 288 289reduce_rings: 290 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 291 ring = priv->rx_ring[ring_ind]; 292 while (ring->actual_size > new_size) { 293 ring->actual_size--; 294 ring->prod--; 295 mlx4_en_free_buf(ring, 296 ring->mbuf + ring->actual_size); 297 } 298 } 299 300 return 0; 301} 302 303static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, 304 struct mlx4_en_rx_ring *ring) 305{ 306 int index; 307 308 en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n", 309 ring->cons, ring->prod); 310 311 /* Unmap and free Rx buffers */ 312 BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); 313 while (ring->cons != ring->prod) { 314 index = ring->cons & ring->size_mask; 315 en_dbg(DRV, priv, "Processing descriptor:%d\n", index); 316 mlx4_en_free_buf(ring, ring->mbuf + index); 317 ++ring->cons; 318 } 319} 320 321void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) 322{ 323 int i; 324 int num_of_eqs; 325 int num_rx_rings; 326 struct mlx4_dev *dev = mdev->dev; 327 328 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { 329 num_of_eqs = max_t(int, MIN_RX_RINGS, 330 min_t(int, 331 mlx4_get_eqs_per_port(mdev->dev, i), 332 DEF_RX_RINGS)); 333 334 num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS : 335 num_of_eqs; 336 mdev->profile.prof[i].rx_ring_num = 337 rounddown_pow_of_two(num_rx_rings); 338 } 339} 340 341void mlx4_en_calc_rx_buf(struct net_device *dev) 342{ 343 struct mlx4_en_priv *priv = netdev_priv(dev); 344 int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN + 345 MLX4_NET_IP_ALIGN; 346 347 if (eff_mtu > MJUM16BYTES) { 348 en_err(priv, "MTU(%u) is too big\n", (unsigned)dev->if_mtu); 349 eff_mtu = MJUM16BYTES; 350 } else if (eff_mtu > MJUM9BYTES) { 351 eff_mtu = MJUM16BYTES; 352 } else if (eff_mtu > MJUMPAGESIZE) { 353 eff_mtu = MJUM9BYTES; 354 } else if (eff_mtu > MCLBYTES) { 355 eff_mtu = MJUMPAGESIZE; 356 } else { 357 eff_mtu = MCLBYTES; 358 } 359 360 priv->rx_mb_size = eff_mtu; 361 362 en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu); 363} 364 365int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, 366 struct mlx4_en_rx_ring **pring, 367 u32 size, int node) 368{ 369 struct mlx4_en_dev *mdev = priv->mdev; 370 struct mlx4_en_rx_ring *ring; 371 int err; 372 int tmp; 373 uint32_t x; 374 375 ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); 376 if (!ring) { 377 en_err(priv, "Failed to allocate RX ring structure\n"); 378 return -ENOMEM; 379 } 380 381 /* Create DMA descriptor TAG */ 382 if ((err = -bus_dma_tag_create( 383 bus_get_dma_tag(mdev->pdev->dev.bsddev), 384 1, /* any alignment */ 385 0, /* no boundary */ 386 BUS_SPACE_MAXADDR, /* lowaddr */ 387 BUS_SPACE_MAXADDR, /* highaddr */ 388 NULL, NULL, /* filter, filterarg */ 389 MJUM16BYTES, /* maxsize */ 390 MLX4_EN_MAX_RX_SEGS, /* nsegments */ 391 MJUM16BYTES, /* maxsegsize */ 392 0, /* flags */ 393 NULL, NULL, /* lockfunc, lockfuncarg */ 394 &ring->dma_tag))) { 395 en_err(priv, "Failed to create DMA tag\n"); 396 goto err_ring; 397 } 398 399 ring->prod = 0; 400 ring->cons = 0; 401 ring->size = size; 402 ring->size_mask = size - 1; 403 404 ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); 405 ring->buf_size = (ring->size * sizeof(struct mlx4_en_rx_desc)) + TXBB_SIZE; 406 407 tmp = size * sizeof(struct mlx4_en_rx_mbuf); 408 409 ring->mbuf = kzalloc(tmp, GFP_KERNEL); 410 if (ring->mbuf == NULL) { 411 err = -ENOMEM; 412 goto err_dma_tag; 413 } 414 415 err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); 416 if (err != 0) 417 goto err_info; 418 419 for (x = 0; x != size; x++) { 420 err = -bus_dmamap_create(ring->dma_tag, 0, 421 &ring->mbuf[x].dma_map); 422 if (err != 0) { 423 while (x--) 424 bus_dmamap_destroy(ring->dma_tag, 425 ring->mbuf[x].dma_map); 426 goto err_info; 427 } 428 } 429 en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n", 430 ring->mbuf, tmp); 431 432 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, 433 ring->buf_size, 2 * PAGE_SIZE); 434 if (err) 435 goto err_dma_map; 436 437 err = mlx4_en_map_buffer(&ring->wqres.buf); 438 if (err) { 439 en_err(priv, "Failed to map RX buffer\n"); 440 goto err_hwq; 441 } 442 ring->buf = ring->wqres.buf.direct.buf; 443 *pring = ring; 444 return 0; 445 446err_hwq: 447 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 448err_dma_map: 449 for (x = 0; x != size; x++) { 450 bus_dmamap_destroy(ring->dma_tag, 451 ring->mbuf[x].dma_map); 452 } 453 bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); 454err_info: 455 vfree(ring->mbuf); 456err_dma_tag: 457 bus_dma_tag_destroy(ring->dma_tag); 458err_ring: 459 kfree(ring); 460 return (err); 461} 462 463int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) 464{ 465 struct mlx4_en_rx_ring *ring; 466#if (MLX4_EN_MAX_RX_SEGS == 1) 467 int i; 468#endif 469 int ring_ind; 470 int err; 471 472 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 473 ring = priv->rx_ring[ring_ind]; 474 475 ring->prod = 0; 476 ring->cons = 0; 477 ring->actual_size = 0; 478 ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; 479 ring->rx_mb_size = priv->rx_mb_size; 480 481 if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) { 482 /* Stamp first unused send wqe */ 483 __be32 *ptr = (__be32 *)ring->buf; 484 __be32 stamp = cpu_to_be32(1 << STAMP_SHIFT); 485 *ptr = stamp; 486 /* Move pointer to start of rx section */ 487 ring->buf += TXBB_SIZE; 488 } 489 490 ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); 491 ring->buf_size = ring->size * sizeof(struct mlx4_en_rx_desc); 492 493 memset(ring->buf, 0, ring->buf_size); 494 mlx4_en_update_rx_prod_db(ring); 495 496#if (MLX4_EN_MAX_RX_SEGS == 1) 497 /* Initialize all descriptors */ 498 for (i = 0; i < ring->size; i++) 499 mlx4_en_init_rx_desc(priv, ring, i); 500#endif 501 ring->rx_mr_key_be = cpu_to_be32(priv->mdev->mr.key); 502 503#ifdef INET 504 /* Configure lro mngr */ 505 if (priv->dev->if_capenable & IFCAP_LRO) { 506 if (tcp_lro_init(&ring->lro)) 507 priv->dev->if_capenable &= ~IFCAP_LRO; 508 else 509 ring->lro.ifp = priv->dev; 510 } 511#endif 512 } 513 514 515 err = mlx4_en_fill_rx_buffers(priv); 516 if (err) 517 goto err_buffers; 518 519 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 520 ring = priv->rx_ring[ring_ind]; 521 522 ring->size_mask = ring->actual_size - 1; 523 mlx4_en_update_rx_prod_db(ring); 524 } 525 526 return 0; 527 528err_buffers: 529 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) 530 mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]); 531 532 ring_ind = priv->rx_ring_num - 1; 533 534 while (ring_ind >= 0) { 535 ring = priv->rx_ring[ring_ind]; 536 if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) 537 ring->buf -= TXBB_SIZE; 538 ring_ind--; 539 } 540 541 return err; 542} 543 544 545void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, 546 struct mlx4_en_rx_ring **pring, 547 u32 size) 548{ 549 struct mlx4_en_dev *mdev = priv->mdev; 550 struct mlx4_en_rx_ring *ring = *pring; 551 uint32_t x; 552 553 mlx4_en_unmap_buffer(&ring->wqres.buf); 554 mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * sizeof(struct mlx4_en_rx_desc) + TXBB_SIZE); 555 for (x = 0; x != size; x++) 556 bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); 557 /* free spare mbuf, if any */ 558 if (ring->spare.mbuf != NULL) { 559 bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, 560 BUS_DMASYNC_POSTREAD); 561 bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); 562 m_freem(ring->spare.mbuf); 563 } 564 bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); 565 vfree(ring->mbuf); 566 bus_dma_tag_destroy(ring->dma_tag); 567 kfree(ring); 568 *pring = NULL; 569#ifdef CONFIG_RFS_ACCEL 570 mlx4_en_cleanup_filters(priv, ring); 571#endif 572} 573 574void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, 575 struct mlx4_en_rx_ring *ring) 576{ 577#ifdef INET 578 tcp_lro_free(&ring->lro); 579#endif 580 mlx4_en_free_rx_buf(priv, ring); 581 if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) 582 ring->buf -= TXBB_SIZE; 583} 584 585 586static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) 587{ 588 int i; 589 int offset = ETHER_HDR_LEN; 590 591 for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { 592 if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) 593 goto out_loopback; 594 } 595 /* Loopback found */ 596 priv->loopback_ok = 1; 597 598out_loopback: 599 m_freem(mb); 600} 601 602 603static inline int invalid_cqe(struct mlx4_en_priv *priv, 604 struct mlx4_cqe *cqe) 605{ 606 /* Drop packet on bad receive or bad checksum */ 607 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == 608 MLX4_CQE_OPCODE_ERROR)) { 609 en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n", 610 ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome, 611 ((struct mlx4_err_cqe *)cqe)->syndrome); 612 return 1; 613 } 614 if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { 615 en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); 616 return 1; 617 } 618 619 return 0; 620} 621 622static struct mbuf * 623mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, 624 struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, 625 int length) 626{ 627#if (MLX4_EN_MAX_RX_SEGS != 1) 628 struct mbuf *mb_head; 629#endif 630 struct mbuf *mb; 631 632 /* optimise reception of small packets */ 633 if (length <= (MHLEN - MLX4_NET_IP_ALIGN) && 634 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { 635 636 /* set packet length */ 637 mb->m_pkthdr.len = mb->m_len = length; 638 639 /* make sure IP header gets aligned */ 640 mb->m_data += MLX4_NET_IP_ALIGN; 641 642 bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, 643 BUS_DMASYNC_POSTREAD); 644 645 bcopy(mtod(mb_list->mbuf, caddr_t), mtod(mb, caddr_t), length); 646 647 return (mb); 648 } 649 650 /* get mbuf */ 651 mb = mb_list->mbuf; 652 653 /* collect used fragment while atomically replacing it */ 654 if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) 655 return (NULL); 656 657 /* range check hardware computed value */ 658 if (unlikely(length > mb->m_pkthdr.len)) 659 length = mb->m_pkthdr.len; 660 661#if (MLX4_EN_MAX_RX_SEGS == 1) 662 /* update total packet length in packet header */ 663 mb->m_len = mb->m_pkthdr.len = length; 664#else 665 mb->m_pkthdr.len = length; 666 for (mb_head = mb; mb != NULL; mb = mb->m_next) { 667 if (mb->m_len > length) 668 mb->m_len = length; 669 length -= mb->m_len; 670 if (likely(length == 0)) { 671 if (likely(mb->m_next != NULL)) { 672 /* trim off empty mbufs */ 673 m_freem(mb->m_next); 674 mb->m_next = NULL; 675 } 676 break; 677 } 678 } 679 /* rewind to first mbuf in chain */ 680 mb = mb_head; 681#endif 682 return (mb); 683} 684 685static __inline int 686mlx4_en_rss_hash(__be16 status, int udp_rss) 687{ 688 enum { 689 status_all = cpu_to_be16( 690 MLX4_CQE_STATUS_IPV4 | 691 MLX4_CQE_STATUS_IPV4F | 692 MLX4_CQE_STATUS_IPV6 | 693 MLX4_CQE_STATUS_TCP | 694 MLX4_CQE_STATUS_UDP), 695 status_ipv4_tcp = cpu_to_be16( 696 MLX4_CQE_STATUS_IPV4 | 697 MLX4_CQE_STATUS_TCP), 698 status_ipv6_tcp = cpu_to_be16( 699 MLX4_CQE_STATUS_IPV6 | 700 MLX4_CQE_STATUS_TCP), 701 status_ipv4_udp = cpu_to_be16( 702 MLX4_CQE_STATUS_IPV4 | 703 MLX4_CQE_STATUS_UDP), 704 status_ipv6_udp = cpu_to_be16( 705 MLX4_CQE_STATUS_IPV6 | 706 MLX4_CQE_STATUS_UDP), 707 status_ipv4 = cpu_to_be16(MLX4_CQE_STATUS_IPV4), 708 status_ipv6 = cpu_to_be16(MLX4_CQE_STATUS_IPV6) 709 }; 710 711 status &= status_all; 712 switch (status) { 713 case status_ipv4_tcp: 714 return (M_HASHTYPE_RSS_TCP_IPV4); 715 case status_ipv6_tcp: 716 return (M_HASHTYPE_RSS_TCP_IPV6); 717 case status_ipv4_udp: 718 return (udp_rss ? M_HASHTYPE_RSS_UDP_IPV4 719 : M_HASHTYPE_RSS_IPV4); 720 case status_ipv6_udp: 721 return (udp_rss ? M_HASHTYPE_RSS_UDP_IPV6 722 : M_HASHTYPE_RSS_IPV6); 723 default: 724 if (status & status_ipv4) 725 return (M_HASHTYPE_RSS_IPV4); 726 if (status & status_ipv6) 727 return (M_HASHTYPE_RSS_IPV6); 728 return (M_HASHTYPE_OPAQUE_HASH); 729 } 730} 731 732/* For cpu arch with cache line of 64B the performance is better when cqe size==64B 733 * To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc) 734 * was added in the beginning of each cqe (the real data is in the corresponding 32B). 735 * The following calc ensures that when factor==1, it means we are aligned to 64B 736 * and we get the real cqe data*/ 737#define CQE_FACTOR_INDEX(index, factor) (((index) << (factor)) + (factor)) 738int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) 739{ 740 struct mlx4_en_priv *priv = netdev_priv(dev); 741 struct mlx4_cqe *cqe; 742 struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; 743 struct mlx4_en_rx_mbuf *mb_list; 744 struct mlx4_en_rx_desc *rx_desc; 745 struct mbuf *mb; 746 struct mlx4_cq *mcq = &cq->mcq; 747 struct mlx4_cqe *buf = cq->buf; 748 int index; 749 unsigned int length; 750 int polled = 0; 751 u32 cons_index = mcq->cons_index; 752 u32 size_mask = ring->size_mask; 753 int size = cq->size; 754 int factor = priv->cqe_factor; 755 const int udp_rss = priv->mdev->profile.udp_rss; 756 757 if (!priv->port_up) 758 return 0; 759 760 /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx 761 * descriptor offset can be deducted from the CQE index instead of 762 * reading 'cqe->index' */ 763 index = cons_index & size_mask; 764 cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; 765 766 /* Process all completed CQEs */ 767 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, 768 cons_index & size)) { 769 mb_list = ring->mbuf + index; 770 rx_desc = ((struct mlx4_en_rx_desc *)ring->buf) + index; 771 772 /* 773 * make sure we read the CQE after we read the ownership bit 774 */ 775 rmb(); 776 777 if (invalid_cqe(priv, cqe)) { 778 goto next; 779 } 780 /* 781 * Packet is OK - process it. 782 */ 783 length = be32_to_cpu(cqe->byte_cnt); 784 length -= ring->fcs_del; 785 786 mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); 787 if (unlikely(!mb)) { 788 ring->errors++; 789 goto next; 790 } 791 792 ring->bytes += length; 793 ring->packets++; 794 795 if (unlikely(priv->validate_loopback)) { 796 validate_loopback(priv, mb); 797 goto next; 798 } 799 800 /* forward Toeplitz compatible hash value */ 801 mb->m_pkthdr.flowid = be32_to_cpu(cqe->immed_rss_invalid); 802 M_HASHTYPE_SET(mb, mlx4_en_rss_hash(cqe->status, udp_rss)); 803 mb->m_pkthdr.rcvif = dev; 804 if (be32_to_cpu(cqe->vlan_my_qpn) & 805 MLX4_CQE_CVLAN_PRESENT_MASK) { 806 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); 807 mb->m_flags |= M_VLANTAG; 808 } 809 if (likely(dev->if_capenable & 810 (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && 811 (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && 812 (cqe->checksum == cpu_to_be16(0xffff))) { 813 priv->port_stats.rx_chksum_good++; 814 mb->m_pkthdr.csum_flags = 815 CSUM_IP_CHECKED | CSUM_IP_VALID | 816 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 817 mb->m_pkthdr.csum_data = htons(0xffff); 818 /* This packet is eligible for LRO if it is: 819 * - DIX Ethernet (type interpretation) 820 * - TCP/IP (v4) 821 * - without IP options 822 * - not an IP fragment 823 */ 824#ifdef INET 825 if (mlx4_en_can_lro(cqe->status) && 826 (dev->if_capenable & IFCAP_LRO)) { 827 if (ring->lro.lro_cnt != 0 && 828 tcp_lro_rx(&ring->lro, mb, 0) == 0) 829 goto next; 830 } 831 832#endif 833 /* LRO not possible, complete processing here */ 834 INC_PERF_COUNTER(priv->pstats.lro_misses); 835 } else { 836 mb->m_pkthdr.csum_flags = 0; 837 priv->port_stats.rx_chksum_none++; 838 } 839 840 /* Push it up the stack */ 841 dev->if_input(dev, mb); 842 843next: 844 ++cons_index; 845 index = cons_index & size_mask; 846 cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; 847 if (++polled == budget) 848 goto out; 849 } 850 /* Flush all pending IP reassembly sessions */ 851out: 852#ifdef INET 853 tcp_lro_flush_all(&ring->lro); 854#endif 855 AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); 856 mcq->cons_index = cons_index; 857 mlx4_cq_set_ci(mcq); 858 wmb(); /* ensure HW sees CQ consumer before we post new buffers */ 859 ring->cons = mcq->cons_index; 860 ring->prod += polled; /* Polled descriptors were realocated in place */ 861 mlx4_en_update_rx_prod_db(ring); 862 return polled; 863 864} 865 866/* Rx CQ polling - called by NAPI */ 867static int mlx4_en_poll_rx_cq(struct mlx4_en_cq *cq, int budget) 868{ 869 struct net_device *dev = cq->dev; 870 int done; 871 872 done = mlx4_en_process_rx_cq(dev, cq, budget); 873 cq->tot_rx += done; 874 875 return done; 876 877} 878void mlx4_en_rx_irq(struct mlx4_cq *mcq) 879{ 880 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); 881 struct mlx4_en_priv *priv = netdev_priv(cq->dev); 882 int done; 883 884 // Shoot one within the irq context 885 // Because there is no NAPI in freeBSD 886 done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); 887 if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { 888 cq->curr_poll_rx_cpu_id = curcpu; 889 taskqueue_enqueue(cq->tq, &cq->cq_task); 890 } 891 else { 892 mlx4_en_arm_cq(priv, cq); 893 } 894} 895 896void mlx4_en_rx_que(void *context, int pending) 897{ 898 struct mlx4_en_cq *cq; 899 struct thread *td; 900 901 cq = context; 902 td = curthread; 903 904 thread_lock(td); 905 sched_bind(td, cq->curr_poll_rx_cpu_id); 906 thread_unlock(td); 907 908 while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) 909 == MLX4_EN_RX_BUDGET); 910 mlx4_en_arm_cq(cq->dev->if_softc, cq); 911} 912 913 914/* RSS related functions */ 915 916static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, 917 struct mlx4_en_rx_ring *ring, 918 enum mlx4_qp_state *state, 919 struct mlx4_qp *qp) 920{ 921 struct mlx4_en_dev *mdev = priv->mdev; 922 struct mlx4_qp_context *context; 923 int err = 0; 924 925 context = kmalloc(sizeof *context , GFP_KERNEL); 926 if (!context) { 927 en_err(priv, "Failed to allocate qp context\n"); 928 return -ENOMEM; 929 } 930 931 err = mlx4_qp_alloc(mdev->dev, qpn, qp, GFP_KERNEL); 932 if (err) { 933 en_err(priv, "Failed to allocate qp #%x\n", qpn); 934 goto out; 935 } 936 qp->event = mlx4_en_sqp_event; 937 938 memset(context, 0, sizeof *context); 939 mlx4_en_fill_qp_context(priv, ring->actual_size, sizeof(struct mlx4_en_rx_desc), 0, 0, 940 qpn, ring->cqn, -1, context); 941 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); 942 943 /* Cancel FCS removal if FW allows */ 944 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { 945 context->param3 |= cpu_to_be32(1 << 29); 946 ring->fcs_del = ETH_FCS_LEN; 947 } else 948 ring->fcs_del = 0; 949 950 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); 951 if (err) { 952 mlx4_qp_remove(mdev->dev, qp); 953 mlx4_qp_free(mdev->dev, qp); 954 } 955 mlx4_en_update_rx_prod_db(ring); 956out: 957 kfree(context); 958 return err; 959} 960 961int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) 962{ 963 int err; 964 u32 qpn; 965 966 err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0); 967 if (err) { 968 en_err(priv, "Failed reserving drop qpn\n"); 969 return err; 970 } 971 err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp, GFP_KERNEL); 972 if (err) { 973 en_err(priv, "Failed allocating drop qp\n"); 974 mlx4_qp_release_range(priv->mdev->dev, qpn, 1); 975 return err; 976 } 977 978 return 0; 979} 980 981void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) 982{ 983 u32 qpn; 984 985 qpn = priv->drop_qp.qpn; 986 mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); 987 mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); 988 mlx4_qp_release_range(priv->mdev->dev, qpn, 1); 989} 990 991const u32 * 992mlx4_en_get_rss_key(struct mlx4_en_priv *priv __unused, 993 u16 *keylen) 994{ 995 static const u32 rsskey[10] = { 996 cpu_to_be32(0xD181C62C), 997 cpu_to_be32(0xF7F4DB5B), 998 cpu_to_be32(0x1983A2FC), 999 cpu_to_be32(0x943E1ADB), 1000 cpu_to_be32(0xD9389E6B), 1001 cpu_to_be32(0xD1039C2C), 1002 cpu_to_be32(0xA74499AD), 1003 cpu_to_be32(0x593D56D9), 1004 cpu_to_be32(0xF3253C06), 1005 cpu_to_be32(0x2ADC1FFC) 1006 }; 1007 1008 if (keylen != NULL) 1009 *keylen = sizeof(rsskey); 1010 return (rsskey); 1011} 1012 1013u8 mlx4_en_get_rss_mask(struct mlx4_en_priv *priv) 1014{ 1015 u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 | 1016 MLX4_RSS_TCP_IPV6); 1017 1018 if (priv->mdev->profile.udp_rss) 1019 rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; 1020 return (rss_mask); 1021} 1022 1023/* Allocate rx qp's and configure them according to rss map */ 1024int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) 1025{ 1026 struct mlx4_en_dev *mdev = priv->mdev; 1027 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 1028 struct mlx4_qp_context context; 1029 struct mlx4_rss_context *rss_context; 1030 const u32 *key; 1031 int rss_rings; 1032 void *ptr; 1033 int i; 1034 int err = 0; 1035 int good_qps = 0; 1036 1037 en_dbg(DRV, priv, "Configuring rss steering\n"); 1038 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, 1039 priv->rx_ring_num, 1040 &rss_map->base_qpn, 0); 1041 if (err) { 1042 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num); 1043 return err; 1044 } 1045 1046 for (i = 0; i < priv->rx_ring_num; i++) { 1047 priv->rx_ring[i]->qpn = rss_map->base_qpn + i; 1048 err = mlx4_en_config_rss_qp(priv, priv->rx_ring[i]->qpn, 1049 priv->rx_ring[i], 1050 &rss_map->state[i], 1051 &rss_map->qps[i]); 1052 if (err) 1053 goto rss_err; 1054 1055 ++good_qps; 1056 } 1057 1058 /* Configure RSS indirection qp */ 1059 err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp, GFP_KERNEL); 1060 if (err) { 1061 en_err(priv, "Failed to allocate RSS indirection QP\n"); 1062 goto rss_err; 1063 } 1064 rss_map->indir_qp.event = mlx4_en_sqp_event; 1065 mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, 1066 priv->rx_ring[0]->cqn, -1, &context); 1067 1068 if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) 1069 rss_rings = priv->rx_ring_num; 1070 else 1071 rss_rings = priv->prof->rss_rings; 1072 1073 ptr = ((u8 *)&context) + offsetof(struct mlx4_qp_context, pri_path) + 1074 MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; 1075 rss_context = ptr; 1076 rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | 1077 (rss_map->base_qpn)); 1078 rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); 1079 if (priv->mdev->profile.udp_rss) 1080 rss_context->base_qpn_udp = rss_context->default_qpn; 1081 rss_context->flags = mlx4_en_get_rss_mask(priv); 1082 rss_context->hash_fn = MLX4_RSS_HASH_TOP; 1083 key = mlx4_en_get_rss_key(priv, NULL); 1084 for (i = 0; i < 10; i++) 1085 rss_context->rss_key[i] = key[i]; 1086 1087 err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, 1088 &rss_map->indir_qp, &rss_map->indir_state); 1089 if (err) 1090 goto indir_err; 1091 1092 return 0; 1093 1094indir_err: 1095 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 1096 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 1097 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 1098 mlx4_qp_free(mdev->dev, &rss_map->indir_qp); 1099rss_err: 1100 for (i = 0; i < good_qps; i++) { 1101 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], 1102 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); 1103 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 1104 mlx4_qp_free(mdev->dev, &rss_map->qps[i]); 1105 } 1106 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); 1107 return err; 1108} 1109 1110void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) 1111{ 1112 struct mlx4_en_dev *mdev = priv->mdev; 1113 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 1114 int i; 1115 1116 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 1117 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 1118 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 1119 mlx4_qp_free(mdev->dev, &rss_map->indir_qp); 1120 1121 for (i = 0; i < priv->rx_ring_num; i++) { 1122 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], 1123 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); 1124 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 1125 mlx4_qp_free(mdev->dev, &rss_map->qps[i]); 1126 } 1127 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); 1128} 1129 1130