1/* 2 * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33#include "opt_inet.h" 34#include <linux/mlx4/cq.h> 35#include <linux/slab.h> 36#include <linux/mlx4/qp.h> 37#include <linux/if_ether.h> 38#include <linux/if_vlan.h> 39#include <linux/vmalloc.h> 40#include <linux/mlx4/driver.h> 41#ifdef CONFIG_NET_RX_BUSY_POLL 42#include <net/busy_poll.h> 43#endif 44 45#include "mlx4_en.h" 46 47 48static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, 49 struct mlx4_en_rx_ring *ring, 50 int index) 51{ 52 struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) 53 (ring->buf + (ring->stride * index)); 54 int possible_frags; 55 int i; 56 57 /* Set size and memtype fields */ 58 rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size); 59 rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); 60 61 /* 62 * If the number of used fragments does not fill up the ring 63 * stride, remaining (unused) fragments must be padded with 64 * null address/size and a special memory key: 65 */ 66 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; 67 for (i = 1; i < possible_frags; i++) { 68 rx_desc->data[i].byte_count = 0; 69 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); 70 rx_desc->data[i].addr = 0; 71 } 72} 73 74static int 75mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, 76 __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list) 77{ 78 bus_dma_segment_t segs[1]; 79 bus_dmamap_t map; 80 struct mbuf *mb; 81 int nsegs; 82 int err; 83 84 /* try to allocate a new spare mbuf */ 85 if (unlikely(ring->spare.mbuf == NULL)) { 86 mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); 87 if (unlikely(mb == NULL)) 88 return (-ENOMEM); 89 /* setup correct length */ 90 mb->m_len = ring->rx_mb_size; 91 92 /* load spare mbuf into BUSDMA */ 93 err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, 94 mb, segs, &nsegs, BUS_DMA_NOWAIT); 95 if (unlikely(err != 0)) { 96 m_freem(mb); 97 return (err); 98 } 99 100 /* store spare info */ 101 ring->spare.mbuf = mb; 102 ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); 103 104 bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, 105 BUS_DMASYNC_PREREAD); 106 } 107 108 /* synchronize and unload the current mbuf, if any */ 109 if (likely(mb_list->mbuf != NULL)) { 110 bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, 111 BUS_DMASYNC_POSTREAD); 112 bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); 113 } 114 115 mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); 116 if (unlikely(mb == NULL)) 117 goto use_spare; 118 119 /* setup correct length */ 120 mb->m_len = ring->rx_mb_size; 121 122 err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, 123 mb, segs, &nsegs, BUS_DMA_NOWAIT); 124 if (unlikely(err != 0)) { 125 m_freem(mb); 126 goto use_spare; 127 } 128 129 *pdma = cpu_to_be64(segs[0].ds_addr); 130 mb_list->mbuf = mb; 131 132 bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); 133 return (0); 134 135use_spare: 136 /* swap DMA maps */ 137 map = mb_list->dma_map; 138 mb_list->dma_map = ring->spare.dma_map; 139 ring->spare.dma_map = map; 140 141 /* swap MBUFs */ 142 mb_list->mbuf = ring->spare.mbuf; 143 ring->spare.mbuf = NULL; 144 145 /* store physical address */ 146 *pdma = ring->spare.paddr_be; 147 return (0); 148} 149 150static void 151mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) 152{ 153 bus_dmamap_t map = mb_list->dma_map; 154 bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); 155 bus_dmamap_unload(ring->dma_tag, map); 156 m_freem(mb_list->mbuf); 157 mb_list->mbuf = NULL; /* safety clearing */ 158} 159 160static int 161mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, 162 struct mlx4_en_rx_ring *ring, int index) 163{ 164 struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) 165 (ring->buf + (index * ring->stride)); 166 struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; 167 168 mb_list->mbuf = NULL; 169 170 if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { 171 priv->port_stats.rx_alloc_failed++; 172 return (-ENOMEM); 173 } 174 return (0); 175} 176 177static inline void 178mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) 179{ 180 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); 181} 182 183static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) 184{ 185 struct mlx4_en_rx_ring *ring; 186 int ring_ind; 187 int buf_ind; 188 int new_size; 189 int err; 190 191 for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { 192 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 193 ring = priv->rx_ring[ring_ind]; 194 195 err = mlx4_en_prepare_rx_desc(priv, ring, 196 ring->actual_size); 197 if (err) { 198 if (ring->actual_size == 0) { 199 en_err(priv, "Failed to allocate " 200 "enough rx buffers\n"); 201 return -ENOMEM; 202 } else { 203 new_size = 204 rounddown_pow_of_two(ring->actual_size); 205 en_warn(priv, "Only %d buffers allocated " 206 "reducing ring size to %d\n", 207 ring->actual_size, new_size); 208 goto reduce_rings; 209 } 210 } 211 ring->actual_size++; 212 ring->prod++; 213 } 214 } 215 return 0; 216 217reduce_rings: 218 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 219 ring = priv->rx_ring[ring_ind]; 220 while (ring->actual_size > new_size) { 221 ring->actual_size--; 222 ring->prod--; 223 mlx4_en_free_buf(ring, 224 ring->mbuf + ring->actual_size); 225 } 226 } 227 228 return 0; 229} 230 231static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, 232 struct mlx4_en_rx_ring *ring) 233{ 234 int index; 235 236 en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n", 237 ring->cons, ring->prod); 238 239 /* Unmap and free Rx buffers */ 240 BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); 241 while (ring->cons != ring->prod) { 242 index = ring->cons & ring->size_mask; 243 en_dbg(DRV, priv, "Processing descriptor:%d\n", index); 244 mlx4_en_free_buf(ring, ring->mbuf + index); 245 ++ring->cons; 246 } 247} 248 249void mlx4_en_calc_rx_buf(struct net_device *dev) 250{ 251 struct mlx4_en_priv *priv = netdev_priv(dev); 252 int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN; 253 254 if (eff_mtu > MJUM16BYTES) { 255 en_err(priv, "MTU(%d) is too big\n", (int)dev->if_mtu); 256 eff_mtu = MJUM16BYTES; 257 } else if (eff_mtu > MJUM9BYTES) { 258 eff_mtu = MJUM16BYTES; 259 } else if (eff_mtu > MJUMPAGESIZE) { 260 eff_mtu = MJUM9BYTES; 261 } else if (eff_mtu > MCLBYTES) { 262 eff_mtu = MJUMPAGESIZE; 263 } else { 264 eff_mtu = MCLBYTES; 265 } 266 267 priv->rx_mb_size = eff_mtu; 268 269 en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu); 270} 271 272int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, 273 struct mlx4_en_rx_ring **pring, 274 u32 size, int node) 275{ 276 struct mlx4_en_dev *mdev = priv->mdev; 277 struct mlx4_en_rx_ring *ring; 278 int err; 279 int tmp; 280 uint32_t x; 281 282 ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); 283 if (!ring) { 284 en_err(priv, "Failed to allocate RX ring structure\n"); 285 return -ENOMEM; 286 } 287 288 /* Create DMA descriptor TAG */ 289 if ((err = -bus_dma_tag_create( 290 bus_get_dma_tag(mdev->pdev->dev.bsddev), 291 1, /* any alignment */ 292 0, /* no boundary */ 293 BUS_SPACE_MAXADDR, /* lowaddr */ 294 BUS_SPACE_MAXADDR, /* highaddr */ 295 NULL, NULL, /* filter, filterarg */ 296 MJUM16BYTES, /* maxsize */ 297 1, /* nsegments */ 298 MJUM16BYTES, /* maxsegsize */ 299 0, /* flags */ 300 NULL, NULL, /* lockfunc, lockfuncarg */ 301 &ring->dma_tag))) { 302 en_err(priv, "Failed to create DMA tag\n"); 303 goto err_ring; 304 } 305 306 ring->prod = 0; 307 ring->cons = 0; 308 ring->size = size; 309 ring->size_mask = size - 1; 310 ring->stride = roundup_pow_of_two( 311 sizeof(struct mlx4_en_rx_desc) + DS_SIZE); 312 ring->log_stride = ffs(ring->stride) - 1; 313 ring->buf_size = ring->size * ring->stride + TXBB_SIZE; 314 315 tmp = size * sizeof(struct mlx4_en_rx_mbuf); 316 317 ring->mbuf = kzalloc(tmp, GFP_KERNEL); 318 if (ring->mbuf == NULL) { 319 err = -ENOMEM; 320 goto err_dma_tag; 321 } 322 323 err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); 324 if (err != 0) 325 goto err_info; 326 327 for (x = 0; x != size; x++) { 328 err = -bus_dmamap_create(ring->dma_tag, 0, 329 &ring->mbuf[x].dma_map); 330 if (err != 0) { 331 while (x--) 332 bus_dmamap_destroy(ring->dma_tag, 333 ring->mbuf[x].dma_map); 334 goto err_info; 335 } 336 } 337 en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n", 338 ring->mbuf, tmp); 339 340 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, 341 ring->buf_size, 2 * PAGE_SIZE); 342 if (err) 343 goto err_dma_map; 344 345 err = mlx4_en_map_buffer(&ring->wqres.buf); 346 if (err) { 347 en_err(priv, "Failed to map RX buffer\n"); 348 goto err_hwq; 349 } 350 ring->buf = ring->wqres.buf.direct.buf; 351 *pring = ring; 352 return 0; 353 354err_hwq: 355 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); 356err_dma_map: 357 for (x = 0; x != size; x++) { 358 bus_dmamap_destroy(ring->dma_tag, 359 ring->mbuf[x].dma_map); 360 } 361 bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); 362err_info: 363 vfree(ring->mbuf); 364err_dma_tag: 365 bus_dma_tag_destroy(ring->dma_tag); 366err_ring: 367 kfree(ring); 368 return (err); 369} 370 371int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) 372{ 373 struct mlx4_en_rx_ring *ring; 374 int i; 375 int ring_ind; 376 int err; 377 int stride = roundup_pow_of_two( 378 sizeof(struct mlx4_en_rx_desc) + DS_SIZE); 379 380 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 381 ring = priv->rx_ring[ring_ind]; 382 383 ring->prod = 0; 384 ring->cons = 0; 385 ring->actual_size = 0; 386 ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; 387 ring->rx_alloc_order = priv->rx_alloc_order; 388 ring->rx_alloc_size = priv->rx_alloc_size; 389 ring->rx_buf_size = priv->rx_buf_size; 390 ring->rx_mb_size = priv->rx_mb_size; 391 392 ring->stride = stride; 393 if (ring->stride <= TXBB_SIZE) 394 ring->buf += TXBB_SIZE; 395 396 ring->log_stride = ffs(ring->stride) - 1; 397 ring->buf_size = ring->size * ring->stride; 398 399 memset(ring->buf, 0, ring->buf_size); 400 mlx4_en_update_rx_prod_db(ring); 401 402 /* Initialize all descriptors */ 403 for (i = 0; i < ring->size; i++) 404 mlx4_en_init_rx_desc(priv, ring, i); 405 406#ifdef INET 407 /* Configure lro mngr */ 408 if (priv->dev->if_capenable & IFCAP_LRO) { 409 if (tcp_lro_init(&ring->lro)) 410 priv->dev->if_capenable &= ~IFCAP_LRO; 411 else 412 ring->lro.ifp = priv->dev; 413 } 414#endif 415 } 416 417 418 err = mlx4_en_fill_rx_buffers(priv); 419 if (err) 420 goto err_buffers; 421 422 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 423 ring = priv->rx_ring[ring_ind]; 424 425 ring->size_mask = ring->actual_size - 1; 426 mlx4_en_update_rx_prod_db(ring); 427 } 428 429 return 0; 430 431err_buffers: 432 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) 433 mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]); 434 435 ring_ind = priv->rx_ring_num - 1; 436 437 while (ring_ind >= 0) { 438 ring = priv->rx_ring[ring_ind]; 439 if (ring->stride <= TXBB_SIZE) 440 ring->buf -= TXBB_SIZE; 441 ring_ind--; 442 } 443 444 return err; 445} 446 447 448void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, 449 struct mlx4_en_rx_ring **pring, 450 u32 size, u16 stride) 451{ 452 struct mlx4_en_dev *mdev = priv->mdev; 453 struct mlx4_en_rx_ring *ring = *pring; 454 uint32_t x; 455 456 mlx4_en_unmap_buffer(&ring->wqres.buf); 457 mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); 458 for (x = 0; x != size; x++) 459 bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); 460 /* free spare mbuf, if any */ 461 if (ring->spare.mbuf != NULL) { 462 bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, 463 BUS_DMASYNC_POSTREAD); 464 bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); 465 m_freem(ring->spare.mbuf); 466 } 467 bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); 468 vfree(ring->mbuf); 469 bus_dma_tag_destroy(ring->dma_tag); 470 kfree(ring); 471 *pring = NULL; 472#ifdef CONFIG_RFS_ACCEL 473 mlx4_en_cleanup_filters(priv, ring); 474#endif 475} 476 477void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, 478 struct mlx4_en_rx_ring *ring) 479{ 480#ifdef INET 481 tcp_lro_free(&ring->lro); 482#endif 483 mlx4_en_free_rx_buf(priv, ring); 484 if (ring->stride <= TXBB_SIZE) 485 ring->buf -= TXBB_SIZE; 486} 487 488 489static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) 490{ 491 int i; 492 int offset = ETHER_HDR_LEN; 493 494 for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { 495 if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) 496 goto out_loopback; 497 } 498 /* Loopback found */ 499 priv->loopback_ok = 1; 500 501out_loopback: 502 m_freem(mb); 503} 504 505 506static inline int invalid_cqe(struct mlx4_en_priv *priv, 507 struct mlx4_cqe *cqe) 508{ 509 /* Drop packet on bad receive or bad checksum */ 510 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == 511 MLX4_CQE_OPCODE_ERROR)) { 512 en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n", 513 ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome, 514 ((struct mlx4_err_cqe *)cqe)->syndrome); 515 return 1; 516 } 517 if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) { 518 en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n"); 519 return 1; 520 } 521 522 return 0; 523} 524 525static struct mbuf * 526mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, 527 struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, 528 int length) 529{ 530 struct mbuf *mb; 531 532 /* get mbuf */ 533 mb = mb_list->mbuf; 534 535 /* collect used fragment while atomically replacing it */ 536 if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) 537 return (NULL); 538 539 /* range check hardware computed value */ 540 if (unlikely(length > mb->m_len)) 541 length = mb->m_len; 542 543 /* update total packet length in packet header */ 544 mb->m_len = mb->m_pkthdr.len = length; 545 return (mb); 546} 547 548/* For cpu arch with cache line of 64B the performance is better when cqe size==64B 549 * To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc) 550 * was added in the beginning of each cqe (the real data is in the corresponding 32B). 551 * The following calc ensures that when factor==1, it means we are alligned to 64B 552 * and we get the real cqe data*/ 553#define CQE_FACTOR_INDEX(index, factor) ((index << factor) + factor) 554int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) 555{ 556 struct mlx4_en_priv *priv = netdev_priv(dev); 557 struct mlx4_cqe *cqe; 558 struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; 559 struct mlx4_en_rx_mbuf *mb_list; 560 struct mlx4_en_rx_desc *rx_desc; 561 struct mbuf *mb; 562 struct mlx4_cq *mcq = &cq->mcq; 563 struct mlx4_cqe *buf = cq->buf; 564#ifdef INET 565 struct lro_entry *queued; 566#endif 567 int index; 568 unsigned int length; 569 int polled = 0; 570 u32 cons_index = mcq->cons_index; 571 u32 size_mask = ring->size_mask; 572 int size = cq->size; 573 int factor = priv->cqe_factor; 574 575 if (!priv->port_up) 576 return 0; 577 578 /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx 579 * descriptor offset can be deducted from the CQE index instead of 580 * reading 'cqe->index' */ 581 index = cons_index & size_mask; 582 cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; 583 584 /* Process all completed CQEs */ 585 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, 586 cons_index & size)) { 587 mb_list = ring->mbuf + index; 588 rx_desc = (struct mlx4_en_rx_desc *) 589 (ring->buf + (index << ring->log_stride)); 590 591 /* 592 * make sure we read the CQE after we read the ownership bit 593 */ 594 rmb(); 595 596 if (invalid_cqe(priv, cqe)) { 597 goto next; 598 } 599 /* 600 * Packet is OK - process it. 601 */ 602 length = be32_to_cpu(cqe->byte_cnt); 603 length -= ring->fcs_del; 604 605 mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); 606 if (unlikely(!mb)) { 607 ring->errors++; 608 goto next; 609 } 610 611 ring->bytes += length; 612 ring->packets++; 613 614 if (unlikely(priv->validate_loopback)) { 615 validate_loopback(priv, mb); 616 goto next; 617 } 618 619 mb->m_pkthdr.flowid = cq->ring; 620 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 621 mb->m_pkthdr.rcvif = dev; 622 if (be32_to_cpu(cqe->vlan_my_qpn) & 623 MLX4_CQE_VLAN_PRESENT_MASK) { 624 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); 625 mb->m_flags |= M_VLANTAG; 626 } 627 if (likely(dev->if_capenable & 628 (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && 629 (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && 630 (cqe->checksum == cpu_to_be16(0xffff))) { 631 priv->port_stats.rx_chksum_good++; 632 mb->m_pkthdr.csum_flags = 633 CSUM_IP_CHECKED | CSUM_IP_VALID | 634 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 635 mb->m_pkthdr.csum_data = htons(0xffff); 636 /* This packet is eligible for LRO if it is: 637 * - DIX Ethernet (type interpretation) 638 * - TCP/IP (v4) 639 * - without IP options 640 * - not an IP fragment 641 */ 642#ifdef INET 643 if (mlx4_en_can_lro(cqe->status) && 644 (dev->if_capenable & IFCAP_LRO)) { 645 if (ring->lro.lro_cnt != 0 && 646 tcp_lro_rx(&ring->lro, mb, 0) == 0) 647 goto next; 648 } 649 650#endif 651 /* LRO not possible, complete processing here */ 652 INC_PERF_COUNTER(priv->pstats.lro_misses); 653 } else { 654 mb->m_pkthdr.csum_flags = 0; 655 priv->port_stats.rx_chksum_none++; 656 } 657 658 /* Push it up the stack */ 659 dev->if_input(dev, mb); 660 661next: 662 ++cons_index; 663 index = cons_index & size_mask; 664 cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; 665 if (++polled == budget) 666 goto out; 667 } 668 /* Flush all pending IP reassembly sessions */ 669out: 670#ifdef INET 671 while ((queued = SLIST_FIRST(&ring->lro.lro_active)) != NULL) { 672 SLIST_REMOVE_HEAD(&ring->lro.lro_active, next); 673 tcp_lro_flush(&ring->lro, queued); 674 } 675#endif 676 AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); 677 mcq->cons_index = cons_index; 678 mlx4_cq_set_ci(mcq); 679 wmb(); /* ensure HW sees CQ consumer before we post new buffers */ 680 ring->cons = mcq->cons_index; 681 ring->prod += polled; /* Polled descriptors were realocated in place */ 682 mlx4_en_update_rx_prod_db(ring); 683 return polled; 684 685} 686 687/* Rx CQ polling - called by NAPI */ 688static int mlx4_en_poll_rx_cq(struct mlx4_en_cq *cq, int budget) 689{ 690 struct net_device *dev = cq->dev; 691 int done; 692 693 done = mlx4_en_process_rx_cq(dev, cq, budget); 694 cq->tot_rx += done; 695 696 return done; 697 698} 699void mlx4_en_rx_irq(struct mlx4_cq *mcq) 700{ 701 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); 702 struct mlx4_en_priv *priv = netdev_priv(cq->dev); 703 int done; 704 705 // Shoot one within the irq context 706 // Because there is no NAPI in freeBSD 707 done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); 708 if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { 709 cq->curr_poll_rx_cpu_id = curcpu; 710 taskqueue_enqueue(cq->tq, &cq->cq_task); 711 } 712 else { 713 mlx4_en_arm_cq(priv, cq); 714 } 715} 716 717void mlx4_en_rx_que(void *context, int pending) 718{ 719 struct mlx4_en_cq *cq; 720 struct thread *td; 721 722 cq = context; 723 td = curthread; 724 725 thread_lock(td); 726 sched_bind(td, cq->curr_poll_rx_cpu_id); 727 thread_unlock(td); 728 729 while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) 730 == MLX4_EN_RX_BUDGET); 731 mlx4_en_arm_cq(cq->dev->if_softc, cq); 732} 733 734 735/* RSS related functions */ 736 737static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, 738 struct mlx4_en_rx_ring *ring, 739 enum mlx4_qp_state *state, 740 struct mlx4_qp *qp) 741{ 742 struct mlx4_en_dev *mdev = priv->mdev; 743 struct mlx4_qp_context *context; 744 int err = 0; 745 746 context = kmalloc(sizeof *context , GFP_KERNEL); 747 if (!context) { 748 en_err(priv, "Failed to allocate qp context\n"); 749 return -ENOMEM; 750 } 751 752 err = mlx4_qp_alloc(mdev->dev, qpn, qp); 753 if (err) { 754 en_err(priv, "Failed to allocate qp #%x\n", qpn); 755 goto out; 756 } 757 qp->event = mlx4_en_sqp_event; 758 759 memset(context, 0, sizeof *context); 760 mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, 761 qpn, ring->cqn, -1, context); 762 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); 763 764 /* Cancel FCS removal if FW allows */ 765 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { 766 context->param3 |= cpu_to_be32(1 << 29); 767 ring->fcs_del = ETH_FCS_LEN; 768 } else 769 ring->fcs_del = 0; 770 771 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); 772 if (err) { 773 mlx4_qp_remove(mdev->dev, qp); 774 mlx4_qp_free(mdev->dev, qp); 775 } 776 mlx4_en_update_rx_prod_db(ring); 777out: 778 kfree(context); 779 return err; 780} 781 782int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) 783{ 784 int err; 785 u32 qpn; 786 787 err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0); 788 if (err) { 789 en_err(priv, "Failed reserving drop qpn\n"); 790 return err; 791 } 792 err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp); 793 if (err) { 794 en_err(priv, "Failed allocating drop qp\n"); 795 mlx4_qp_release_range(priv->mdev->dev, qpn, 1); 796 return err; 797 } 798 799 return 0; 800} 801 802void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) 803{ 804 u32 qpn; 805 806 qpn = priv->drop_qp.qpn; 807 mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); 808 mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); 809 mlx4_qp_release_range(priv->mdev->dev, qpn, 1); 810} 811 812/* Allocate rx qp's and configure them according to rss map */ 813int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) 814{ 815 struct mlx4_en_dev *mdev = priv->mdev; 816 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 817 struct mlx4_qp_context context; 818 struct mlx4_rss_context *rss_context; 819 int rss_rings; 820 void *ptr; 821 u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 | 822 MLX4_RSS_TCP_IPV6); 823 int i; 824 int err = 0; 825 int good_qps = 0; 826 static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, 0x1983A2FC, 827 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, 0xA74499AD, 828 0x593D56D9, 0xF3253C06, 0x2ADC1FFC}; 829 830 en_dbg(DRV, priv, "Configuring rss steering\n"); 831 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, 832 priv->rx_ring_num, 833 &rss_map->base_qpn, 0); 834 if (err) { 835 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num); 836 return err; 837 } 838 839 for (i = 0; i < priv->rx_ring_num; i++) { 840 priv->rx_ring[i]->qpn = rss_map->base_qpn + i; 841 err = mlx4_en_config_rss_qp(priv, priv->rx_ring[i]->qpn, 842 priv->rx_ring[i], 843 &rss_map->state[i], 844 &rss_map->qps[i]); 845 if (err) 846 goto rss_err; 847 848 ++good_qps; 849 } 850 851 /* Configure RSS indirection qp */ 852 err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp); 853 if (err) { 854 en_err(priv, "Failed to allocate RSS indirection QP\n"); 855 goto rss_err; 856 } 857 rss_map->indir_qp.event = mlx4_en_sqp_event; 858 mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, 859 priv->rx_ring[0]->cqn, -1, &context); 860 861 if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) 862 rss_rings = priv->rx_ring_num; 863 else 864 rss_rings = priv->prof->rss_rings; 865 866 ptr = ((u8 *)&context) + offsetof(struct mlx4_qp_context, pri_path) + 867 MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; 868 rss_context = ptr; 869 rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | 870 (rss_map->base_qpn)); 871 rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); 872 if (priv->mdev->profile.udp_rss) { 873 rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; 874 rss_context->base_qpn_udp = rss_context->default_qpn; 875 } 876 rss_context->flags = rss_mask; 877 rss_context->hash_fn = MLX4_RSS_HASH_TOP; 878 for (i = 0; i < 10; i++) 879 rss_context->rss_key[i] = cpu_to_be32(rsskey[i]); 880 881 err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, 882 &rss_map->indir_qp, &rss_map->indir_state); 883 if (err) 884 goto indir_err; 885 886 return 0; 887 888indir_err: 889 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 890 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 891 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 892 mlx4_qp_free(mdev->dev, &rss_map->indir_qp); 893rss_err: 894 for (i = 0; i < good_qps; i++) { 895 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], 896 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); 897 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 898 mlx4_qp_free(mdev->dev, &rss_map->qps[i]); 899 } 900 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); 901 return err; 902} 903 904void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) 905{ 906 struct mlx4_en_dev *mdev = priv->mdev; 907 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 908 int i; 909 910 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 911 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 912 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 913 mlx4_qp_free(mdev->dev, &rss_map->indir_qp); 914 915 for (i = 0; i < priv->rx_ring_num; i++) { 916 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], 917 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); 918 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 919 mlx4_qp_free(mdev->dev, &rss_map->qps[i]); 920 } 921 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); 922} 923 924