1/* 2 3 * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted p`rovided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 * 33 34 #include "opt_inet.h" 35 #include <linux/mlx4/cq.h> 36 #include <linux/slab.h> 37 #include <linux/mlx4/qp.h> 38 #include <linux/if_ether.h> 39 #include <linux/if_vlan.h> 40 #include <linux/vmalloc.h> 41 #include <linux/mlx4/driver.h> 42 #ifdef CONFIG_NET_RX_BUSY_POLL 43 #include <net/busy_poll.h> 44 #endif 45 46 #include "mlx4_en.h" 47 */ 48#include <asm/byteorder.h> 49 50#include <linux/mlx4/cq.h> 51#include <linux/log2.h> 52#include <linux/gfp.h> 53 54#include <debug.h> 55 56#include "mlx4_en.h" 57#include "mlx4_devif_queue.h" 58#include <net_interfaces/flags.h> 59 60static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, 61 struct mlx4_en_rx_ring *ring, int index) { 62 struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) (ring->buf 63 + (ring->stride * index)); 64 int possible_frags; 65 int i; 66 67 /*Set size and memtype fields*/ 68 rx_desc->data[0].byte_count = cpu_to_be32(0); 69 rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); 70 71 /** If the number of used fragments does not fill up the ring 72 * stride, remaining (unused) fragments must be padded with 73 * null address/size and a special memory key:*/ 74 75 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; 76 /*printf("possible_frags: %d\n", possible_frags);*/ 77 for (i = 1; i < possible_frags; i++) { 78 rx_desc->data[i].byte_count = 0; 79 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); 80 rx_desc->data[i].addr = 0; 81 } 82} 83 84// static int mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, __be64 *pdma, 85// struct mlx4_en_rx_mbuf *mb_list) { 86// 87// /*bus_dma_segment_t segs[1]; 88// bus_dmamap_t map;*/ 89// void *mb; 90// genpaddr_t t; 91// /*int nsegs; 92// int err;*/ 93// 94// /*try to allocate a new spare mbuf*/ 95// /*if (unlikely(ring->spare.mbuf == NULL)) { 96// mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); 97// if (unlikely(mb == NULL)) 98// return (-ENOMEM); 99// setup correct length 100// mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; 101// 102// make sure IP header gets aligned 103// m_adj(mb, MLX4_NET_IP_ALIGN); 104// 105// load spare mbuf into BUSDMA 106// err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, mb, 107// segs, &nsegs, BUS_DMA_NOWAIT); 108// if (unlikely(err != 0)) { 109// m_freem(mb); 110// return (err); 111// } 112// 113// store spare info 114// ring->spare.mbuf = mb; 115// ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); 116// 117// bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, 118// BUS_DMASYNC_PREREAD); 119// }*/ 120// 121// /*synchronize and unload the current mbuf, if any*/ 122// /*if (likely(mb_list->mbuf != NULL)) { 123// bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_POSTREAD); 124// bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); 125// } 126// */ 127// mb = dma_alloc(16384, &t);/*m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size);*/ 128// if (/*unlikely(*/mb == NULL/*)*/) 129// goto use_spare; 130// 131// /*setup correct length*/ 132// // mb->m_pkthdr.len = mb->m_len = MAX_ETH_FRAME_SIZE; 133// 134// /*make sure IP header gets aligned*/ 135// /*m_adj(mb, MLX4_NET_IP_ALIGN);*/ 136// 137// /*err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, mb, segs, 138// &nsegs, BUS_DMA_NOWAIT); 139// if (unlikely(err != 0)) { 140// m_freem(mb); 141// goto use_spare; 142// }*/ 143// 144// *pdma = cpu_to_be64(t); 145// mb_list->buffer = mb; 146// 147// /*bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD);*/ 148// return (0); 149// 150// use_spare: 151// /*swap DMA maps 152// map = mb_list->dma_map; 153// mb_list->dma_map = ring->spare.dma_map; 154// ring->spare.dma_map = map;*/ 155// 156// /*swap MBUFs*/ 157// /*mb_list->mbuf = ring->spare.mbuf; 158// ring->spare.mbuf = NULL;*/ 159// 160// /*store physical address*/ 161// /**pdma = ring->spare.paddr_be;*/ 162// return (0); 163// } 164/* 165 static void 166 mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) 167 { 168 bus_dmamap_t map = mb_list->dma_map; 169 bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); 170 bus_dmamap_unload(ring->dma_tag, map); 171 m_freem(mb_list->mbuf); 172 mb_list->mbuf = NULL; safety clearing 173 } 174 */ 175// static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, 176// struct mlx4_en_rx_ring *ring, int index) { 177// struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) (ring->buf 178// + (index * ring->stride)); 179// struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; 180// 181// mb_list->buffer = NULL; 182// 183// if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { 184// priv->port_stats.rx_alloc_failed++; 185// return (-ENOMEM); 186// } 187// 188// /*rx_desc->data[0].addr = 0;*/ 189// 190// return (0); 191// } 192 193static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) { 194 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); 195} 196 197// static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) { 198// struct mlx4_en_rx_ring *ring; 199// int ring_ind; 200// int buf_ind; 201// int new_size; 202// int err; 203// 204// for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) { 205// for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 206// ring = priv->rx_ring[ring_ind]; 207// 208// err = mlx4_en_prepare_rx_desc(priv, ring, ring->actual_size); 209// if (err) { 210// if (ring->actual_size == 0) { 211// MLX4_ERR("Failed to allocate " 212// "enough rx buffers\n"); 213// return -ENOMEM; 214// } else { 215// new_size = rounddown_pow_of_two(ring->actual_size); 216// MLX4_WARN("Only %d buffers allocated " 217// "reducing ring size to %d\n", ring->actual_size, 218// new_size); 219// goto reduce_rings; 220// } 221// } 222// ring->actual_size++; 223// ring->prod++; 224// } 225// } 226// return 0; 227// 228// reduce_rings: for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 229// ring = priv->rx_ring[ring_ind]; 230// while (ring->actual_size > new_size) { 231// ring->actual_size--; 232// ring->prod--; 233// /*mlx4_en_free_buf(ring, ring->mbuf + ring->actual_size);*/ 234// } 235// } 236// 237// return 0; 238// } 239 240/* 241 static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, 242 struct mlx4_en_rx_ring *ring) 243 { 244 int index; 245 246 MLX4_DEBUG( "Freeing Rx buf - cons:%d prod:%d\n", 247 ring->cons, ring->prod); 248 249 Unmap and free Rx buffers 250 BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size); 251 while (ring->cons != ring->prod) { 252 index = ring->cons & ring->size_mask; 253 MLX4_DEBUG( "Processing descriptor:%d\n", index); 254 mlx4_en_free_buf(ring, ring->mbuf + index); 255 ++ring->cons; 256 } 257 } 258 */ 259void mlx4_en_calc_rx_buf(struct mlx4_en_priv *priv) { 260 /*struct mlx4_en_priv *priv = netdev_priv(dev);*/ 261 int eff_mtu = priv->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN + 262 MLX4_NET_IP_ALIGN; 263 264 if (eff_mtu > MJUM16BYTES) { 265 MLX4_ERR("MTU(%d) is too big\n", (int )priv->if_mtu); 266 eff_mtu = MJUM16BYTES; 267 } else if (eff_mtu > MJUM9BYTES) { 268 eff_mtu = MJUM16BYTES; 269 } else if (eff_mtu > MJUMPAGESIZE) { 270 eff_mtu = MJUM9BYTES; 271 } else if (eff_mtu > MCLBYTES) { 272 eff_mtu = MJUMPAGESIZE; 273 } else { 274 eff_mtu = MCLBYTES; 275 } 276 277 priv->rx_mb_size = eff_mtu; 278 279 MLX4_DEBUG("Effective RX MTU: %d bytes\n", eff_mtu); 280} 281 282int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, 283 struct mlx4_en_rx_ring **pring, u32 size, int node) { 284 struct mlx4_en_dev *mdev = priv->mdev; 285 struct mlx4_en_rx_ring *ring; 286 int err; 287 int tmp; 288 /*uint32_t x;*/ 289 290 ring = calloc(1, sizeof(struct mlx4_en_rx_ring)); 291 if (!ring) { 292 MLX4_ERR("Failed to allocate RX ring structure\n"); 293 return -ENOMEM; 294 } 295 296 /*Create DMA descriptor TAG*/ 297 /*if ((err = -bus_dma_tag_create(bus_get_dma_tag(mdev->pdev->dev.bsddev), 1, any alignment 298 0, no boundary 299 BUS_SPACE_MAXADDR, lowaddr 300 BUS_SPACE_MAXADDR, highaddr 301 NULL, NULL, filter, filterarg 302 MJUM16BYTES, maxsize 303 1, nsegments 304 MJUM16BYTES, maxsegsize 305 0, flags 306 NULL, NULL, lockfunc, lockfuncarg 307 &ring->dma_tag))) { 308 MLX4_ERR( "Failed to create DMA tag\n"); 309 goto err_ring; 310 }*/ 311 312 ring->prod = 0; 313 ring->cons = 0; 314 ring->size = size; 315 ring->size_mask = size - 1; 316 ring->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + DS_SIZE); 317 ring->log_stride = ffs(ring->stride) - 1; 318 ring->buf_size = ring->size * ring->stride + TXBB_SIZE; 319 320 tmp = size * sizeof(struct mlx4_en_rx_mbuf); 321 322 ring->mbuf = calloc(1, tmp); 323 if (ring->mbuf == NULL) { 324 err = -ENOMEM; 325 goto err_dma_tag; 326 } 327 328 /*err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); 329 if (err != 0) 330 goto err_info; 331 332 for (x = 0; x != size; x++) { 333 err = -bus_dmamap_create(ring->dma_tag, 0, &ring->mbuf[x].dma_map); 334 if (err != 0) { 335 while (x--) 336 bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); 337 goto err_info; 338 } 339 }*/ 340 MLX4_DEBUG("Allocated MBUF ring at addr:%p size:%d\n", ring->mbuf, tmp); 341 342 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 343 2 * BASE_PAGE_SIZE); 344 if (err) 345 goto err_dma_map; 346 347 err = mlx4_en_map_buffer(&ring->wqres.buf); 348 if (err) { 349 MLX4_ERR("Failed to map RX buffer\n"); 350 goto err_hwq; 351 } 352 ring->buf = ring->wqres.buf.direct.buf; 353 *pring = ring; 354 return 0; 355 356 err_hwq: /*mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);*/ 357 err_dma_map: /*for (x = 0; x != size; x++) { 358 bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); 359 } 360 bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map);*/ 361 /*err_info:*/free(ring->mbuf); 362 err_dma_tag: /*bus_dma_tag_destroy(ring->dma_tag);*/ 363 /*err_ring:*/free(ring); 364 return (err); 365} 366 367errval_t mlx4_en_enqueue_rx(mlx4_queue_t *queue, regionid_t rid, 368 genoffset_t offset, genoffset_t length, 369 genoffset_t valid_data, genoffset_t valid_length, 370 uint64_t flags) 371{ 372 // debug_printf("%s:%s: %lx:%ld:%ld:%ld:%lx\n", queue->name, __func__, offset, length, valid_data, valid_length, flags); 373 struct mlx4_en_priv *priv = queue->priv; 374 unsigned rx_ind = 0; 375 struct mlx4_en_rx_ring *ring = priv->rx_ring[rx_ind]; 376 unsigned index = ring->prod & ring->size_mask; 377 struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) (ring->buf 378 + (index * ring->stride)); 379 struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; 380 381 rx_desc->data[0].addr = cpu_to_be64(queue->region_base + offset + valid_data); 382 rx_desc->data[0].byte_count = cpu_to_be32(valid_length); 383 mb_list->buffer = queue->region_mapped + offset + valid_data; 384 mb_list->offset = offset; 385 mb_list->length = length; 386 387 ring->prod++; 388 mlx4_en_update_rx_prod_db(ring); 389 390 // debug_printf("%s: [%d] %d:%d %d:%d %lx:%p\n", __func__, rx_ind, ring->cons, ring->prod, ring->size, ring->size_mask, queue->region_base + offset + valid_data, queue->region_mapped + offset + valid_data); 391 return SYS_ERR_OK; 392} 393 394int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { 395 struct mlx4_en_rx_ring *ring; 396 int i; 397 int ring_ind; 398 int err; 399 int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + DS_SIZE); 400 401 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 402 ring = priv->rx_ring[ring_ind]; 403 404 ring->prod = 0; 405 ring->cons = 0; 406 ring->actual_size = ring->size; 407 ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; 408 ring->rx_mb_size = priv->rx_mb_size; 409 410 ring->stride = stride; 411 if (ring->stride <= TXBB_SIZE) 412 ring->buf += TXBB_SIZE; 413 414 ring->log_stride = ffs(ring->stride) - 1; 415 ring->buf_size = ring->size * ring->stride; 416 417 memset(ring->buf, 0, ring->buf_size); 418 mlx4_en_update_rx_prod_db(ring); 419 420 /*Initialize all descriptors*/ 421 for (i = 0; i < ring->size; i++) 422 mlx4_en_init_rx_desc(priv, ring, i); 423 424#ifdef INET 425 Configure lro mngr 426 if (priv->dev->if_capenable & IFCAP_LRO) { 427 if (tcp_lro_init(&ring->lro)) 428 priv->dev->if_capenable &= ~IFCAP_LRO; 429 else 430 ring->lro.ifp = priv->dev; 431 } 432#endif 433 } 434 435 // err = mlx4_en_fill_rx_buffers(priv); 436 // if (err) 437 // goto err_buffers; 438 439 // for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { 440 // ring = priv->rx_ring[ring_ind]; 441 // 442 // ring->size_mask = ring->size - 1; 443 // mlx4_en_update_rx_prod_db(ring); 444 // } 445 446 return 0; 447 448 // err_buffers: /*for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) 449 // mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]);*/ 450 451 ring_ind = priv->rx_ring_num - 1; 452 453 while (ring_ind >= 0) { 454 ring = priv->rx_ring[ring_ind]; 455 if (ring->stride <= TXBB_SIZE) 456 ring->buf -= TXBB_SIZE; 457 ring_ind--; 458 } 459 460 return err; 461} 462/* 463 464 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, 465 struct mlx4_en_rx_ring **pring, 466 u32 size, u16 stride) 467 { 468 struct mlx4_en_dev *mdev = priv->mdev; 469 struct mlx4_en_rx_ring *ring = *pring; 470 uint32_t x; 471 472 mlx4_en_unmap_buffer(&ring->wqres.buf); 473 mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); 474 for (x = 0; x != size; x++) 475 bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); 476 free spare mbuf, if any 477 if (ring->spare.mbuf != NULL) { 478 bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, 479 BUS_DMASYNC_POSTREAD); 480 bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); 481 m_freem(ring->spare.mbuf); 482 } 483 bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); 484 vfree(ring->mbuf); 485 bus_dma_tag_destroy(ring->dma_tag); 486 kfree(ring); 487 *pring = NULL; 488 #ifdef CONFIG_RFS_ACCEL 489 mlx4_en_cleanup_filters(priv, ring); 490 #endif 491 } 492 493 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, 494 struct mlx4_en_rx_ring *ring) 495 { 496 #ifdef INET 497 tcp_lro_free(&ring->lro); 498 #endif 499 mlx4_en_free_rx_buf(priv, ring); 500 if (ring->stride <= TXBB_SIZE) 501 ring->buf -= TXBB_SIZE; 502 } 503 504 505 static void validate_loopback(struct mlx4_en_priv *priv, struct mbuf *mb) 506 { 507 int i; 508 int offset = ETHER_HDR_LEN; 509 510 for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) { 511 if (*(mb->m_data + offset) != (unsigned char) (i & 0xff)) 512 goto out_loopback; 513 } 514 Loopback found 515 priv->loopback_ok = 1; 516 517 out_loopback: 518 m_freem(mb); 519 } 520 521 */ 522static inline int invalid_cqe(struct mlx4_en_priv *priv, struct mlx4_cqe *cqe) { 523 /*Drop packet on bad receive or bad checksum*/ 524 if (/*unlikely(*/ 525 (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) 526 == MLX4_CQE_OPCODE_ERROR/*)*/) { 527 MLX4_ERR("CQE completed in error - vendor syndrom:%d syndrom:%d\n", 528 ((struct mlx4_err_cqe * )cqe)->vendor_err_syndrome, 529 ((struct mlx4_err_cqe * )cqe)->syndrome); 530 return 1; 531 } 532 if (/*unlikely(*/cqe->badfcs_enc & MLX4_CQE_BAD_FCS/*)*/) { 533 MLX4_DEBUG("Accepted frame with bad FCS\n"); 534 return 1; 535 } 536 537 return 0; 538} 539 540static void * 541mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, 542 struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, 543 int length) { 544 void *mb; 545 546 /*get mbuf*/ 547 mb = mb_list->buffer; 548 549 /*collect used fragment while atomically replacing it*/ 550 // if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) 551 // return (NULL); 552 // 553 // /*range check hardware computed value*/ 554 // if (/*unlikely(*/length > mb->m_len/*)*/) 555 // length = mb->m_len; 556 // 557 // /*update total packet length in packet header*/ 558 // mb->m_len = mb->m_pkthdr.len = length; 559 return (mb); 560} 561 562#define CQE_FACTOR_INDEX(index, factor) ((index << factor) + factor) 563 564errval_t mlx4_en_dequeue_rx(mlx4_queue_t *queue, regionid_t* rid, genoffset_t* offset, 565 genoffset_t* length, genoffset_t* valid_data, 566 genoffset_t* valid_length, uint64_t* flags) 567{ 568 struct mlx4_en_priv *priv = queue->priv; 569 struct mlx4_en_cq *cq = priv->rx_cq[0]; 570 struct mlx4_cqe *buf = cq->buf; 571 struct mlx4_en_rx_ring *ring = priv->rx_ring[0]; 572 struct mlx4_cq *mcq = &cq->mcq; 573 u32 cons_index = mcq->cons_index; 574 u32 size_mask = ring->size_mask; 575 int size = cq->size; 576 int factor = priv->cqe_factor; 577 int index; 578 struct mlx4_cqe *cqe; 579 580 index = cons_index & size_mask; 581 cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; 582 583 // printf("cqe->owner_sr_opcode %d; cons_index %d:%d; size %d; cq->buf %p; index %d; qpn %x\n", 584 // cqe->owner_sr_opcode, cons_index, ring->prod, size, cq->buf, index, cqe->vlan_my_qpn); 585 // debug_printf("%s: qpn %x\n", __func__, cqe->vlan_my_qpn); 586 587 if (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { 588 struct mlx4_en_rx_mbuf *mb_list; 589 struct mlx4_en_rx_desc *rx_desc; 590 591 mb_list = ring->mbuf + index; 592 rx_desc = (struct mlx4_en_rx_desc *) (ring->buf 593 + (index << ring->log_stride)); 594 rmb(); 595 if (invalid_cqe(priv, cqe)) { 596 assert(0); 597 } 598 599 *rid = queue->region_id; 600 *offset = mb_list->offset; 601 *length = mb_list->length; 602 *valid_data = 0; 603 *valid_length = be32_to_cpu(cqe->byte_cnt) - ring->fcs_del; 604 *flags = NETIF_RXFLAG; 605 ring->bytes += *valid_length; 606 ring->packets++; 607 608 // uint16_t *data; 609 // data = (uint16_t *)mb_list->buffer; 610 // debug_printf("Got packet: %p %04x:%04x %04x:%04x %04x:%04x %04x:%04x, %04x:%04x %04x:%04x %04x:%04x %04x:%04x, %04x:%04x %04x:%04x %04x:%04x %04x:%04x, %04x:%04x %04x:%04x %04x:%04x %04x:%04x\n", mb_list->buffer, 611 // data[0x0], data[0x1], data[0x2], data[0x3], data[0x4], data[0x5], data[0x6], data[0x7], 612 // data[0x8], data[0x9], data[0xa], data[0xb], data[0xc], data[0xd], data[0xe], data[0xf], 613 // data[0x10], data[0x11], data[0x12], data[0x13], data[0x14], data[0x15], data[0x16], data[0x17], 614 // data[0x18], data[0x19], data[0x1a], data[0x1b], data[0x1c], data[0x1d], data[0x1e], data[0x1f]); 615 616 mcq->cons_index = cons_index + 1; 617 mlx4_cq_set_ci(mcq); 618 wmb(); 619 /*ensure HW sees CQ consumer before we post new buffers*/ 620 ring->cons = mcq->cons_index; 621 } else { 622 return DEVQ_ERR_QUEUE_EMPTY; 623 } 624 // debug_print_to_log("DEQRX %d", *valid_length); 625 // debug_printf("%s: [%d] %d:%d %d:%d %lx:%p\n", __func__, 0, ring->cons, ring->prod, ring->size, ring->size_mask, queue->region_base + offset + valid_data, queue->region_mapped + offset + valid_data); 626 // debug_printf("%s:%s: %lx:%ld:%ld:%ld:%lx\n", queue->name, __func__, *offset, *length, *valid_data, *valid_length, *flags); 627 return SYS_ERR_OK; 628} 629 630/* 631 For cpu arch with cache line of 64B the performance is better when cqe size==64B 632 * To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc) 633 * was added in the beginning of each cqe (the real data is in the corresponding 32B). 634 * The following calc ensures that when factor==1, it means we are alligned to 64B 635 * and we get the real cqe data 636 */ 637int mlx4_en_process_rx_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, 638 int budget) { 639 /*struct mlx4_en_priv *priv = netdev_priv(dev);*/ 640 struct mlx4_cqe *cqe; 641 struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; 642 struct mlx4_en_rx_mbuf *mb_list; 643 struct mlx4_en_rx_desc *rx_desc; 644 void *mb; 645 struct mlx4_cq *mcq = &cq->mcq; 646 struct mlx4_cqe *buf = cq->buf; 647#ifdef INET 648 struct lro_entry *queued; 649#endif 650 int index; 651 unsigned int length; 652 int polled = 0; 653 u32 cons_index = mcq->cons_index; 654 u32 size_mask = ring->size_mask; 655 int size = cq->size; 656 int factor = priv->cqe_factor; 657 658 // uint16_t *data; 659 // int i; 660 /*data = (uint32_t *) ring->buf; 661 for (i = 0; i < ring->buf_size / 4; ++i) { 662 printf("0x%x\n", data[i]); 663 } 664 printf("\n");*/ 665 666 if (!priv->port_up) 667 return 0; 668 669 /*We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx 670 * descriptor offset can be deducted from the CQE index instead of 671 * reading 'cqe->index'*/ 672 index = cons_index & size_mask; 673 cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; 674 675 printf("cqe->owner_sr_opcode %d; cons_index %d:%d; size %d; cq->buf %p; index %d\n", 676 cqe->owner_sr_opcode, cons_index, ring->prod, size, cq->buf, index); 677 678 /*Process all completed CQEs*/ 679 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { 680 mb_list = ring->mbuf + index; 681 rx_desc = (struct mlx4_en_rx_desc *) (ring->buf 682 + (index << ring->log_stride)); 683 684 /** make sure we read the CQE after we read the ownership bit*/ 685 686 rmb(); 687 688 if (invalid_cqe(priv, cqe)) { 689 goto next; 690 } 691 uint8_t *cqe_data = (void *)cqe; 692 char line[256]; 693 int i, s = 0; 694 for (i = 0; i < 32; i++ ) { 695 s += sprintf(line + s, "%02x", cqe_data[i]); 696 if ((i & 3) == 3) 697 line[s++] = ' '; 698 } 699 line[s] = 0; 700 /** Packet is OK - process it.*/ 701 702 length = be32_to_cpu(cqe->byte_cnt); 703 length -= ring->fcs_del; 704 705 mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); 706 if (/*unlikely(*/!mb/*)*/) { 707 ring->errors++; 708 goto next; 709 } 710 711 ring->bytes += length; 712 ring->packets++; 713 714 // data = (uint16_t *) mb; 715 // debug_printf("Got packet: %p:%d: %04x:%04x %04x:%04x %04x:%04x %04x:%04x, %04x:%04x %04x:%04x %04x:%04x %04x:%04x, %04x:%04x %04x:%04x %04x:%04x %04x:%04x, %04x:%04x %04x:%04x %04x:%04x %04x:%04x\n", data, length, 716 // data[0x0], data[0x1], data[0x2], data[0x3], data[0x4], data[0x5], data[0x6], data[0x7], 717 // data[0x8], data[0x9], data[0xa], data[0xb], data[0xc], data[0xd], data[0xe], data[0xf], 718 // data[0x10], data[0x11], data[0x12], data[0x13], data[0x14], data[0x15], data[0x16], data[0x17], 719 // data[0x18], data[0x19], data[0x1a], data[0x1b], data[0x1c], data[0x1d], data[0x1e], data[0x1f]); 720 /*if (unlikely(priv->validate_loopback)) { 721 validate_loopback(priv, mb); 722 goto next; 723 }*/ 724 725 /*forward Toeplitz compatible hash value*/ 726 // mb->m_pkthdr.flowid = be32_to_cpu(cqe->immed_rss_invalid); 727 // M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 728 /*mb->m_pkthdr.rcvif = dev;*/ 729 // if (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_VLAN_PRESENT_MASK) { 730 // mb->m_pkthdr.ether_vtag= be16_to_cpu(cqe->sl_vid); 731 // mb->m_flags |= M_VLANTAG; 732 // } 733 if (/*likely(dev->if_capenable 734 & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 735 &&*/(cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) 736 && (cqe->checksum == cpu_to_be16(0xffff))) { 737 priv->port_stats.rx_chksum_good++; 738 // mb->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID 739 // | CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 740 // mb->m_pkthdr.csum_data= bswap16(0xffff); 741 /*This packet is eligible for LRO if it is: 742 * - DIX Ethernet (type interpretation) 743 * - TCP/IP (v4) 744 * - without IP options 745 * - not an IP fragment*/ 746 747// #ifdef INET 748// if (mlx4_en_can_lro(cqe->status) && 749// (dev->if_capenable & IFCAP_LRO)) { 750// if (ring->lro.lro_cnt != 0 && 751// tcp_lro_rx(&ring->lro, mb, 0) == 0) 752// goto next; 753// } 754// 755// #endif 756 /*LRO not possible, complete processing here*/ 757 INC_PERF_COUNTER(priv->pstats.lro_misses); 758 } else { 759 // mb->m_pkthdr.csum_flags = 0; 760 priv->port_stats.rx_chksum_none++; 761 } 762 763 /*Push it up the stack 764 dev->if_input(dev, mb);*/ 765 766 next: ++cons_index; 767 index = cons_index & size_mask; 768 cqe = &buf[CQE_FACTOR_INDEX(index, factor)]; 769 if (++polled == budget) 770 goto out; 771 } 772 /*Flush all pending IP reassembly sessions*/ 773 out: 774#ifdef INET 775 while ((queued = SLIST_FIRST(&ring->lro.lro_active)) != NULL) { 776 SLIST_REMOVE_HEAD(&ring->lro.lro_active, next); 777 tcp_lro_flush(&ring->lro, queued); 778 } 779#endif 780 AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); 781 mcq->cons_index = cons_index; 782 mlx4_cq_set_ci(mcq); 783 wmb(); 784 /*ensure HW sees CQ consumer before we post new buffers*/ 785 ring->cons = mcq->cons_index; 786 787 788 // ring->prod += polled; /*Polled descriptors were realocated in place*/ 789 // mlx4_en_update_rx_prod_db(ring); 790 791 792 return polled; 793 794} 795 796 797void mlx4_en_rx_irq(struct mlx4_cq *mcq) { 798 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); 799 struct mlx4_en_priv *priv = cq->dev; 800 // int done; 801 802 // Shoot one within the irq context 803 // Because there is no NAPI in freeBSD 804 // done = mlx4_en_process_rx_cq(priv, cq, MLX4_EN_RX_BUDGET); 805 // cq->tot_rx += done; 806 // if (priv->port_up && (done == MLX4_EN_RX_BUDGET)) { 807 // cq->curr_poll_rx_cpu_id = curcpu; 808 // taskqueue_enqueue(cq->tq, &cq->cq_task); 809 // } else { 810 mlx4_en_arm_cq(priv, cq); 811 // } 812} 813/* 814 void mlx4_en_rx_que(void *context, int pending) { 815 struct mlx4_en_cq *cq; 816 struct thread *td; 817 818 cq = context; 819 td = curthread; 820 821 thread_lock(td); 822 sched_bind(td, cq->curr_poll_rx_cpu_id); 823 thread_unlock(td); 824 825 while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) == MLX4_EN_RX_BUDGET) 826 ; 827 mlx4_en_arm_cq(cq->dev->if_softc, cq); 828 } 829 830 831 RSS related functions 832 */ 833static inline int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, 834 struct mlx4_en_rx_ring *ring, enum mlx4_qp_state *state, 835 struct mlx4_qp *qp) { 836 struct mlx4_en_dev *mdev = priv->mdev; 837 struct mlx4_qp_context *context; 838 int err = 0; 839 840 context = malloc(sizeof *context); 841 if (!context) { 842 MLX4_ERR("Failed to allocate qp context\n"); 843 return -ENOMEM; 844 } 845 846 err = mlx4_qp_alloc(mdev->dev, qpn, qp); 847 if (err) { 848 MLX4_ERR("Failed to allocate qp #%x\n", qpn); 849 goto out; 850 } 851 qp->event = mlx4_en_sqp_event; 852 853 memset(context, 0, sizeof *context); 854 mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, qpn, 855 ring->cqn, -1, context); 856 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma); 857 858 /*Cancel FCS removal if FW allows*/ 859 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) { 860 context->param3 |= cpu_to_be32(1 << 29); 861 ring->fcs_del = ETH_FCS_LEN; 862 } else 863 ring->fcs_del = 0; 864 865 /*ring->wqres.mtt.offset = 0xf0f0f;*/ 866 867 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state); 868 if (err) { 869 /*mlx4_qp_remove(mdev->dev, qp); 870 mlx4_qp_free(mdev->dev, qp);*/ 871 } 872 873 mlx4_en_update_rx_prod_db(ring); 874 875 out: free(context); 876 return err; 877} 878 879int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) { 880 int err; 881 int qpn; 882 883 err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0); 884 if (err) { 885 MLX4_ERR("Failed reserving drop qpn\n"); 886 return err; 887 } 888 err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp); 889 if (err) { 890 MLX4_ERR("Failed allocating drop qp\n"); 891 /*mlx4_qp_release_range(priv->mdev->dev, qpn, 1);*/ 892 return err; 893 } 894 895 return 0; 896} 897/* 898 void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) 899 { 900 u32 qpn; 901 902 qpn = priv->drop_qp.qpn; 903 mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp); 904 mlx4_qp_free(priv->mdev->dev, &priv->drop_qp); 905 mlx4_qp_release_range(priv->mdev->dev, qpn, 1); 906 } 907 908 Allocate rx qp's and configure them according to rss map 909 */ 910int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) { 911 struct mlx4_en_dev *mdev = priv->mdev; 912 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 913 struct mlx4_qp_context context; 914 /*uint32_t *data = (uint32_t *) &context;*/ 915 struct mlx4_rss_context *rss_context; 916 int rss_rings; 917 void *ptr; 918 u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 919 | MLX4_RSS_TCP_IPV6); 920 int i; 921 int err = 0; 922 int good_qps = 0; 923 static const u32 rsskey[10] = { 0xD181C62C, 0xF7F4DB5B, 0x1983A2FC, 924 0x943E1ADB, 0xD9389E6B, 0xD1039C2C, 0xA74499AD, 0x593D56D9, 925 0xF3253C06, 0x2ADC1FFC }; 926 927 MLX4_DEBUG("Configuring rss steering\n"); 928 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num, priv->rx_ring_num, 929 &rss_map->base_qpn, 0); 930 if (err) { 931 MLX4_ERR("Failed reserving %d qps\n", priv->rx_ring_num); 932 return err; 933 } 934 935 for (i = 0; i < priv->rx_ring_num; i++) { 936 priv->rx_ring[i]->qpn = rss_map->base_qpn + i; 937 err = mlx4_en_config_rss_qp(priv, priv->rx_ring[i]->qpn, 938 priv->rx_ring[i], &rss_map->state[i], &rss_map->qps[i]); 939 if (err) 940 goto rss_err; 941 942 ++good_qps; 943 } 944 945 /*Configure RSS indirection qp*/ 946 err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp); 947 if (err) { 948 MLX4_ERR("Failed to allocate RSS indirection QP\n"); 949 goto rss_err; 950 } 951 rss_map->indir_qp.event = mlx4_en_sqp_event; 952 mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn, 953 priv->rx_ring[0]->cqn, -1, &context); 954 955 if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num) 956 rss_rings = priv->rx_ring_num; 957 else 958 rss_rings = priv->prof->rss_rings; 959 960 ptr = ((u8 *) &context) + offsetof(struct mlx4_qp_context, pri_path) 961 + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; 962 rss_context = ptr; 963 rss_context->base_qpn = cpu_to_be32( 964 ilog2(rss_rings) << 24 | (rss_map->base_qpn)); 965 rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn); 966 if (priv->mdev->profile.udp_rss) { 967 rss_mask |= MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6; 968 rss_context->base_qpn_udp = rss_context->default_qpn; 969 } 970 rss_context->flags = rss_mask; 971 rss_context->hash_fn = MLX4_RSS_HASH_TOP; 972 for (i = 0; i < 10; i++) 973 rss_context->rss_key[i] = cpu_to_be32(rsskey[i]); 974 975 /*for (i = 0; i < sizeof context / 4; ++i) { 976 printf("context: 0x%x\n", data[i]); 977 }*/ 978 979 err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context, 980 &rss_map->indir_qp, &rss_map->indir_state); 981 if (err) 982 goto indir_err; 983 984 return 0; 985 986 indir_err: mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 987 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 988 /*mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 989 mlx4_qp_free(mdev->dev, &rss_map->indir_qp);*/ 990 rss_err: for (i = 0; i < good_qps; i++) { 991 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], MLX4_QP_STATE_RST, 992 NULL, 0, 0, &rss_map->qps[i]); 993 /*mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 994 mlx4_qp_free(mdev->dev, &rss_map->qps[i]);*/ 995 } 996 /*mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);*/ 997 return err; 998} 999/* 1000 void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) 1001 { 1002 struct mlx4_en_dev *mdev = priv->mdev; 1003 struct mlx4_en_rss_map *rss_map = &priv->rss_map; 1004 int i; 1005 1006 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state, 1007 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp); 1008 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp); 1009 mlx4_qp_free(mdev->dev, &rss_map->indir_qp); 1010 1011 for (i = 0; i < priv->rx_ring_num; i++) { 1012 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i], 1013 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]); 1014 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]); 1015 mlx4_qp_free(mdev->dev, &rss_map->qps[i]); 1016 } 1017 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num); 1018 } 1019 1020 */ 1021