1/**************************************************************************** 2 * Driver for Solarflare Solarstorm network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2009 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11#include <linux/socket.h> 12#include <linux/in.h> 13#include <linux/slab.h> 14#include <linux/ip.h> 15#include <linux/tcp.h> 16#include <linux/udp.h> 17#include <net/ip.h> 18#include <net/checksum.h> 19#include "net_driver.h" 20#include "efx.h" 21#include "nic.h" 22#include "selftest.h" 23#include "workarounds.h" 24 25/* Number of RX descriptors pushed at once. */ 26#define EFX_RX_BATCH 8 27 28/* Maximum size of a buffer sharing a page */ 29#define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state)) 30 31/* Size of buffer allocated for skb header area. */ 32#define EFX_SKB_HEADERS 64u 33 34/* 35 * rx_alloc_method - RX buffer allocation method 36 * 37 * This driver supports two methods for allocating and using RX buffers: 38 * each RX buffer may be backed by an skb or by an order-n page. 39 * 40 * When LRO is in use then the second method has a lower overhead, 41 * since we don't have to allocate then free skbs on reassembled frames. 42 * 43 * Values: 44 * - RX_ALLOC_METHOD_AUTO = 0 45 * - RX_ALLOC_METHOD_SKB = 1 46 * - RX_ALLOC_METHOD_PAGE = 2 47 * 48 * The heuristic for %RX_ALLOC_METHOD_AUTO is a simple hysteresis count 49 * controlled by the parameters below. 50 * 51 * - Since pushing and popping descriptors are separated by the rx_queue 52 * size, so the watermarks should be ~rxd_size. 53 * - The performance win by using page-based allocation for LRO is less 54 * than the performance hit of using page-based allocation of non-LRO, 55 * so the watermarks should reflect this. 56 * 57 * Per channel we maintain a single variable, updated by each channel: 58 * 59 * rx_alloc_level += (lro_performed ? RX_ALLOC_FACTOR_LRO : 60 * RX_ALLOC_FACTOR_SKB) 61 * Per NAPI poll interval, we constrain rx_alloc_level to 0..MAX (which 62 * limits the hysteresis), and update the allocation strategy: 63 * 64 * rx_alloc_method = (rx_alloc_level > RX_ALLOC_LEVEL_LRO ? 65 * RX_ALLOC_METHOD_PAGE : RX_ALLOC_METHOD_SKB) 66 */ 67static int rx_alloc_method = RX_ALLOC_METHOD_AUTO; 68 69#define RX_ALLOC_LEVEL_LRO 0x2000 70#define RX_ALLOC_LEVEL_MAX 0x3000 71#define RX_ALLOC_FACTOR_LRO 1 72#define RX_ALLOC_FACTOR_SKB (-2) 73 74/* This is the percentage fill level below which new RX descriptors 75 * will be added to the RX descriptor ring. 76 */ 77static unsigned int rx_refill_threshold = 90; 78 79/* This is the percentage fill level to which an RX queue will be refilled 80 * when the "RX refill threshold" is reached. 81 */ 82static unsigned int rx_refill_limit = 95; 83 84/* 85 * RX maximum head room required. 86 * 87 * This must be at least 1 to prevent overflow and at least 2 to allow 88 * pipelined receives. 89 */ 90#define EFX_RXD_HEAD_ROOM 2 91 92static inline unsigned int efx_rx_buf_offset(struct efx_rx_buffer *buf) 93{ 94 /* Offset is always within one page, so we don't need to consider 95 * the page order. 96 */ 97 return (__force unsigned long) buf->data & (PAGE_SIZE - 1); 98} 99static inline unsigned int efx_rx_buf_size(struct efx_nic *efx) 100{ 101 return PAGE_SIZE << efx->rx_buffer_order; 102} 103 104static inline u32 efx_rx_buf_hash(struct efx_rx_buffer *buf) 105{ 106#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || NET_IP_ALIGN % 4 == 0 107 return __le32_to_cpup((const __le32 *)(buf->data - 4)); 108#else 109 const u8 *data = (const u8 *)(buf->data - 4); 110 return ((u32)data[0] | 111 (u32)data[1] << 8 | 112 (u32)data[2] << 16 | 113 (u32)data[3] << 24); 114#endif 115} 116 117/** 118 * efx_init_rx_buffers_skb - create EFX_RX_BATCH skb-based RX buffers 119 * 120 * @rx_queue: Efx RX queue 121 * 122 * This allocates EFX_RX_BATCH skbs, maps them for DMA, and populates a 123 * struct efx_rx_buffer for each one. Return a negative error code or 0 124 * on success. May fail having only inserted fewer than EFX_RX_BATCH 125 * buffers. 126 */ 127static int efx_init_rx_buffers_skb(struct efx_rx_queue *rx_queue) 128{ 129 struct efx_nic *efx = rx_queue->efx; 130 struct net_device *net_dev = efx->net_dev; 131 struct efx_rx_buffer *rx_buf; 132 int skb_len = efx->rx_buffer_len; 133 unsigned index, count; 134 135 for (count = 0; count < EFX_RX_BATCH; ++count) { 136 index = rx_queue->added_count & EFX_RXQ_MASK; 137 rx_buf = efx_rx_buffer(rx_queue, index); 138 139 rx_buf->skb = netdev_alloc_skb(net_dev, skb_len); 140 if (unlikely(!rx_buf->skb)) 141 return -ENOMEM; 142 rx_buf->page = NULL; 143 144 /* Adjust the SKB for padding and checksum */ 145 skb_reserve(rx_buf->skb, NET_IP_ALIGN); 146 rx_buf->len = skb_len - NET_IP_ALIGN; 147 rx_buf->data = (char *)rx_buf->skb->data; 148 rx_buf->skb->ip_summed = CHECKSUM_UNNECESSARY; 149 150 rx_buf->dma_addr = pci_map_single(efx->pci_dev, 151 rx_buf->data, rx_buf->len, 152 PCI_DMA_FROMDEVICE); 153 if (unlikely(pci_dma_mapping_error(efx->pci_dev, 154 rx_buf->dma_addr))) { 155 dev_kfree_skb_any(rx_buf->skb); 156 rx_buf->skb = NULL; 157 return -EIO; 158 } 159 160 ++rx_queue->added_count; 161 ++rx_queue->alloc_skb_count; 162 } 163 164 return 0; 165} 166 167/** 168 * efx_init_rx_buffers_page - create EFX_RX_BATCH page-based RX buffers 169 * 170 * @rx_queue: Efx RX queue 171 * 172 * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA, 173 * and populates struct efx_rx_buffers for each one. Return a negative error 174 * code or 0 on success. If a single page can be split between two buffers, 175 * then the page will either be inserted fully, or not at at all. 176 */ 177static int efx_init_rx_buffers_page(struct efx_rx_queue *rx_queue) 178{ 179 struct efx_nic *efx = rx_queue->efx; 180 struct efx_rx_buffer *rx_buf; 181 struct page *page; 182 void *page_addr; 183 struct efx_rx_page_state *state; 184 dma_addr_t dma_addr; 185 unsigned index, count; 186 187 /* We can split a page between two buffers */ 188 BUILD_BUG_ON(EFX_RX_BATCH & 1); 189 190 for (count = 0; count < EFX_RX_BATCH; ++count) { 191 page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC, 192 efx->rx_buffer_order); 193 if (unlikely(page == NULL)) 194 return -ENOMEM; 195 dma_addr = pci_map_page(efx->pci_dev, page, 0, 196 efx_rx_buf_size(efx), 197 PCI_DMA_FROMDEVICE); 198 if (unlikely(pci_dma_mapping_error(efx->pci_dev, dma_addr))) { 199 __free_pages(page, efx->rx_buffer_order); 200 return -EIO; 201 } 202 page_addr = page_address(page); 203 state = page_addr; 204 state->refcnt = 0; 205 state->dma_addr = dma_addr; 206 207 page_addr += sizeof(struct efx_rx_page_state); 208 dma_addr += sizeof(struct efx_rx_page_state); 209 210 split: 211 index = rx_queue->added_count & EFX_RXQ_MASK; 212 rx_buf = efx_rx_buffer(rx_queue, index); 213 rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN; 214 rx_buf->skb = NULL; 215 rx_buf->page = page; 216 rx_buf->data = page_addr + EFX_PAGE_IP_ALIGN; 217 rx_buf->len = efx->rx_buffer_len - EFX_PAGE_IP_ALIGN; 218 ++rx_queue->added_count; 219 ++rx_queue->alloc_page_count; 220 ++state->refcnt; 221 222 if ((~count & 1) && (efx->rx_buffer_len <= EFX_RX_HALF_PAGE)) { 223 /* Use the second half of the page */ 224 get_page(page); 225 dma_addr += (PAGE_SIZE >> 1); 226 page_addr += (PAGE_SIZE >> 1); 227 ++count; 228 goto split; 229 } 230 } 231 232 return 0; 233} 234 235static void efx_unmap_rx_buffer(struct efx_nic *efx, 236 struct efx_rx_buffer *rx_buf) 237{ 238 if (rx_buf->page) { 239 struct efx_rx_page_state *state; 240 241 EFX_BUG_ON_PARANOID(rx_buf->skb); 242 243 state = page_address(rx_buf->page); 244 if (--state->refcnt == 0) { 245 pci_unmap_page(efx->pci_dev, 246 state->dma_addr, 247 efx_rx_buf_size(efx), 248 PCI_DMA_FROMDEVICE); 249 } 250 } else if (likely(rx_buf->skb)) { 251 pci_unmap_single(efx->pci_dev, rx_buf->dma_addr, 252 rx_buf->len, PCI_DMA_FROMDEVICE); 253 } 254} 255 256static void efx_free_rx_buffer(struct efx_nic *efx, 257 struct efx_rx_buffer *rx_buf) 258{ 259 if (rx_buf->page) { 260 __free_pages(rx_buf->page, efx->rx_buffer_order); 261 rx_buf->page = NULL; 262 } else if (likely(rx_buf->skb)) { 263 dev_kfree_skb_any(rx_buf->skb); 264 rx_buf->skb = NULL; 265 } 266} 267 268static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue, 269 struct efx_rx_buffer *rx_buf) 270{ 271 efx_unmap_rx_buffer(rx_queue->efx, rx_buf); 272 efx_free_rx_buffer(rx_queue->efx, rx_buf); 273} 274 275/* Attempt to resurrect the other receive buffer that used to share this page, 276 * which had previously been passed up to the kernel and freed. */ 277static void efx_resurrect_rx_buffer(struct efx_rx_queue *rx_queue, 278 struct efx_rx_buffer *rx_buf) 279{ 280 struct efx_rx_page_state *state = page_address(rx_buf->page); 281 struct efx_rx_buffer *new_buf; 282 unsigned fill_level, index; 283 284 /* +1 because efx_rx_packet() incremented removed_count. +1 because 285 * we'd like to insert an additional descriptor whilst leaving 286 * EFX_RXD_HEAD_ROOM for the non-recycle path */ 287 fill_level = (rx_queue->added_count - rx_queue->removed_count + 2); 288 if (unlikely(fill_level >= EFX_RXQ_SIZE - EFX_RXD_HEAD_ROOM)) { 289 /* We could place "state" on a list, and drain the list in 290 * efx_fast_push_rx_descriptors(). For now, this will do. */ 291 return; 292 } 293 294 ++state->refcnt; 295 get_page(rx_buf->page); 296 297 index = rx_queue->added_count & EFX_RXQ_MASK; 298 new_buf = efx_rx_buffer(rx_queue, index); 299 new_buf->dma_addr = rx_buf->dma_addr ^ (PAGE_SIZE >> 1); 300 new_buf->skb = NULL; 301 new_buf->page = rx_buf->page; 302 new_buf->data = (void *) 303 ((__force unsigned long)rx_buf->data ^ (PAGE_SIZE >> 1)); 304 new_buf->len = rx_buf->len; 305 ++rx_queue->added_count; 306} 307 308/* Recycle the given rx buffer directly back into the rx_queue. There is 309 * always room to add this buffer, because we've just popped a buffer. */ 310static void efx_recycle_rx_buffer(struct efx_channel *channel, 311 struct efx_rx_buffer *rx_buf) 312{ 313 struct efx_nic *efx = channel->efx; 314 struct efx_rx_queue *rx_queue = &efx->rx_queue[channel->channel]; 315 struct efx_rx_buffer *new_buf; 316 unsigned index; 317 318 if (rx_buf->page != NULL && efx->rx_buffer_len <= EFX_RX_HALF_PAGE && 319 page_count(rx_buf->page) == 1) 320 efx_resurrect_rx_buffer(rx_queue, rx_buf); 321 322 index = rx_queue->added_count & EFX_RXQ_MASK; 323 new_buf = efx_rx_buffer(rx_queue, index); 324 325 memcpy(new_buf, rx_buf, sizeof(*new_buf)); 326 rx_buf->page = NULL; 327 rx_buf->skb = NULL; 328 ++rx_queue->added_count; 329} 330 331/** 332 * efx_fast_push_rx_descriptors - push new RX descriptors quickly 333 * @rx_queue: RX descriptor queue 334 * This will aim to fill the RX descriptor queue up to 335 * @rx_queue->@fast_fill_limit. If there is insufficient atomic 336 * memory to do so, a slow fill will be scheduled. 337 * 338 * The caller must provide serialisation (none is used here). In practise, 339 * this means this function must run from the NAPI handler, or be called 340 * when NAPI is disabled. 341 */ 342void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue) 343{ 344 struct efx_channel *channel = rx_queue->channel; 345 unsigned fill_level; 346 int space, rc = 0; 347 348 /* Calculate current fill level, and exit if we don't need to fill */ 349 fill_level = (rx_queue->added_count - rx_queue->removed_count); 350 EFX_BUG_ON_PARANOID(fill_level > EFX_RXQ_SIZE); 351 if (fill_level >= rx_queue->fast_fill_trigger) 352 goto out; 353 354 /* Record minimum fill level */ 355 if (unlikely(fill_level < rx_queue->min_fill)) { 356 if (fill_level) 357 rx_queue->min_fill = fill_level; 358 } 359 360 space = rx_queue->fast_fill_limit - fill_level; 361 if (space < EFX_RX_BATCH) 362 goto out; 363 364 netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, 365 "RX queue %d fast-filling descriptor ring from" 366 " level %d to level %d using %s allocation\n", 367 rx_queue->queue, fill_level, rx_queue->fast_fill_limit, 368 channel->rx_alloc_push_pages ? "page" : "skb"); 369 370 do { 371 if (channel->rx_alloc_push_pages) 372 rc = efx_init_rx_buffers_page(rx_queue); 373 else 374 rc = efx_init_rx_buffers_skb(rx_queue); 375 if (unlikely(rc)) { 376 /* Ensure that we don't leave the rx queue empty */ 377 if (rx_queue->added_count == rx_queue->removed_count) 378 efx_schedule_slow_fill(rx_queue); 379 goto out; 380 } 381 } while ((space -= EFX_RX_BATCH) >= EFX_RX_BATCH); 382 383 netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, 384 "RX queue %d fast-filled descriptor ring " 385 "to level %d\n", rx_queue->queue, 386 rx_queue->added_count - rx_queue->removed_count); 387 388 out: 389 if (rx_queue->notified_count != rx_queue->added_count) 390 efx_nic_notify_rx_desc(rx_queue); 391} 392 393void efx_rx_slow_fill(unsigned long context) 394{ 395 struct efx_rx_queue *rx_queue = (struct efx_rx_queue *)context; 396 struct efx_channel *channel = rx_queue->channel; 397 398 /* Post an event to cause NAPI to run and refill the queue */ 399 efx_nic_generate_fill_event(channel); 400 ++rx_queue->slow_fill_count; 401} 402 403static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue, 404 struct efx_rx_buffer *rx_buf, 405 int len, bool *discard, 406 bool *leak_packet) 407{ 408 struct efx_nic *efx = rx_queue->efx; 409 unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding; 410 411 if (likely(len <= max_len)) 412 return; 413 414 /* The packet must be discarded, but this is only a fatal error 415 * if the caller indicated it was 416 */ 417 *discard = true; 418 419 if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) { 420 if (net_ratelimit()) 421 netif_err(efx, rx_err, efx->net_dev, 422 " RX queue %d seriously overlength " 423 "RX event (0x%x > 0x%x+0x%x). Leaking\n", 424 rx_queue->queue, len, max_len, 425 efx->type->rx_buffer_padding); 426 /* If this buffer was skb-allocated, then the meta 427 * data at the end of the skb will be trashed. So 428 * we have no choice but to leak the fragment. 429 */ 430 *leak_packet = (rx_buf->skb != NULL); 431 efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY); 432 } else { 433 if (net_ratelimit()) 434 netif_err(efx, rx_err, efx->net_dev, 435 " RX queue %d overlength RX event " 436 "(0x%x > 0x%x)\n", 437 rx_queue->queue, len, max_len); 438 } 439 440 rx_queue->channel->n_rx_overlength++; 441} 442 443/* Pass a received packet up through the generic LRO stack 444 * 445 * Handles driverlink veto, and passes the fragment up via 446 * the appropriate LRO method 447 */ 448static void efx_rx_packet_lro(struct efx_channel *channel, 449 struct efx_rx_buffer *rx_buf, 450 bool checksummed) 451{ 452 struct napi_struct *napi = &channel->napi_str; 453 gro_result_t gro_result; 454 455 /* Pass the skb/page into the LRO engine */ 456 if (rx_buf->page) { 457 struct efx_nic *efx = channel->efx; 458 struct page *page = rx_buf->page; 459 struct sk_buff *skb; 460 461 EFX_BUG_ON_PARANOID(rx_buf->skb); 462 rx_buf->page = NULL; 463 464 skb = napi_get_frags(napi); 465 if (!skb) { 466 put_page(page); 467 return; 468 } 469 470 if (efx->net_dev->features & NETIF_F_RXHASH) 471 skb->rxhash = efx_rx_buf_hash(rx_buf); 472 473 skb_shinfo(skb)->frags[0].page = page; 474 skb_shinfo(skb)->frags[0].page_offset = 475 efx_rx_buf_offset(rx_buf); 476 skb_shinfo(skb)->frags[0].size = rx_buf->len; 477 skb_shinfo(skb)->nr_frags = 1; 478 479 skb->len = rx_buf->len; 480 skb->data_len = rx_buf->len; 481 skb->truesize += rx_buf->len; 482 skb->ip_summed = 483 checksummed ? CHECKSUM_UNNECESSARY : CHECKSUM_NONE; 484 485 skb_record_rx_queue(skb, channel->channel); 486 487 gro_result = napi_gro_frags(napi); 488 } else { 489 struct sk_buff *skb = rx_buf->skb; 490 491 EFX_BUG_ON_PARANOID(!skb); 492 EFX_BUG_ON_PARANOID(!checksummed); 493 rx_buf->skb = NULL; 494 495 gro_result = napi_gro_receive(napi, skb); 496 } 497 498 if (gro_result == GRO_NORMAL) { 499 channel->rx_alloc_level += RX_ALLOC_FACTOR_SKB; 500 } else if (gro_result != GRO_DROP) { 501 channel->rx_alloc_level += RX_ALLOC_FACTOR_LRO; 502 channel->irq_mod_score += 2; 503 } 504} 505 506void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, 507 unsigned int len, bool checksummed, bool discard) 508{ 509 struct efx_nic *efx = rx_queue->efx; 510 struct efx_channel *channel = rx_queue->channel; 511 struct efx_rx_buffer *rx_buf; 512 bool leak_packet = false; 513 514 rx_buf = efx_rx_buffer(rx_queue, index); 515 EFX_BUG_ON_PARANOID(!rx_buf->data); 516 EFX_BUG_ON_PARANOID(rx_buf->skb && rx_buf->page); 517 EFX_BUG_ON_PARANOID(!(rx_buf->skb || rx_buf->page)); 518 519 /* This allows the refill path to post another buffer. 520 * EFX_RXD_HEAD_ROOM ensures that the slot we are using 521 * isn't overwritten yet. 522 */ 523 rx_queue->removed_count++; 524 525 /* Validate the length encoded in the event vs the descriptor pushed */ 526 efx_rx_packet__check_len(rx_queue, rx_buf, len, 527 &discard, &leak_packet); 528 529 netif_vdbg(efx, rx_status, efx->net_dev, 530 "RX queue %d received id %x at %llx+%x %s%s\n", 531 rx_queue->queue, index, 532 (unsigned long long)rx_buf->dma_addr, len, 533 (checksummed ? " [SUMMED]" : ""), 534 (discard ? " [DISCARD]" : "")); 535 536 /* Discard packet, if instructed to do so */ 537 if (unlikely(discard)) { 538 if (unlikely(leak_packet)) 539 channel->n_skbuff_leaks++; 540 else 541 efx_recycle_rx_buffer(channel, rx_buf); 542 543 /* Don't hold off the previous receive */ 544 rx_buf = NULL; 545 goto out; 546 } 547 548 /* Release card resources - assumes all RX buffers consumed in-order 549 * per RX queue 550 */ 551 efx_unmap_rx_buffer(efx, rx_buf); 552 553 /* Prefetch nice and early so data will (hopefully) be in cache by 554 * the time we look at it. 555 */ 556 prefetch(rx_buf->data); 557 558 /* Pipeline receives so that we give time for packet headers to be 559 * prefetched into cache. 560 */ 561 rx_buf->len = len; 562out: 563 if (rx_queue->channel->rx_pkt) 564 __efx_rx_packet(rx_queue->channel, 565 rx_queue->channel->rx_pkt, 566 rx_queue->channel->rx_pkt_csummed); 567 rx_queue->channel->rx_pkt = rx_buf; 568 rx_queue->channel->rx_pkt_csummed = checksummed; 569} 570 571/* Handle a received packet. Second half: Touches packet payload. */ 572void __efx_rx_packet(struct efx_channel *channel, 573 struct efx_rx_buffer *rx_buf, bool checksummed) 574{ 575 struct efx_nic *efx = channel->efx; 576 struct sk_buff *skb; 577 578 rx_buf->data += efx->type->rx_buffer_hash_size; 579 rx_buf->len -= efx->type->rx_buffer_hash_size; 580 581 /* If we're in loopback test, then pass the packet directly to the 582 * loopback layer, and free the rx_buf here 583 */ 584 if (unlikely(efx->loopback_selftest)) { 585 efx_loopback_rx_packet(efx, rx_buf->data, rx_buf->len); 586 efx_free_rx_buffer(efx, rx_buf); 587 return; 588 } 589 590 if (rx_buf->skb) { 591 prefetch(skb_shinfo(rx_buf->skb)); 592 593 skb_reserve(rx_buf->skb, efx->type->rx_buffer_hash_size); 594 skb_put(rx_buf->skb, rx_buf->len); 595 596 if (efx->net_dev->features & NETIF_F_RXHASH) 597 rx_buf->skb->rxhash = efx_rx_buf_hash(rx_buf); 598 599 /* Move past the ethernet header. rx_buf->data still points 600 * at the ethernet header */ 601 rx_buf->skb->protocol = eth_type_trans(rx_buf->skb, 602 efx->net_dev); 603 604 skb_record_rx_queue(rx_buf->skb, channel->channel); 605 } 606 607 if (likely(checksummed || rx_buf->page)) { 608 efx_rx_packet_lro(channel, rx_buf, checksummed); 609 return; 610 } 611 612 /* We now own the SKB */ 613 skb = rx_buf->skb; 614 rx_buf->skb = NULL; 615 EFX_BUG_ON_PARANOID(!skb); 616 617 /* Set the SKB flags */ 618 skb->ip_summed = CHECKSUM_NONE; 619 620 /* Pass the packet up */ 621 netif_receive_skb(skb); 622 623 /* Update allocation strategy method */ 624 channel->rx_alloc_level += RX_ALLOC_FACTOR_SKB; 625} 626 627void efx_rx_strategy(struct efx_channel *channel) 628{ 629 enum efx_rx_alloc_method method = rx_alloc_method; 630 631 /* Only makes sense to use page based allocation if LRO is enabled */ 632 if (!(channel->efx->net_dev->features & NETIF_F_GRO)) { 633 method = RX_ALLOC_METHOD_SKB; 634 } else if (method == RX_ALLOC_METHOD_AUTO) { 635 /* Constrain the rx_alloc_level */ 636 if (channel->rx_alloc_level < 0) 637 channel->rx_alloc_level = 0; 638 else if (channel->rx_alloc_level > RX_ALLOC_LEVEL_MAX) 639 channel->rx_alloc_level = RX_ALLOC_LEVEL_MAX; 640 641 /* Decide on the allocation method */ 642 method = ((channel->rx_alloc_level > RX_ALLOC_LEVEL_LRO) ? 643 RX_ALLOC_METHOD_PAGE : RX_ALLOC_METHOD_SKB); 644 } 645 646 /* Push the option */ 647 channel->rx_alloc_push_pages = (method == RX_ALLOC_METHOD_PAGE); 648} 649 650int efx_probe_rx_queue(struct efx_rx_queue *rx_queue) 651{ 652 struct efx_nic *efx = rx_queue->efx; 653 unsigned int rxq_size; 654 int rc; 655 656 netif_dbg(efx, probe, efx->net_dev, 657 "creating RX queue %d\n", rx_queue->queue); 658 659 /* Allocate RX buffers */ 660 rxq_size = EFX_RXQ_SIZE * sizeof(*rx_queue->buffer); 661 rx_queue->buffer = kzalloc(rxq_size, GFP_KERNEL); 662 if (!rx_queue->buffer) 663 return -ENOMEM; 664 665 rc = efx_nic_probe_rx(rx_queue); 666 if (rc) { 667 kfree(rx_queue->buffer); 668 rx_queue->buffer = NULL; 669 } 670 return rc; 671} 672 673void efx_init_rx_queue(struct efx_rx_queue *rx_queue) 674{ 675 unsigned int max_fill, trigger, limit; 676 677 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, 678 "initialising RX queue %d\n", rx_queue->queue); 679 680 /* Initialise ptr fields */ 681 rx_queue->added_count = 0; 682 rx_queue->notified_count = 0; 683 rx_queue->removed_count = 0; 684 rx_queue->min_fill = -1U; 685 rx_queue->min_overfill = -1U; 686 687 /* Initialise limit fields */ 688 max_fill = EFX_RXQ_SIZE - EFX_RXD_HEAD_ROOM; 689 trigger = max_fill * min(rx_refill_threshold, 100U) / 100U; 690 limit = max_fill * min(rx_refill_limit, 100U) / 100U; 691 692 rx_queue->max_fill = max_fill; 693 rx_queue->fast_fill_trigger = trigger; 694 rx_queue->fast_fill_limit = limit; 695 696 /* Set up RX descriptor ring */ 697 efx_nic_init_rx(rx_queue); 698} 699 700void efx_fini_rx_queue(struct efx_rx_queue *rx_queue) 701{ 702 int i; 703 struct efx_rx_buffer *rx_buf; 704 705 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, 706 "shutting down RX queue %d\n", rx_queue->queue); 707 708 del_timer_sync(&rx_queue->slow_fill); 709 efx_nic_fini_rx(rx_queue); 710 711 /* Release RX buffers NB start at index 0 not current HW ptr */ 712 if (rx_queue->buffer) { 713 for (i = 0; i <= EFX_RXQ_MASK; i++) { 714 rx_buf = efx_rx_buffer(rx_queue, i); 715 efx_fini_rx_buffer(rx_queue, rx_buf); 716 } 717 } 718} 719 720void efx_remove_rx_queue(struct efx_rx_queue *rx_queue) 721{ 722 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, 723 "destroying RX queue %d\n", rx_queue->queue); 724 725 efx_nic_remove_rx(rx_queue); 726 727 kfree(rx_queue->buffer); 728 rx_queue->buffer = NULL; 729} 730 731 732module_param(rx_alloc_method, int, 0644); 733MODULE_PARM_DESC(rx_alloc_method, "Allocation method used for RX buffers"); 734 735module_param(rx_refill_threshold, uint, 0444); 736MODULE_PARM_DESC(rx_refill_threshold, 737 "RX descriptor ring fast/slow fill threshold (%)"); 738