1 2 3 4#include <linux/etherdevice.h> 5#include <linux/ip.h> 6#include <linux/ipv6.h> 7#include <net/ip6_checksum.h> 8#include "bnx2x_cmn.h" 9 10#ifdef BCM_VLAN 11#include <linux/if_vlan.h> 12#endif 13 14static int bnx2x_poll(struct napi_struct *napi, int budget); 15 16/* free skb in the packet ring at pos idx 17 * return idx of last bd freed 18 */ 19static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp, 20 u16 idx) 21{ 22 struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx]; 23 struct eth_tx_start_bd *tx_start_bd; 24 struct eth_tx_bd *tx_data_bd; 25 struct sk_buff *skb = tx_buf->skb; 26 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons; 27 int nbd; 28 29 /* prefetch skb end pointer to speedup dev_kfree_skb() */ 30 prefetch(&skb->end); 31 32 DP(BNX2X_MSG_OFF, "pkt_idx %d buff @(%p)->skb %p\n", 33 idx, tx_buf, skb); 34 35 /* unmap first bd */ 36 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx); 37 tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd; 38 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd), 39 BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE); 40 41 nbd = le16_to_cpu(tx_start_bd->nbd) - 1; 42#ifdef BNX2X_STOP_ON_ERROR 43 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) { 44 BNX2X_ERR("BAD nbd!\n"); 45 bnx2x_panic(); 46 } 47#endif 48 new_cons = nbd + tx_buf->first_bd; 49 50 /* Get the next bd */ 51 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 52 53 /* Skip a parse bd... */ 54 --nbd; 55 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 56 57 /* ...and the TSO split header bd since they have no mapping */ 58 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) { 59 --nbd; 60 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 61 } 62 63 /* now free frags */ 64 while (nbd > 0) { 65 66 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx); 67 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd; 68 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd), 69 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE); 70 if (--nbd) 71 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx)); 72 } 73 74 /* release skb */ 75 WARN_ON(!skb); 76 dev_kfree_skb(skb); 77 tx_buf->first_bd = 0; 78 tx_buf->skb = NULL; 79 80 return new_cons; 81} 82 83int bnx2x_tx_int(struct bnx2x_fastpath *fp) 84{ 85 struct bnx2x *bp = fp->bp; 86 struct netdev_queue *txq; 87 u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons; 88 89#ifdef BNX2X_STOP_ON_ERROR 90 if (unlikely(bp->panic)) 91 return -1; 92#endif 93 94 txq = netdev_get_tx_queue(bp->dev, fp->index); 95 hw_cons = le16_to_cpu(*fp->tx_cons_sb); 96 sw_cons = fp->tx_pkt_cons; 97 98 while (sw_cons != hw_cons) { 99 u16 pkt_cons; 100 101 pkt_cons = TX_BD(sw_cons); 102 103 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */ 104 105 DP(NETIF_MSG_TX_DONE, "hw_cons %u sw_cons %u pkt_cons %u\n", 106 hw_cons, sw_cons, pkt_cons); 107 108/* if (NEXT_TX_IDX(sw_cons) != hw_cons) { 109 rmb(); 110 prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb); 111 } 112*/ 113 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons); 114 sw_cons++; 115 } 116 117 fp->tx_pkt_cons = sw_cons; 118 fp->tx_bd_cons = bd_cons; 119 120 /* Need to make the tx_bd_cons update visible to start_xmit() 121 * before checking for netif_tx_queue_stopped(). Without the 122 * memory barrier, there is a small possibility that 123 * start_xmit() will miss it and cause the queue to be stopped 124 * forever. 125 */ 126 smp_mb(); 127 128 /* TBD need a thresh? */ 129 if (unlikely(netif_tx_queue_stopped(txq))) { 130 /* Taking tx_lock() is needed to prevent reenabling the queue 131 * while it's empty. This could have happen if rx_action() gets 132 * suspended in bnx2x_tx_int() after the condition before 133 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()): 134 * 135 * stops the queue->sees fresh tx_bd_cons->releases the queue-> 136 * sends some packets consuming the whole queue again-> 137 * stops the queue 138 */ 139 140 __netif_tx_lock(txq, smp_processor_id()); 141 142 if ((netif_tx_queue_stopped(txq)) && 143 (bp->state == BNX2X_STATE_OPEN) && 144 (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)) 145 netif_tx_wake_queue(txq); 146 147 __netif_tx_unlock(txq); 148 } 149 return 0; 150} 151 152static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp, 153 u16 idx) 154{ 155 u16 last_max = fp->last_max_sge; 156 157 if (SUB_S16(idx, last_max) > 0) 158 fp->last_max_sge = idx; 159} 160 161static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp, 162 struct eth_fast_path_rx_cqe *fp_cqe) 163{ 164 struct bnx2x *bp = fp->bp; 165 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) - 166 le16_to_cpu(fp_cqe->len_on_bd)) >> 167 SGE_PAGE_SHIFT; 168 u16 last_max, last_elem, first_elem; 169 u16 delta = 0; 170 u16 i; 171 172 if (!sge_len) 173 return; 174 175 /* First mark all used pages */ 176 for (i = 0; i < sge_len; i++) 177 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i]))); 178 179 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n", 180 sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1])); 181 182 /* Here we assume that the last SGE index is the biggest */ 183 prefetch((void *)(fp->sge_mask)); 184 bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1])); 185 186 last_max = RX_SGE(fp->last_max_sge); 187 last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT; 188 first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT; 189 190 /* If ring is not full */ 191 if (last_elem + 1 != first_elem) 192 last_elem++; 193 194 /* Now update the prod */ 195 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) { 196 if (likely(fp->sge_mask[i])) 197 break; 198 199 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK; 200 delta += RX_SGE_MASK_ELEM_SZ; 201 } 202 203 if (delta > 0) { 204 fp->rx_sge_prod += delta; 205 /* clear page-end entries */ 206 bnx2x_clear_sge_mask_next_elems(fp); 207 } 208 209 DP(NETIF_MSG_RX_STATUS, 210 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n", 211 fp->last_max_sge, fp->rx_sge_prod); 212} 213 214static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue, 215 struct sk_buff *skb, u16 cons, u16 prod) 216{ 217 struct bnx2x *bp = fp->bp; 218 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons]; 219 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod]; 220 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod]; 221 dma_addr_t mapping; 222 223 /* move empty skb from pool to prod and map it */ 224 prod_rx_buf->skb = fp->tpa_pool[queue].skb; 225 mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data, 226 bp->rx_buf_size, DMA_FROM_DEVICE); 227 dma_unmap_addr_set(prod_rx_buf, mapping, mapping); 228 229 /* move partial skb from cons to pool (don't unmap yet) */ 230 fp->tpa_pool[queue] = *cons_rx_buf; 231 232 /* mark bin state as start - print error if current state != stop */ 233 if (fp->tpa_state[queue] != BNX2X_TPA_STOP) 234 BNX2X_ERR("start of bin not in stop [%d]\n", queue); 235 236 fp->tpa_state[queue] = BNX2X_TPA_START; 237 238 /* point prod_bd to new skb */ 239 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 240 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 241 242#ifdef BNX2X_STOP_ON_ERROR 243 fp->tpa_queue_used |= (1 << queue); 244#ifdef _ASM_GENERIC_INT_L64_H 245 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n", 246#else 247 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n", 248#endif 249 fp->tpa_queue_used); 250#endif 251} 252 253static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp, 254 struct sk_buff *skb, 255 struct eth_fast_path_rx_cqe *fp_cqe, 256 u16 cqe_idx) 257{ 258 struct sw_rx_page *rx_pg, old_rx_pg; 259 u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd); 260 u32 i, frag_len, frag_size, pages; 261 int err; 262 int j; 263 264 frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd; 265 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT; 266 267 /* This is needed in order to enable forwarding support */ 268 if (frag_size) 269 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE, 270 max(frag_size, (u32)len_on_bd)); 271 272#ifdef BNX2X_STOP_ON_ERROR 273 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) { 274 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n", 275 pages, cqe_idx); 276 BNX2X_ERR("fp_cqe->pkt_len = %d fp_cqe->len_on_bd = %d\n", 277 fp_cqe->pkt_len, len_on_bd); 278 bnx2x_panic(); 279 return -EINVAL; 280 } 281#endif 282 283 /* Run through the SGL and compose the fragmented skb */ 284 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) { 285 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j])); 286 287 /* FW gives the indices of the SGE as if the ring is an array 288 (meaning that "next" element will consume 2 indices) */ 289 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE)); 290 rx_pg = &fp->rx_page_ring[sge_idx]; 291 old_rx_pg = *rx_pg; 292 293 /* If we fail to allocate a substitute page, we simply stop 294 where we are and drop the whole packet */ 295 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx); 296 if (unlikely(err)) { 297 fp->eth_q_stats.rx_skb_alloc_failed++; 298 return err; 299 } 300 301 /* Unmap the page as we r going to pass it to the stack */ 302 dma_unmap_page(&bp->pdev->dev, 303 dma_unmap_addr(&old_rx_pg, mapping), 304 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE); 305 306 /* Add one frag and update the appropriate fields in the skb */ 307 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len); 308 309 skb->data_len += frag_len; 310 skb->truesize += frag_len; 311 skb->len += frag_len; 312 313 frag_size -= frag_len; 314 } 315 316 return 0; 317} 318 319static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp, 320 u16 queue, int pad, int len, union eth_rx_cqe *cqe, 321 u16 cqe_idx) 322{ 323 struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue]; 324 struct sk_buff *skb = rx_buf->skb; 325 /* alloc new skb */ 326 struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size); 327 328 /* Unmap skb in the pool anyway, as we are going to change 329 pool entry status to BNX2X_TPA_STOP even if new skb allocation 330 fails. */ 331 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping), 332 bp->rx_buf_size, DMA_FROM_DEVICE); 333 334 if (likely(new_skb)) { 335 /* fix ip xsum and give it to the stack */ 336 /* (no need to map the new skb) */ 337#ifdef BCM_VLAN 338 int is_vlan_cqe = 339 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) & 340 PARSING_FLAGS_VLAN); 341 int is_not_hwaccel_vlan_cqe = 342 (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG))); 343#endif 344 345 prefetch(skb); 346 prefetch(((char *)(skb)) + 128); 347 348#ifdef BNX2X_STOP_ON_ERROR 349 if (pad + len > bp->rx_buf_size) { 350 BNX2X_ERR("skb_put is about to fail... " 351 "pad %d len %d rx_buf_size %d\n", 352 pad, len, bp->rx_buf_size); 353 bnx2x_panic(); 354 return; 355 } 356#endif 357 358 skb_reserve(skb, pad); 359 skb_put(skb, len); 360 361 skb->protocol = eth_type_trans(skb, bp->dev); 362 skb->ip_summed = CHECKSUM_UNNECESSARY; 363 364 { 365 struct iphdr *iph; 366 367 iph = (struct iphdr *)skb->data; 368#ifdef BCM_VLAN 369 /* If there is no Rx VLAN offloading - 370 take VLAN tag into an account */ 371 if (unlikely(is_not_hwaccel_vlan_cqe)) 372 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN); 373#endif 374 iph->check = 0; 375 iph->check = ip_fast_csum((u8 *)iph, iph->ihl); 376 } 377 378 if (!bnx2x_fill_frag_skb(bp, fp, skb, 379 &cqe->fast_path_cqe, cqe_idx)) { 380#ifdef BCM_VLAN 381 if ((bp->vlgrp != NULL) && is_vlan_cqe && 382 (!is_not_hwaccel_vlan_cqe)) 383 vlan_gro_receive(&fp->napi, bp->vlgrp, 384 le16_to_cpu(cqe->fast_path_cqe. 385 vlan_tag), skb); 386 else 387#endif 388 napi_gro_receive(&fp->napi, skb); 389 } else { 390 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages" 391 " - dropping packet!\n"); 392 dev_kfree_skb(skb); 393 } 394 395 396 /* put new skb in bin */ 397 fp->tpa_pool[queue].skb = new_skb; 398 399 } else { 400 /* else drop the packet and keep the buffer in the bin */ 401 DP(NETIF_MSG_RX_STATUS, 402 "Failed to allocate new skb - dropping packet!\n"); 403 fp->eth_q_stats.rx_skb_alloc_failed++; 404 } 405 406 fp->tpa_state[queue] = BNX2X_TPA_STOP; 407} 408 409/* Set Toeplitz hash value in the skb using the value from the 410 * CQE (calculated by HW). 411 */ 412static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe, 413 struct sk_buff *skb) 414{ 415 /* Set Toeplitz hash from CQE */ 416 if ((bp->dev->features & NETIF_F_RXHASH) && 417 (cqe->fast_path_cqe.status_flags & 418 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) 419 skb->rxhash = 420 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result); 421} 422 423int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget) 424{ 425 struct bnx2x *bp = fp->bp; 426 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons; 427 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod; 428 int rx_pkt = 0; 429 430#ifdef BNX2X_STOP_ON_ERROR 431 if (unlikely(bp->panic)) 432 return 0; 433#endif 434 435 /* CQ "next element" is of the size of the regular element, 436 that's why it's ok here */ 437 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb); 438 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT) 439 hw_comp_cons++; 440 441 bd_cons = fp->rx_bd_cons; 442 bd_prod = fp->rx_bd_prod; 443 bd_prod_fw = bd_prod; 444 sw_comp_cons = fp->rx_comp_cons; 445 sw_comp_prod = fp->rx_comp_prod; 446 447 /* Memory barrier necessary as speculative reads of the rx 448 * buffer can be ahead of the index in the status block 449 */ 450 rmb(); 451 452 DP(NETIF_MSG_RX_STATUS, 453 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n", 454 fp->index, hw_comp_cons, sw_comp_cons); 455 456 while (sw_comp_cons != hw_comp_cons) { 457 struct sw_rx_bd *rx_buf = NULL; 458 struct sk_buff *skb; 459 union eth_rx_cqe *cqe; 460 u8 cqe_fp_flags; 461 u16 len, pad; 462 463 comp_ring_cons = RCQ_BD(sw_comp_cons); 464 bd_prod = RX_BD(bd_prod); 465 bd_cons = RX_BD(bd_cons); 466 467 /* Prefetch the page containing the BD descriptor 468 at producer's index. It will be needed when new skb is 469 allocated */ 470 prefetch((void *)(PAGE_ALIGN((unsigned long) 471 (&fp->rx_desc_ring[bd_prod])) - 472 PAGE_SIZE + 1)); 473 474 cqe = &fp->rx_comp_ring[comp_ring_cons]; 475 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags; 476 477 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x" 478 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags), 479 cqe_fp_flags, cqe->fast_path_cqe.status_flags, 480 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result), 481 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), 482 le16_to_cpu(cqe->fast_path_cqe.pkt_len)); 483 484 /* is this a slowpath msg? */ 485 if (unlikely(CQE_TYPE(cqe_fp_flags))) { 486 bnx2x_sp_event(fp, cqe); 487 goto next_cqe; 488 489 /* this is an rx packet */ 490 } else { 491 rx_buf = &fp->rx_buf_ring[bd_cons]; 492 skb = rx_buf->skb; 493 prefetch(skb); 494 len = le16_to_cpu(cqe->fast_path_cqe.pkt_len); 495 pad = cqe->fast_path_cqe.placement_offset; 496 497 /* If CQE is marked both TPA_START and TPA_END 498 it is a non-TPA CQE */ 499 if ((!fp->disable_tpa) && 500 (TPA_TYPE(cqe_fp_flags) != 501 (TPA_TYPE_START | TPA_TYPE_END))) { 502 u16 queue = cqe->fast_path_cqe.queue_index; 503 504 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) { 505 DP(NETIF_MSG_RX_STATUS, 506 "calling tpa_start on queue %d\n", 507 queue); 508 509 bnx2x_tpa_start(fp, queue, skb, 510 bd_cons, bd_prod); 511 512 /* Set Toeplitz hash for an LRO skb */ 513 bnx2x_set_skb_rxhash(bp, cqe, skb); 514 515 goto next_rx; 516 } 517 518 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) { 519 DP(NETIF_MSG_RX_STATUS, 520 "calling tpa_stop on queue %d\n", 521 queue); 522 523 if (!BNX2X_RX_SUM_FIX(cqe)) 524 BNX2X_ERR("STOP on none TCP " 525 "data\n"); 526 527 /* This is a size of the linear data 528 on this skb */ 529 len = le16_to_cpu(cqe->fast_path_cqe. 530 len_on_bd); 531 bnx2x_tpa_stop(bp, fp, queue, pad, 532 len, cqe, comp_ring_cons); 533#ifdef BNX2X_STOP_ON_ERROR 534 if (bp->panic) 535 return 0; 536#endif 537 538 bnx2x_update_sge_prod(fp, 539 &cqe->fast_path_cqe); 540 goto next_cqe; 541 } 542 } 543 544 dma_sync_single_for_device(&bp->pdev->dev, 545 dma_unmap_addr(rx_buf, mapping), 546 pad + RX_COPY_THRESH, 547 DMA_FROM_DEVICE); 548 prefetch(((char *)(skb)) + 128); 549 550 /* is this an error packet? */ 551 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) { 552 DP(NETIF_MSG_RX_ERR, 553 "ERROR flags %x rx packet %u\n", 554 cqe_fp_flags, sw_comp_cons); 555 fp->eth_q_stats.rx_err_discard_pkt++; 556 goto reuse_rx; 557 } 558 559 /* Since we don't have a jumbo ring 560 * copy small packets if mtu > 1500 561 */ 562 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) && 563 (len <= RX_COPY_THRESH)) { 564 struct sk_buff *new_skb; 565 566 new_skb = netdev_alloc_skb(bp->dev, 567 len + pad); 568 if (new_skb == NULL) { 569 DP(NETIF_MSG_RX_ERR, 570 "ERROR packet dropped " 571 "because of alloc failure\n"); 572 fp->eth_q_stats.rx_skb_alloc_failed++; 573 goto reuse_rx; 574 } 575 576 /* aligned copy */ 577 skb_copy_from_linear_data_offset(skb, pad, 578 new_skb->data + pad, len); 579 skb_reserve(new_skb, pad); 580 skb_put(new_skb, len); 581 582 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod); 583 584 skb = new_skb; 585 586 } else 587 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) { 588 dma_unmap_single(&bp->pdev->dev, 589 dma_unmap_addr(rx_buf, mapping), 590 bp->rx_buf_size, 591 DMA_FROM_DEVICE); 592 skb_reserve(skb, pad); 593 skb_put(skb, len); 594 595 } else { 596 DP(NETIF_MSG_RX_ERR, 597 "ERROR packet dropped because " 598 "of alloc failure\n"); 599 fp->eth_q_stats.rx_skb_alloc_failed++; 600reuse_rx: 601 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod); 602 goto next_rx; 603 } 604 605 skb->protocol = eth_type_trans(skb, bp->dev); 606 607 /* Set Toeplitz hash for a none-LRO skb */ 608 bnx2x_set_skb_rxhash(bp, cqe, skb); 609 610 skb->ip_summed = CHECKSUM_NONE; 611 if (bp->rx_csum) { 612 if (likely(BNX2X_RX_CSUM_OK(cqe))) 613 skb->ip_summed = CHECKSUM_UNNECESSARY; 614 else 615 fp->eth_q_stats.hw_csum_err++; 616 } 617 } 618 619 skb_record_rx_queue(skb, fp->index); 620 621#ifdef BCM_VLAN 622 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) && 623 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) & 624 PARSING_FLAGS_VLAN)) 625 vlan_gro_receive(&fp->napi, bp->vlgrp, 626 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb); 627 else 628#endif 629 napi_gro_receive(&fp->napi, skb); 630 631 632next_rx: 633 rx_buf->skb = NULL; 634 635 bd_cons = NEXT_RX_IDX(bd_cons); 636 bd_prod = NEXT_RX_IDX(bd_prod); 637 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw); 638 rx_pkt++; 639next_cqe: 640 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod); 641 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons); 642 643 if (rx_pkt == budget) 644 break; 645 } /* while */ 646 647 fp->rx_bd_cons = bd_cons; 648 fp->rx_bd_prod = bd_prod_fw; 649 fp->rx_comp_cons = sw_comp_cons; 650 fp->rx_comp_prod = sw_comp_prod; 651 652 /* Update producers */ 653 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod, 654 fp->rx_sge_prod); 655 656 fp->rx_pkt += rx_pkt; 657 fp->rx_calls++; 658 659 return rx_pkt; 660} 661 662static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie) 663{ 664 struct bnx2x_fastpath *fp = fp_cookie; 665 struct bnx2x *bp = fp->bp; 666 667 /* Return here if interrupt is disabled */ 668 if (unlikely(atomic_read(&bp->intr_sem) != 0)) { 669 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n"); 670 return IRQ_HANDLED; 671 } 672 673 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n", 674 fp->index, fp->sb_id); 675 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0); 676 677#ifdef BNX2X_STOP_ON_ERROR 678 if (unlikely(bp->panic)) 679 return IRQ_HANDLED; 680#endif 681 682 /* Handle Rx and Tx according to MSI-X vector */ 683 prefetch(fp->rx_cons_sb); 684 prefetch(fp->tx_cons_sb); 685 prefetch(&fp->status_blk->u_status_block.status_block_index); 686 prefetch(&fp->status_blk->c_status_block.status_block_index); 687 napi_schedule(&bnx2x_fp(bp, fp->index, napi)); 688 689 return IRQ_HANDLED; 690} 691 692 693/* HW Lock for shared dual port PHYs */ 694void bnx2x_acquire_phy_lock(struct bnx2x *bp) 695{ 696 mutex_lock(&bp->port.phy_mutex); 697 698 if (bp->port.need_hw_lock) 699 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 700} 701 702void bnx2x_release_phy_lock(struct bnx2x *bp) 703{ 704 if (bp->port.need_hw_lock) 705 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO); 706 707 mutex_unlock(&bp->port.phy_mutex); 708} 709 710void bnx2x_link_report(struct bnx2x *bp) 711{ 712 if (bp->flags & MF_FUNC_DIS) { 713 netif_carrier_off(bp->dev); 714 netdev_err(bp->dev, "NIC Link is Down\n"); 715 return; 716 } 717 718 if (bp->link_vars.link_up) { 719 u16 line_speed; 720 721 if (bp->state == BNX2X_STATE_OPEN) 722 netif_carrier_on(bp->dev); 723 netdev_info(bp->dev, "NIC Link is Up, "); 724 725 line_speed = bp->link_vars.line_speed; 726 if (IS_E1HMF(bp)) { 727 u16 vn_max_rate; 728 729 vn_max_rate = 730 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >> 731 FUNC_MF_CFG_MAX_BW_SHIFT) * 100; 732 if (vn_max_rate < line_speed) 733 line_speed = vn_max_rate; 734 } 735 pr_cont("%d Mbps ", line_speed); 736 737 if (bp->link_vars.duplex == DUPLEX_FULL) 738 pr_cont("full duplex"); 739 else 740 pr_cont("half duplex"); 741 742 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) { 743 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) { 744 pr_cont(", receive "); 745 if (bp->link_vars.flow_ctrl & 746 BNX2X_FLOW_CTRL_TX) 747 pr_cont("& transmit "); 748 } else { 749 pr_cont(", transmit "); 750 } 751 pr_cont("flow control ON"); 752 } 753 pr_cont("\n"); 754 755 } else { /* link_down */ 756 netif_carrier_off(bp->dev); 757 netdev_err(bp->dev, "NIC Link is Down\n"); 758 } 759} 760 761void bnx2x_init_rx_rings(struct bnx2x *bp) 762{ 763 int func = BP_FUNC(bp); 764 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 : 765 ETH_MAX_AGGREGATION_QUEUES_E1H; 766 u16 ring_prod, cqe_ring_prod; 767 int i, j; 768 769 bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN; 770 DP(NETIF_MSG_IFUP, 771 "mtu %d rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size); 772 773 if (bp->flags & TPA_ENABLE_FLAG) { 774 775 for_each_queue(bp, j) { 776 struct bnx2x_fastpath *fp = &bp->fp[j]; 777 778 for (i = 0; i < max_agg_queues; i++) { 779 fp->tpa_pool[i].skb = 780 netdev_alloc_skb(bp->dev, bp->rx_buf_size); 781 if (!fp->tpa_pool[i].skb) { 782 BNX2X_ERR("Failed to allocate TPA " 783 "skb pool for queue[%d] - " 784 "disabling TPA on this " 785 "queue!\n", j); 786 bnx2x_free_tpa_pool(bp, fp, i); 787 fp->disable_tpa = 1; 788 break; 789 } 790 dma_unmap_addr_set((struct sw_rx_bd *) 791 &bp->fp->tpa_pool[i], 792 mapping, 0); 793 fp->tpa_state[i] = BNX2X_TPA_STOP; 794 } 795 } 796 } 797 798 for_each_queue(bp, j) { 799 struct bnx2x_fastpath *fp = &bp->fp[j]; 800 801 fp->rx_bd_cons = 0; 802 fp->rx_cons_sb = BNX2X_RX_SB_INDEX; 803 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX; 804 805 /* "next page" elements initialization */ 806 /* SGE ring */ 807 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) { 808 struct eth_rx_sge *sge; 809 810 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2]; 811 sge->addr_hi = 812 cpu_to_le32(U64_HI(fp->rx_sge_mapping + 813 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 814 sge->addr_lo = 815 cpu_to_le32(U64_LO(fp->rx_sge_mapping + 816 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES))); 817 } 818 819 bnx2x_init_sge_ring_bit_mask(fp); 820 821 /* RX BD ring */ 822 for (i = 1; i <= NUM_RX_RINGS; i++) { 823 struct eth_rx_bd *rx_bd; 824 825 rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2]; 826 rx_bd->addr_hi = 827 cpu_to_le32(U64_HI(fp->rx_desc_mapping + 828 BCM_PAGE_SIZE*(i % NUM_RX_RINGS))); 829 rx_bd->addr_lo = 830 cpu_to_le32(U64_LO(fp->rx_desc_mapping + 831 BCM_PAGE_SIZE*(i % NUM_RX_RINGS))); 832 } 833 834 /* CQ ring */ 835 for (i = 1; i <= NUM_RCQ_RINGS; i++) { 836 struct eth_rx_cqe_next_page *nextpg; 837 838 nextpg = (struct eth_rx_cqe_next_page *) 839 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1]; 840 nextpg->addr_hi = 841 cpu_to_le32(U64_HI(fp->rx_comp_mapping + 842 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 843 nextpg->addr_lo = 844 cpu_to_le32(U64_LO(fp->rx_comp_mapping + 845 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS))); 846 } 847 848 /* Allocate SGEs and initialize the ring elements */ 849 for (i = 0, ring_prod = 0; 850 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) { 851 852 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) { 853 BNX2X_ERR("was only able to allocate " 854 "%d rx sges\n", i); 855 BNX2X_ERR("disabling TPA for queue[%d]\n", j); 856 /* Cleanup already allocated elements */ 857 bnx2x_free_rx_sge_range(bp, fp, ring_prod); 858 bnx2x_free_tpa_pool(bp, fp, max_agg_queues); 859 fp->disable_tpa = 1; 860 ring_prod = 0; 861 break; 862 } 863 ring_prod = NEXT_SGE_IDX(ring_prod); 864 } 865 fp->rx_sge_prod = ring_prod; 866 867 /* Allocate BDs and initialize BD ring */ 868 fp->rx_comp_cons = 0; 869 cqe_ring_prod = ring_prod = 0; 870 for (i = 0; i < bp->rx_ring_size; i++) { 871 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) { 872 BNX2X_ERR("was only able to allocate " 873 "%d rx skbs on queue[%d]\n", i, j); 874 fp->eth_q_stats.rx_skb_alloc_failed++; 875 break; 876 } 877 ring_prod = NEXT_RX_IDX(ring_prod); 878 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod); 879 WARN_ON(ring_prod <= i); 880 } 881 882 fp->rx_bd_prod = ring_prod; 883 /* must not have more available CQEs than BDs */ 884 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT, 885 cqe_ring_prod); 886 fp->rx_pkt = fp->rx_calls = 0; 887 888 /* Warning! 889 * this will generate an interrupt (to the TSTORM) 890 * must only be done after chip is initialized 891 */ 892 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod, 893 fp->rx_sge_prod); 894 if (j != 0) 895 continue; 896 897 REG_WR(bp, BAR_USTRORM_INTMEM + 898 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func), 899 U64_LO(fp->rx_comp_mapping)); 900 REG_WR(bp, BAR_USTRORM_INTMEM + 901 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4, 902 U64_HI(fp->rx_comp_mapping)); 903 } 904} 905static void bnx2x_free_tx_skbs(struct bnx2x *bp) 906{ 907 int i; 908 909 for_each_queue(bp, i) { 910 struct bnx2x_fastpath *fp = &bp->fp[i]; 911 912 u16 bd_cons = fp->tx_bd_cons; 913 u16 sw_prod = fp->tx_pkt_prod; 914 u16 sw_cons = fp->tx_pkt_cons; 915 916 while (sw_cons != sw_prod) { 917 bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons)); 918 sw_cons++; 919 } 920 } 921} 922 923static void bnx2x_free_rx_skbs(struct bnx2x *bp) 924{ 925 int i, j; 926 927 for_each_queue(bp, j) { 928 struct bnx2x_fastpath *fp = &bp->fp[j]; 929 930 for (i = 0; i < NUM_RX_BD; i++) { 931 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i]; 932 struct sk_buff *skb = rx_buf->skb; 933 934 if (skb == NULL) 935 continue; 936 937 dma_unmap_single(&bp->pdev->dev, 938 dma_unmap_addr(rx_buf, mapping), 939 bp->rx_buf_size, DMA_FROM_DEVICE); 940 941 rx_buf->skb = NULL; 942 dev_kfree_skb(skb); 943 } 944 if (!fp->disable_tpa) 945 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ? 946 ETH_MAX_AGGREGATION_QUEUES_E1 : 947 ETH_MAX_AGGREGATION_QUEUES_E1H); 948 } 949} 950 951void bnx2x_free_skbs(struct bnx2x *bp) 952{ 953 bnx2x_free_tx_skbs(bp); 954 bnx2x_free_rx_skbs(bp); 955} 956 957static void bnx2x_free_msix_irqs(struct bnx2x *bp) 958{ 959 int i, offset = 1; 960 961 free_irq(bp->msix_table[0].vector, bp->dev); 962 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n", 963 bp->msix_table[0].vector); 964 965#ifdef BCM_CNIC 966 offset++; 967#endif 968 for_each_queue(bp, i) { 969 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq " 970 "state %x\n", i, bp->msix_table[i + offset].vector, 971 bnx2x_fp(bp, i, state)); 972 973 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]); 974 } 975} 976 977void bnx2x_free_irq(struct bnx2x *bp, bool disable_only) 978{ 979 if (bp->flags & USING_MSIX_FLAG) { 980 if (!disable_only) 981 bnx2x_free_msix_irqs(bp); 982 pci_disable_msix(bp->pdev); 983 bp->flags &= ~USING_MSIX_FLAG; 984 985 } else if (bp->flags & USING_MSI_FLAG) { 986 if (!disable_only) 987 free_irq(bp->pdev->irq, bp->dev); 988 pci_disable_msi(bp->pdev); 989 bp->flags &= ~USING_MSI_FLAG; 990 991 } else if (!disable_only) 992 free_irq(bp->pdev->irq, bp->dev); 993} 994 995static int bnx2x_enable_msix(struct bnx2x *bp) 996{ 997 int i, rc, offset = 1; 998 int igu_vec = 0; 999 1000 bp->msix_table[0].entry = igu_vec; 1001 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec); 1002 1003#ifdef BCM_CNIC 1004 igu_vec = BP_L_ID(bp) + offset; 1005 bp->msix_table[1].entry = igu_vec; 1006 DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec); 1007 offset++; 1008#endif 1009 for_each_queue(bp, i) { 1010 igu_vec = BP_L_ID(bp) + offset + i; 1011 bp->msix_table[i + offset].entry = igu_vec; 1012 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d " 1013 "(fastpath #%u)\n", i + offset, igu_vec, i); 1014 } 1015 1016 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1017 BNX2X_NUM_QUEUES(bp) + offset); 1018 1019 /* 1020 * reconfigure number of tx/rx queues according to available 1021 * MSI-X vectors 1022 */ 1023 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) { 1024 /* vectors available for FP */ 1025 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START; 1026 1027 DP(NETIF_MSG_IFUP, 1028 "Trying to use less MSI-X vectors: %d\n", rc); 1029 1030 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc); 1031 1032 if (rc) { 1033 DP(NETIF_MSG_IFUP, 1034 "MSI-X is not attainable rc %d\n", rc); 1035 return rc; 1036 } 1037 1038 bp->num_queues = min(bp->num_queues, fp_vec); 1039 1040 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n", 1041 bp->num_queues); 1042 } else if (rc) { 1043 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc); 1044 return rc; 1045 } 1046 1047 bp->flags |= USING_MSIX_FLAG; 1048 1049 return 0; 1050} 1051 1052static int bnx2x_req_msix_irqs(struct bnx2x *bp) 1053{ 1054 int i, rc, offset = 1; 1055 1056 rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0, 1057 bp->dev->name, bp->dev); 1058 if (rc) { 1059 BNX2X_ERR("request sp irq failed\n"); 1060 return -EBUSY; 1061 } 1062 1063#ifdef BCM_CNIC 1064 offset++; 1065#endif 1066 for_each_queue(bp, i) { 1067 struct bnx2x_fastpath *fp = &bp->fp[i]; 1068 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", 1069 bp->dev->name, i); 1070 1071 rc = request_irq(bp->msix_table[i + offset].vector, 1072 bnx2x_msix_fp_int, 0, fp->name, fp); 1073 if (rc) { 1074 BNX2X_ERR("request fp #%d irq failed rc %d\n", i, rc); 1075 bnx2x_free_msix_irqs(bp); 1076 return -EBUSY; 1077 } 1078 1079 fp->state = BNX2X_FP_STATE_IRQ; 1080 } 1081 1082 i = BNX2X_NUM_QUEUES(bp); 1083 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d" 1084 " ... fp[%d] %d\n", 1085 bp->msix_table[0].vector, 1086 0, bp->msix_table[offset].vector, 1087 i - 1, bp->msix_table[offset + i - 1].vector); 1088 1089 return 0; 1090} 1091 1092static int bnx2x_enable_msi(struct bnx2x *bp) 1093{ 1094 int rc; 1095 1096 rc = pci_enable_msi(bp->pdev); 1097 if (rc) { 1098 DP(NETIF_MSG_IFUP, "MSI is not attainable\n"); 1099 return -1; 1100 } 1101 bp->flags |= USING_MSI_FLAG; 1102 1103 return 0; 1104} 1105 1106static int bnx2x_req_irq(struct bnx2x *bp) 1107{ 1108 unsigned long flags; 1109 int rc; 1110 1111 if (bp->flags & USING_MSI_FLAG) 1112 flags = 0; 1113 else 1114 flags = IRQF_SHARED; 1115 1116 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags, 1117 bp->dev->name, bp->dev); 1118 if (!rc) 1119 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ; 1120 1121 return rc; 1122} 1123 1124static void bnx2x_napi_enable(struct bnx2x *bp) 1125{ 1126 int i; 1127 1128 for_each_queue(bp, i) 1129 napi_enable(&bnx2x_fp(bp, i, napi)); 1130} 1131 1132static void bnx2x_napi_disable(struct bnx2x *bp) 1133{ 1134 int i; 1135 1136 for_each_queue(bp, i) 1137 napi_disable(&bnx2x_fp(bp, i, napi)); 1138} 1139 1140void bnx2x_netif_start(struct bnx2x *bp) 1141{ 1142 int intr_sem; 1143 1144 intr_sem = atomic_dec_and_test(&bp->intr_sem); 1145 smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */ 1146 1147 if (intr_sem) { 1148 if (netif_running(bp->dev)) { 1149 bnx2x_napi_enable(bp); 1150 bnx2x_int_enable(bp); 1151 if (bp->state == BNX2X_STATE_OPEN) 1152 netif_tx_wake_all_queues(bp->dev); 1153 } 1154 } 1155} 1156 1157void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw) 1158{ 1159 bnx2x_int_disable_sync(bp, disable_hw); 1160 bnx2x_napi_disable(bp); 1161 netif_tx_disable(bp->dev); 1162} 1163static int bnx2x_set_num_queues(struct bnx2x *bp) 1164{ 1165 int rc = 0; 1166 1167 switch (bp->int_mode) { 1168 case INT_MODE_INTx: 1169 case INT_MODE_MSI: 1170 bp->num_queues = 1; 1171 DP(NETIF_MSG_IFUP, "set number of queues to 1\n"); 1172 break; 1173 default: 1174 /* Set number of queues according to bp->multi_mode value */ 1175 bnx2x_set_num_queues_msix(bp); 1176 1177 DP(NETIF_MSG_IFUP, "set number of queues to %d\n", 1178 bp->num_queues); 1179 1180 /* if we can't use MSI-X we only need one fp, 1181 * so try to enable MSI-X with the requested number of fp's 1182 * and fallback to MSI or legacy INTx with one fp 1183 */ 1184 rc = bnx2x_enable_msix(bp); 1185 if (rc) 1186 /* failed to enable MSI-X */ 1187 bp->num_queues = 1; 1188 break; 1189 } 1190 bp->dev->real_num_tx_queues = bp->num_queues; 1191 return rc; 1192} 1193 1194/* must be called with rtnl_lock */ 1195int bnx2x_nic_load(struct bnx2x *bp, int load_mode) 1196{ 1197 u32 load_code; 1198 int i, rc; 1199 1200#ifdef BNX2X_STOP_ON_ERROR 1201 if (unlikely(bp->panic)) 1202 return -EPERM; 1203#endif 1204 1205 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD; 1206 1207 rc = bnx2x_set_num_queues(bp); 1208 1209 if (bnx2x_alloc_mem(bp)) { 1210 bnx2x_free_irq(bp, true); 1211 return -ENOMEM; 1212 } 1213 1214 for_each_queue(bp, i) 1215 bnx2x_fp(bp, i, disable_tpa) = 1216 ((bp->flags & TPA_ENABLE_FLAG) == 0); 1217 1218 for_each_queue(bp, i) 1219 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), 1220 bnx2x_poll, 128); 1221 1222 bnx2x_napi_enable(bp); 1223 1224 if (bp->flags & USING_MSIX_FLAG) { 1225 rc = bnx2x_req_msix_irqs(bp); 1226 if (rc) { 1227 bnx2x_free_irq(bp, true); 1228 goto load_error1; 1229 } 1230 } else { 1231 /* Fall to INTx if failed to enable MSI-X due to lack of 1232 memory (in bnx2x_set_num_queues()) */ 1233 if ((rc != -ENOMEM) && (bp->int_mode != INT_MODE_INTx)) 1234 bnx2x_enable_msi(bp); 1235 bnx2x_ack_int(bp); 1236 rc = bnx2x_req_irq(bp); 1237 if (rc) { 1238 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc); 1239 bnx2x_free_irq(bp, true); 1240 goto load_error1; 1241 } 1242 if (bp->flags & USING_MSI_FLAG) { 1243 bp->dev->irq = bp->pdev->irq; 1244 netdev_info(bp->dev, "using MSI IRQ %d\n", 1245 bp->pdev->irq); 1246 } 1247 } 1248 1249 /* Send LOAD_REQUEST command to MCP 1250 Returns the type of LOAD command: 1251 if it is the first port to be initialized 1252 common blocks should be initialized, otherwise - not 1253 */ 1254 if (!BP_NOMCP(bp)) { 1255 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ); 1256 if (!load_code) { 1257 BNX2X_ERR("MCP response failure, aborting\n"); 1258 rc = -EBUSY; 1259 goto load_error2; 1260 } 1261 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) { 1262 rc = -EBUSY; /* other port in diagnostic mode */ 1263 goto load_error2; 1264 } 1265 1266 } else { 1267 int port = BP_PORT(bp); 1268 1269 DP(NETIF_MSG_IFUP, "NO MCP - load counts %d, %d, %d\n", 1270 load_count[0], load_count[1], load_count[2]); 1271 load_count[0]++; 1272 load_count[1 + port]++; 1273 DP(NETIF_MSG_IFUP, "NO MCP - new load counts %d, %d, %d\n", 1274 load_count[0], load_count[1], load_count[2]); 1275 if (load_count[0] == 1) 1276 load_code = FW_MSG_CODE_DRV_LOAD_COMMON; 1277 else if (load_count[1 + port] == 1) 1278 load_code = FW_MSG_CODE_DRV_LOAD_PORT; 1279 else 1280 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION; 1281 } 1282 1283 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) || 1284 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) 1285 bp->port.pmf = 1; 1286 else 1287 bp->port.pmf = 0; 1288 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf); 1289 1290 /* Initialize HW */ 1291 rc = bnx2x_init_hw(bp, load_code); 1292 if (rc) { 1293 BNX2X_ERR("HW init failed, aborting\n"); 1294 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE); 1295 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP); 1296 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE); 1297 goto load_error2; 1298 } 1299 1300 /* Setup NIC internals and enable interrupts */ 1301 bnx2x_nic_init(bp, load_code); 1302 1303 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) && 1304 (bp->common.shmem2_base)) 1305 SHMEM2_WR(bp, dcc_support, 1306 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV | 1307 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV)); 1308 1309 /* Send LOAD_DONE command to MCP */ 1310 if (!BP_NOMCP(bp)) { 1311 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE); 1312 if (!load_code) { 1313 BNX2X_ERR("MCP response failure, aborting\n"); 1314 rc = -EBUSY; 1315 goto load_error3; 1316 } 1317 } 1318 1319 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT; 1320 1321 rc = bnx2x_setup_leading(bp); 1322 if (rc) { 1323 BNX2X_ERR("Setup leading failed!\n"); 1324#ifndef BNX2X_STOP_ON_ERROR 1325 goto load_error3; 1326#else 1327 bp->panic = 1; 1328 return -EBUSY; 1329#endif 1330 } 1331 1332 if (CHIP_IS_E1H(bp)) 1333 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) { 1334 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n"); 1335 bp->flags |= MF_FUNC_DIS; 1336 } 1337 1338 if (bp->state == BNX2X_STATE_OPEN) { 1339#ifdef BCM_CNIC 1340 /* Enable Timer scan */ 1341 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1); 1342#endif 1343 for_each_nondefault_queue(bp, i) { 1344 rc = bnx2x_setup_multi(bp, i); 1345 if (rc) 1346#ifdef BCM_CNIC 1347 goto load_error4; 1348#else 1349 goto load_error3; 1350#endif 1351 } 1352 1353 if (CHIP_IS_E1(bp)) 1354 bnx2x_set_eth_mac_addr_e1(bp, 1); 1355 else 1356 bnx2x_set_eth_mac_addr_e1h(bp, 1); 1357#ifdef BCM_CNIC 1358 /* Set iSCSI L2 MAC */ 1359 mutex_lock(&bp->cnic_mutex); 1360 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) { 1361 bnx2x_set_iscsi_eth_mac_addr(bp, 1); 1362 bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET; 1363 bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping, 1364 CNIC_SB_ID(bp)); 1365 } 1366 mutex_unlock(&bp->cnic_mutex); 1367#endif 1368 } 1369 1370 if (bp->port.pmf) 1371 bnx2x_initial_phy_init(bp, load_mode); 1372 1373 /* Start fast path */ 1374 switch (load_mode) { 1375 case LOAD_NORMAL: 1376 if (bp->state == BNX2X_STATE_OPEN) { 1377 /* Tx queue should be only reenabled */ 1378 netif_tx_wake_all_queues(bp->dev); 1379 } 1380 /* Initialize the receive filter. */ 1381 bnx2x_set_rx_mode(bp->dev); 1382 break; 1383 1384 case LOAD_OPEN: 1385 netif_tx_start_all_queues(bp->dev); 1386 if (bp->state != BNX2X_STATE_OPEN) 1387 netif_tx_disable(bp->dev); 1388 /* Initialize the receive filter. */ 1389 bnx2x_set_rx_mode(bp->dev); 1390 break; 1391 1392 case LOAD_DIAG: 1393 /* Initialize the receive filter. */ 1394 bnx2x_set_rx_mode(bp->dev); 1395 bp->state = BNX2X_STATE_DIAG; 1396 break; 1397 1398 default: 1399 break; 1400 } 1401 1402 if (!bp->port.pmf) 1403 bnx2x__link_status_update(bp); 1404 1405 /* start the timer */ 1406 mod_timer(&bp->timer, jiffies + bp->current_interval); 1407 1408#ifdef BCM_CNIC 1409 bnx2x_setup_cnic_irq_info(bp); 1410 if (bp->state == BNX2X_STATE_OPEN) 1411 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD); 1412#endif 1413 bnx2x_inc_load_cnt(bp); 1414 1415 return 0; 1416 1417#ifdef BCM_CNIC 1418load_error4: 1419 /* Disable Timer scan */ 1420 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0); 1421#endif 1422load_error3: 1423 bnx2x_int_disable_sync(bp, 1); 1424 if (!BP_NOMCP(bp)) { 1425 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP); 1426 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE); 1427 } 1428 bp->port.pmf = 0; 1429 /* Free SKBs, SGEs, TPA pool and driver internals */ 1430 bnx2x_free_skbs(bp); 1431 for_each_queue(bp, i) 1432 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 1433load_error2: 1434 /* Release IRQs */ 1435 bnx2x_free_irq(bp, false); 1436load_error1: 1437 bnx2x_napi_disable(bp); 1438 for_each_queue(bp, i) 1439 netif_napi_del(&bnx2x_fp(bp, i, napi)); 1440 bnx2x_free_mem(bp); 1441 1442 return rc; 1443} 1444 1445/* must be called with rtnl_lock */ 1446int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode) 1447{ 1448 int i; 1449 1450 if (bp->state == BNX2X_STATE_CLOSED) { 1451 /* Interface has been removed - nothing to recover */ 1452 bp->recovery_state = BNX2X_RECOVERY_DONE; 1453 bp->is_leader = 0; 1454 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08); 1455 smp_wmb(); 1456 1457 return -EINVAL; 1458 } 1459 1460#ifdef BCM_CNIC 1461 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD); 1462#endif 1463 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT; 1464 1465 /* Set "drop all" */ 1466 bp->rx_mode = BNX2X_RX_MODE_NONE; 1467 bnx2x_set_storm_rx_mode(bp); 1468 1469 /* Disable HW interrupts, NAPI and Tx */ 1470 bnx2x_netif_stop(bp, 1); 1471 netif_carrier_off(bp->dev); 1472 1473 del_timer_sync(&bp->timer); 1474 SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb, 1475 (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq)); 1476 bnx2x_stats_handle(bp, STATS_EVENT_STOP); 1477 1478 /* Release IRQs */ 1479 bnx2x_free_irq(bp, false); 1480 1481 /* Cleanup the chip if needed */ 1482 if (unload_mode != UNLOAD_RECOVERY) 1483 bnx2x_chip_cleanup(bp, unload_mode); 1484 1485 bp->port.pmf = 0; 1486 1487 /* Free SKBs, SGEs, TPA pool and driver internals */ 1488 bnx2x_free_skbs(bp); 1489 for_each_queue(bp, i) 1490 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE); 1491 for_each_queue(bp, i) 1492 netif_napi_del(&bnx2x_fp(bp, i, napi)); 1493 bnx2x_free_mem(bp); 1494 1495 bp->state = BNX2X_STATE_CLOSED; 1496 1497 /* The last driver must disable a "close the gate" if there is no 1498 * parity attention or "process kill" pending. 1499 */ 1500 if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) && 1501 bnx2x_reset_is_done(bp)) 1502 bnx2x_disable_close_the_gate(bp); 1503 1504 /* Reset MCP mail box sequence if there is on going recovery */ 1505 if (unload_mode == UNLOAD_RECOVERY) 1506 bp->fw_seq = 0; 1507 1508 return 0; 1509} 1510int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state) 1511{ 1512 u16 pmcsr; 1513 1514 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr); 1515 1516 switch (state) { 1517 case PCI_D0: 1518 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, 1519 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) | 1520 PCI_PM_CTRL_PME_STATUS)); 1521 1522 if (pmcsr & PCI_PM_CTRL_STATE_MASK) 1523 /* delay required during transition out of D3hot */ 1524 msleep(20); 1525 break; 1526 1527 case PCI_D3hot: 1528 /* If there are other clients above don't 1529 shut down the power */ 1530 if (atomic_read(&bp->pdev->enable_cnt) != 1) 1531 return 0; 1532 /* Don't shut down the power for emulation and FPGA */ 1533 if (CHIP_REV_IS_SLOW(bp)) 1534 return 0; 1535 1536 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 1537 pmcsr |= 3; 1538 1539 if (bp->wol) 1540 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 1541 1542 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, 1543 pmcsr); 1544 1545 /* No more memory access after this point until 1546 * device is brought back to D0. 1547 */ 1548 break; 1549 1550 default: 1551 return -EINVAL; 1552 } 1553 return 0; 1554} 1555 1556 1557 1558/* 1559 * net_device service functions 1560 */ 1561 1562static int bnx2x_poll(struct napi_struct *napi, int budget) 1563{ 1564 int work_done = 0; 1565 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath, 1566 napi); 1567 struct bnx2x *bp = fp->bp; 1568 1569 while (1) { 1570#ifdef BNX2X_STOP_ON_ERROR 1571 if (unlikely(bp->panic)) { 1572 napi_complete(napi); 1573 return 0; 1574 } 1575#endif 1576 1577 if (bnx2x_has_tx_work(fp)) 1578 bnx2x_tx_int(fp); 1579 1580 if (bnx2x_has_rx_work(fp)) { 1581 work_done += bnx2x_rx_int(fp, budget - work_done); 1582 1583 /* must not complete if we consumed full budget */ 1584 if (work_done >= budget) 1585 break; 1586 } 1587 1588 /* Fall out from the NAPI loop if needed */ 1589 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) { 1590 bnx2x_update_fpsb_idx(fp); 1591 /* bnx2x_has_rx_work() reads the status block, thus we need 1592 * to ensure that status block indices have been actually read 1593 * (bnx2x_update_fpsb_idx) prior to this check 1594 * (bnx2x_has_rx_work) so that we won't write the "newer" 1595 * value of the status block to IGU (if there was a DMA right 1596 * after bnx2x_has_rx_work and if there is no rmb, the memory 1597 * reading (bnx2x_update_fpsb_idx) may be postponed to right 1598 * before bnx2x_ack_sb). In this case there will never be 1599 * another interrupt until there is another update of the 1600 * status block, while there is still unhandled work. 1601 */ 1602 rmb(); 1603 1604 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) { 1605 napi_complete(napi); 1606 /* Re-enable interrupts */ 1607 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID, 1608 le16_to_cpu(fp->fp_c_idx), 1609 IGU_INT_NOP, 1); 1610 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 1611 le16_to_cpu(fp->fp_u_idx), 1612 IGU_INT_ENABLE, 1); 1613 break; 1614 } 1615 } 1616 } 1617 1618 return work_done; 1619} 1620 1621 1622/* we split the first BD into headers and data BDs 1623 * to ease the pain of our fellow microcode engineers 1624 * we use one mapping for both BDs 1625 * So far this has only been observed to happen 1626 * in Other Operating Systems(TM) 1627 */ 1628static noinline u16 bnx2x_tx_split(struct bnx2x *bp, 1629 struct bnx2x_fastpath *fp, 1630 struct sw_tx_bd *tx_buf, 1631 struct eth_tx_start_bd **tx_bd, u16 hlen, 1632 u16 bd_prod, int nbd) 1633{ 1634 struct eth_tx_start_bd *h_tx_bd = *tx_bd; 1635 struct eth_tx_bd *d_tx_bd; 1636 dma_addr_t mapping; 1637 int old_len = le16_to_cpu(h_tx_bd->nbytes); 1638 1639 /* first fix first BD */ 1640 h_tx_bd->nbd = cpu_to_le16(nbd); 1641 h_tx_bd->nbytes = cpu_to_le16(hlen); 1642 1643 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d " 1644 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi, 1645 h_tx_bd->addr_lo, h_tx_bd->nbd); 1646 1647 /* now get a new data BD 1648 * (after the pbd) and fill it */ 1649 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 1650 d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd; 1651 1652 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi), 1653 le32_to_cpu(h_tx_bd->addr_lo)) + hlen; 1654 1655 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 1656 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 1657 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen); 1658 1659 /* this marks the BD as one that has no individual mapping */ 1660 tx_buf->flags |= BNX2X_TSO_SPLIT_BD; 1661 1662 DP(NETIF_MSG_TX_QUEUED, 1663 "TSO split data size is %d (%x:%x)\n", 1664 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo); 1665 1666 /* update tx_bd */ 1667 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd; 1668 1669 return bd_prod; 1670} 1671 1672static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix) 1673{ 1674 if (fix > 0) 1675 csum = (u16) ~csum_fold(csum_sub(csum, 1676 csum_partial(t_header - fix, fix, 0))); 1677 1678 else if (fix < 0) 1679 csum = (u16) ~csum_fold(csum_add(csum, 1680 csum_partial(t_header, -fix, 0))); 1681 1682 return swab16(csum); 1683} 1684 1685static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb) 1686{ 1687 u32 rc; 1688 1689 if (skb->ip_summed != CHECKSUM_PARTIAL) 1690 rc = XMIT_PLAIN; 1691 1692 else { 1693 if (skb->protocol == htons(ETH_P_IPV6)) { 1694 rc = XMIT_CSUM_V6; 1695 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 1696 rc |= XMIT_CSUM_TCP; 1697 1698 } else { 1699 rc = XMIT_CSUM_V4; 1700 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1701 rc |= XMIT_CSUM_TCP; 1702 } 1703 } 1704 1705 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) 1706 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP); 1707 1708 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) 1709 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6); 1710 1711 return rc; 1712} 1713 1714#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3) 1715/* check if packet requires linearization (packet is too fragmented) 1716 no need to check fragmentation if page size > 8K (there will be no 1717 violation to FW restrictions) */ 1718static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb, 1719 u32 xmit_type) 1720{ 1721 int to_copy = 0; 1722 int hlen = 0; 1723 int first_bd_sz = 0; 1724 1725 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */ 1726 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) { 1727 1728 if (xmit_type & XMIT_GSO) { 1729 unsigned short lso_mss = skb_shinfo(skb)->gso_size; 1730 /* Check if LSO packet needs to be copied: 1731 3 = 1 (for headers BD) + 2 (for PBD and last BD) */ 1732 int wnd_size = MAX_FETCH_BD - 3; 1733 /* Number of windows to check */ 1734 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size; 1735 int wnd_idx = 0; 1736 int frag_idx = 0; 1737 u32 wnd_sum = 0; 1738 1739 /* Headers length */ 1740 hlen = (int)(skb_transport_header(skb) - skb->data) + 1741 tcp_hdrlen(skb); 1742 1743 /* Amount of data (w/o headers) on linear part of SKB*/ 1744 first_bd_sz = skb_headlen(skb) - hlen; 1745 1746 wnd_sum = first_bd_sz; 1747 1748 /* Calculate the first sum - it's special */ 1749 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++) 1750 wnd_sum += 1751 skb_shinfo(skb)->frags[frag_idx].size; 1752 1753 /* If there was data on linear skb data - check it */ 1754 if (first_bd_sz > 0) { 1755 if (unlikely(wnd_sum < lso_mss)) { 1756 to_copy = 1; 1757 goto exit_lbl; 1758 } 1759 1760 wnd_sum -= first_bd_sz; 1761 } 1762 1763 /* Others are easier: run through the frag list and 1764 check all windows */ 1765 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) { 1766 wnd_sum += 1767 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size; 1768 1769 if (unlikely(wnd_sum < lso_mss)) { 1770 to_copy = 1; 1771 break; 1772 } 1773 wnd_sum -= 1774 skb_shinfo(skb)->frags[wnd_idx].size; 1775 } 1776 } else { 1777 /* in non-LSO too fragmented packet should always 1778 be linearized */ 1779 to_copy = 1; 1780 } 1781 } 1782 1783exit_lbl: 1784 if (unlikely(to_copy)) 1785 DP(NETIF_MSG_TX_QUEUED, 1786 "Linearization IS REQUIRED for %s packet. " 1787 "num_frags %d hlen %d first_bd_sz %d\n", 1788 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO", 1789 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz); 1790 1791 return to_copy; 1792} 1793#endif 1794 1795/* called with netif_tx_lock 1796 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call 1797 * netif_wake_queue() 1798 */ 1799netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev) 1800{ 1801 struct bnx2x *bp = netdev_priv(dev); 1802 struct bnx2x_fastpath *fp; 1803 struct netdev_queue *txq; 1804 struct sw_tx_bd *tx_buf; 1805 struct eth_tx_start_bd *tx_start_bd; 1806 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL; 1807 struct eth_tx_parse_bd *pbd = NULL; 1808 u16 pkt_prod, bd_prod; 1809 int nbd, fp_index; 1810 dma_addr_t mapping; 1811 u32 xmit_type = bnx2x_xmit_type(bp, skb); 1812 int i; 1813 u8 hlen = 0; 1814 __le16 pkt_size = 0; 1815 struct ethhdr *eth; 1816 u8 mac_type = UNICAST_ADDRESS; 1817 1818#ifdef BNX2X_STOP_ON_ERROR 1819 if (unlikely(bp->panic)) 1820 return NETDEV_TX_BUSY; 1821#endif 1822 1823 fp_index = skb_get_queue_mapping(skb); 1824 txq = netdev_get_tx_queue(dev, fp_index); 1825 1826 fp = &bp->fp[fp_index]; 1827 1828 if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) { 1829 fp->eth_q_stats.driver_xoff++; 1830 netif_tx_stop_queue(txq); 1831 BNX2X_ERR("BUG! Tx ring full when queue awake!\n"); 1832 return NETDEV_TX_BUSY; 1833 } 1834 1835 DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x protocol %x protocol(%x,%x)" 1836 " gso type %x xmit_type %x\n", 1837 skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr, 1838 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type); 1839 1840 eth = (struct ethhdr *)skb->data; 1841 1842 /* set flag according to packet type (UNICAST_ADDRESS is default)*/ 1843 if (unlikely(is_multicast_ether_addr(eth->h_dest))) { 1844 if (is_broadcast_ether_addr(eth->h_dest)) 1845 mac_type = BROADCAST_ADDRESS; 1846 else 1847 mac_type = MULTICAST_ADDRESS; 1848 } 1849 1850#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3) 1851 /* First, check if we need to linearize the skb (due to FW 1852 restrictions). No need to check fragmentation if page size > 8K 1853 (there will be no violation to FW restrictions) */ 1854 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) { 1855 /* Statistics of linearization */ 1856 bp->lin_cnt++; 1857 if (skb_linearize(skb) != 0) { 1858 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - " 1859 "silently dropping this SKB\n"); 1860 dev_kfree_skb_any(skb); 1861 return NETDEV_TX_OK; 1862 } 1863 } 1864#endif 1865 1866 /* 1867 Please read carefully. First we use one BD which we mark as start, 1868 then we have a parsing info BD (used for TSO or xsum), 1869 and only then we have the rest of the TSO BDs. 1870 (don't forget to mark the last one as last, 1871 and to unmap only AFTER you write to the BD ...) 1872 And above all, all pdb sizes are in words - NOT DWORDS! 1873 */ 1874 1875 pkt_prod = fp->tx_pkt_prod++; 1876 bd_prod = TX_BD(fp->tx_bd_prod); 1877 1878 /* get a tx_buf and first BD */ 1879 tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)]; 1880 tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd; 1881 1882 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; 1883 tx_start_bd->general_data = (mac_type << 1884 ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT); 1885 /* header nbd */ 1886 tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT); 1887 1888 /* remember the first BD of the packet */ 1889 tx_buf->first_bd = fp->tx_bd_prod; 1890 tx_buf->skb = skb; 1891 tx_buf->flags = 0; 1892 1893 DP(NETIF_MSG_TX_QUEUED, 1894 "sending pkt %u @%p next_idx %u bd %u @%p\n", 1895 pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd); 1896 1897#ifdef BCM_VLAN 1898 if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) && 1899 (bp->flags & HW_VLAN_TX_FLAG)) { 1900 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb)); 1901 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG; 1902 } else 1903#endif 1904 tx_start_bd->vlan = cpu_to_le16(pkt_prod); 1905 1906 /* turn on parsing and get a BD */ 1907 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 1908 pbd = &fp->tx_desc_ring[bd_prod].parse_bd; 1909 1910 memset(pbd, 0, sizeof(struct eth_tx_parse_bd)); 1911 1912 if (xmit_type & XMIT_CSUM) { 1913 hlen = (skb_network_header(skb) - skb->data) / 2; 1914 1915 /* for now NS flag is not used in Linux */ 1916 pbd->global_data = 1917 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) << 1918 ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT)); 1919 1920 pbd->ip_hlen = (skb_transport_header(skb) - 1921 skb_network_header(skb)) / 2; 1922 1923 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2; 1924 1925 pbd->total_hlen = cpu_to_le16(hlen); 1926 hlen = hlen*2; 1927 1928 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM; 1929 1930 if (xmit_type & XMIT_CSUM_V4) 1931 tx_start_bd->bd_flags.as_bitfield |= 1932 ETH_TX_BD_FLAGS_IP_CSUM; 1933 else 1934 tx_start_bd->bd_flags.as_bitfield |= 1935 ETH_TX_BD_FLAGS_IPV6; 1936 1937 if (xmit_type & XMIT_CSUM_TCP) { 1938 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check); 1939 1940 } else { 1941 s8 fix = SKB_CS_OFF(skb); /* signed! */ 1942 1943 pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG; 1944 1945 DP(NETIF_MSG_TX_QUEUED, 1946 "hlen %d fix %d csum before fix %x\n", 1947 le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb)); 1948 1949 /* HW bug: fixup the CSUM */ 1950 pbd->tcp_pseudo_csum = 1951 bnx2x_csum_fix(skb_transport_header(skb), 1952 SKB_CS(skb), fix); 1953 1954 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n", 1955 pbd->tcp_pseudo_csum); 1956 } 1957 } 1958 1959 mapping = dma_map_single(&bp->pdev->dev, skb->data, 1960 skb_headlen(skb), DMA_TO_DEVICE); 1961 1962 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 1963 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 1964 nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */ 1965 tx_start_bd->nbd = cpu_to_le16(nbd); 1966 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb)); 1967 pkt_size = tx_start_bd->nbytes; 1968 1969 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d" 1970 " nbytes %d flags %x vlan %x\n", 1971 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo, 1972 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes), 1973 tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan)); 1974 1975 if (xmit_type & XMIT_GSO) { 1976 1977 DP(NETIF_MSG_TX_QUEUED, 1978 "TSO packet len %d hlen %d total len %d tso size %d\n", 1979 skb->len, hlen, skb_headlen(skb), 1980 skb_shinfo(skb)->gso_size); 1981 1982 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO; 1983 1984 if (unlikely(skb_headlen(skb) > hlen)) 1985 bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd, 1986 hlen, bd_prod, ++nbd); 1987 1988 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size); 1989 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq); 1990 pbd->tcp_flags = pbd_tcp_flags(skb); 1991 1992 if (xmit_type & XMIT_GSO_V4) { 1993 pbd->ip_id = swab16(ip_hdr(skb)->id); 1994 pbd->tcp_pseudo_csum = 1995 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr, 1996 ip_hdr(skb)->daddr, 1997 0, IPPROTO_TCP, 0)); 1998 1999 } else 2000 pbd->tcp_pseudo_csum = 2001 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 2002 &ipv6_hdr(skb)->daddr, 2003 0, IPPROTO_TCP, 0)); 2004 2005 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN; 2006 } 2007 tx_data_bd = (struct eth_tx_bd *)tx_start_bd; 2008 2009 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2010 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2011 2012 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 2013 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd; 2014 if (total_pkt_bd == NULL) 2015 total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd; 2016 2017 mapping = dma_map_page(&bp->pdev->dev, frag->page, 2018 frag->page_offset, 2019 frag->size, DMA_TO_DEVICE); 2020 2021 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping)); 2022 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping)); 2023 tx_data_bd->nbytes = cpu_to_le16(frag->size); 2024 le16_add_cpu(&pkt_size, frag->size); 2025 2026 DP(NETIF_MSG_TX_QUEUED, 2027 "frag %d bd @%p addr (%x:%x) nbytes %d\n", 2028 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo, 2029 le16_to_cpu(tx_data_bd->nbytes)); 2030 } 2031 2032 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd); 2033 2034 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod)); 2035 2036 /* now send a tx doorbell, counting the next BD 2037 * if the packet contains or ends with it 2038 */ 2039 if (TX_BD_POFF(bd_prod) < nbd) 2040 nbd++; 2041 2042 if (total_pkt_bd != NULL) 2043 total_pkt_bd->total_pkt_bytes = pkt_size; 2044 2045 if (pbd) 2046 DP(NETIF_MSG_TX_QUEUED, 2047 "PBD @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u" 2048 " tcp_flags %x xsum %x seq %u hlen %u\n", 2049 pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id, 2050 pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum, 2051 pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen)); 2052 2053 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod); 2054 2055 /* 2056 * Make sure that the BD data is updated before updating the producer 2057 * since FW might read the BD right after the producer is updated. 2058 * This is only applicable for weak-ordered memory model archs such 2059 * as IA-64. The following barrier is also mandatory since FW will 2060 * assumes packets must have BDs. 2061 */ 2062 wmb(); 2063 2064 fp->tx_db.data.prod += nbd; 2065 barrier(); 2066 DOORBELL(bp, fp->index, fp->tx_db.raw); 2067 2068 mmiowb(); 2069 2070 fp->tx_bd_prod += nbd; 2071 2072 if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) { 2073 netif_tx_stop_queue(txq); 2074 2075 /* paired memory barrier is in bnx2x_tx_int(), we have to keep 2076 * ordering of set_bit() in netif_tx_stop_queue() and read of 2077 * fp->bd_tx_cons */ 2078 smp_mb(); 2079 2080 fp->eth_q_stats.driver_xoff++; 2081 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3) 2082 netif_tx_wake_queue(txq); 2083 } 2084 fp->tx_pkt++; 2085 2086 return NETDEV_TX_OK; 2087} 2088/* called with rtnl_lock */ 2089int bnx2x_change_mac_addr(struct net_device *dev, void *p) 2090{ 2091 struct sockaddr *addr = p; 2092 struct bnx2x *bp = netdev_priv(dev); 2093 2094 if (!is_valid_ether_addr((u8 *)(addr->sa_data))) 2095 return -EINVAL; 2096 2097 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 2098 if (netif_running(dev)) { 2099 if (CHIP_IS_E1(bp)) 2100 bnx2x_set_eth_mac_addr_e1(bp, 1); 2101 else 2102 bnx2x_set_eth_mac_addr_e1h(bp, 1); 2103 } 2104 2105 return 0; 2106} 2107 2108/* called with rtnl_lock */ 2109int bnx2x_change_mtu(struct net_device *dev, int new_mtu) 2110{ 2111 struct bnx2x *bp = netdev_priv(dev); 2112 int rc = 0; 2113 2114 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 2115 printk(KERN_ERR "Handling parity error recovery. Try again later\n"); 2116 return -EAGAIN; 2117 } 2118 2119 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) || 2120 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) 2121 return -EINVAL; 2122 2123 /* This does not race with packet allocation 2124 * because the actual alloc size is 2125 * only updated as part of load 2126 */ 2127 dev->mtu = new_mtu; 2128 2129 if (netif_running(dev)) { 2130 bnx2x_nic_unload(bp, UNLOAD_NORMAL); 2131 rc = bnx2x_nic_load(bp, LOAD_NORMAL); 2132 } 2133 2134 return rc; 2135} 2136 2137void bnx2x_tx_timeout(struct net_device *dev) 2138{ 2139 struct bnx2x *bp = netdev_priv(dev); 2140 2141#ifdef BNX2X_STOP_ON_ERROR 2142 if (!bp->panic) 2143 bnx2x_panic(); 2144#endif 2145 /* This allows the netif to be shutdown gracefully before resetting */ 2146 schedule_delayed_work(&bp->reset_task, 0); 2147} 2148 2149#ifdef BCM_VLAN 2150/* called with rtnl_lock */ 2151void bnx2x_vlan_rx_register(struct net_device *dev, 2152 struct vlan_group *vlgrp) 2153{ 2154 struct bnx2x *bp = netdev_priv(dev); 2155 2156 bp->vlgrp = vlgrp; 2157 2158 /* Set flags according to the required capabilities */ 2159 bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG); 2160 2161 if (dev->features & NETIF_F_HW_VLAN_TX) 2162 bp->flags |= HW_VLAN_TX_FLAG; 2163 2164 if (dev->features & NETIF_F_HW_VLAN_RX) 2165 bp->flags |= HW_VLAN_RX_FLAG; 2166 2167 if (netif_running(dev)) 2168 bnx2x_set_client_config(bp); 2169} 2170 2171#endif 2172int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state) 2173{ 2174 struct net_device *dev = pci_get_drvdata(pdev); 2175 struct bnx2x *bp; 2176 2177 if (!dev) { 2178 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 2179 return -ENODEV; 2180 } 2181 bp = netdev_priv(dev); 2182 2183 rtnl_lock(); 2184 2185 pci_save_state(pdev); 2186 2187 if (!netif_running(dev)) { 2188 rtnl_unlock(); 2189 return 0; 2190 } 2191 2192 netif_device_detach(dev); 2193 2194 bnx2x_nic_unload(bp, UNLOAD_CLOSE); 2195 2196 bnx2x_set_power_state(bp, pci_choose_state(pdev, state)); 2197 2198 rtnl_unlock(); 2199 2200 return 0; 2201} 2202 2203int bnx2x_resume(struct pci_dev *pdev) 2204{ 2205 struct net_device *dev = pci_get_drvdata(pdev); 2206 struct bnx2x *bp; 2207 int rc; 2208 2209 if (!dev) { 2210 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n"); 2211 return -ENODEV; 2212 } 2213 bp = netdev_priv(dev); 2214 2215 if (bp->recovery_state != BNX2X_RECOVERY_DONE) { 2216 printk(KERN_ERR "Handling parity error recovery. Try again later\n"); 2217 return -EAGAIN; 2218 } 2219 2220 rtnl_lock(); 2221 2222 pci_restore_state(pdev); 2223 2224 if (!netif_running(dev)) { 2225 rtnl_unlock(); 2226 return 0; 2227 } 2228 2229 bnx2x_set_power_state(bp, PCI_D0); 2230 netif_device_attach(dev); 2231 2232 rc = bnx2x_nic_load(bp, LOAD_OPEN); 2233 2234 rtnl_unlock(); 2235 2236 return rc; 2237} 2238