1/************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43#include <linux/tcp.h> 44#include <linux/netdevice.h> 45#include <linux/skbuff.h> 46#include <linux/string.h> 47#include <linux/module.h> 48#include <linux/pci.h> 49#include <linux/dma-mapping.h> 50#include <linux/etherdevice.h> 51#include <linux/if_ether.h> 52#include <linux/if_vlan.h> 53#include <linux/dca.h> 54#include <linux/ip.h> 55#include <linux/inet.h> 56#include <linux/in.h> 57#include <linux/ethtool.h> 58#include <linux/firmware.h> 59#include <linux/delay.h> 60#include <linux/timer.h> 61#include <linux/vmalloc.h> 62#include <linux/crc32.h> 63#include <linux/moduleparam.h> 64#include <linux/io.h> 65#include <linux/log2.h> 66#include <linux/slab.h> 67#include <linux/prefetch.h> 68#include <net/checksum.h> 69#include <net/gso.h> 70#include <net/ip.h> 71#include <net/tcp.h> 72#include <asm/byteorder.h> 73#include <asm/processor.h> 74 75#include "myri10ge_mcp.h" 76#include "myri10ge_mcp_gen_header.h" 77 78#define MYRI10GE_VERSION_STR "1.5.3-1.534" 79 80MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 81MODULE_AUTHOR("Maintainer: help@myri.com"); 82MODULE_VERSION(MYRI10GE_VERSION_STR); 83MODULE_LICENSE("Dual BSD/GPL"); 84 85#define MYRI10GE_MAX_ETHER_MTU 9014 86 87#define MYRI10GE_ETH_STOPPED 0 88#define MYRI10GE_ETH_STOPPING 1 89#define MYRI10GE_ETH_STARTING 2 90#define MYRI10GE_ETH_RUNNING 3 91#define MYRI10GE_ETH_OPEN_FAILED 4 92 93#define MYRI10GE_EEPROM_STRINGS_SIZE 256 94#define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 95 96#define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 97#define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 98 99#define MYRI10GE_ALLOC_ORDER 0 100#define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 101#define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 102 103#define MYRI10GE_MAX_SLICES 32 104 105struct myri10ge_rx_buffer_state { 106 struct page *page; 107 int page_offset; 108 DEFINE_DMA_UNMAP_ADDR(bus); 109 DEFINE_DMA_UNMAP_LEN(len); 110}; 111 112struct myri10ge_tx_buffer_state { 113 struct sk_buff *skb; 114 int last; 115 DEFINE_DMA_UNMAP_ADDR(bus); 116 DEFINE_DMA_UNMAP_LEN(len); 117}; 118 119struct myri10ge_cmd { 120 u32 data0; 121 u32 data1; 122 u32 data2; 123}; 124 125struct myri10ge_rx_buf { 126 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 127 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 128 struct myri10ge_rx_buffer_state *info; 129 struct page *page; 130 dma_addr_t bus; 131 int page_offset; 132 int cnt; 133 int fill_cnt; 134 int alloc_fail; 135 int mask; /* number of rx slots -1 */ 136 int watchdog_needed; 137}; 138 139struct myri10ge_tx_buf { 140 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 141 __be32 __iomem *send_go; /* "go" doorbell ptr */ 142 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 143 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 144 char *req_bytes; 145 struct myri10ge_tx_buffer_state *info; 146 int mask; /* number of transmit slots -1 */ 147 int req ____cacheline_aligned; /* transmit slots submitted */ 148 int pkt_start; /* packets started */ 149 int stop_queue; 150 int linearized; 151 int done ____cacheline_aligned; /* transmit slots completed */ 152 int pkt_done; /* packets completed */ 153 int wake_queue; 154 int queue_active; 155}; 156 157struct myri10ge_rx_done { 158 struct mcp_slot *entry; 159 dma_addr_t bus; 160 int cnt; 161 int idx; 162}; 163 164struct myri10ge_slice_netstats { 165 unsigned long rx_packets; 166 unsigned long tx_packets; 167 unsigned long rx_bytes; 168 unsigned long tx_bytes; 169 unsigned long rx_dropped; 170 unsigned long tx_dropped; 171}; 172 173struct myri10ge_slice_state { 174 struct myri10ge_tx_buf tx; /* transmit ring */ 175 struct myri10ge_rx_buf rx_small; 176 struct myri10ge_rx_buf rx_big; 177 struct myri10ge_rx_done rx_done; 178 struct net_device *dev; 179 struct napi_struct napi; 180 struct myri10ge_priv *mgp; 181 struct myri10ge_slice_netstats stats; 182 __be32 __iomem *irq_claim; 183 struct mcp_irq_data *fw_stats; 184 dma_addr_t fw_stats_bus; 185 int watchdog_tx_done; 186 int watchdog_tx_req; 187 int watchdog_rx_done; 188 int stuck; 189#ifdef CONFIG_MYRI10GE_DCA 190 int cached_dca_tag; 191 int cpu; 192 __be32 __iomem *dca_tag; 193#endif 194 char irq_desc[32]; 195}; 196 197struct myri10ge_priv { 198 struct myri10ge_slice_state *ss; 199 int tx_boundary; /* boundary transmits cannot cross */ 200 int num_slices; 201 int running; /* running? */ 202 int small_bytes; 203 int big_bytes; 204 int max_intr_slots; 205 struct net_device *dev; 206 u8 __iomem *sram; 207 int sram_size; 208 unsigned long board_span; 209 unsigned long iomem_base; 210 __be32 __iomem *irq_deassert; 211 char *mac_addr_string; 212 struct mcp_cmd_response *cmd; 213 dma_addr_t cmd_bus; 214 struct pci_dev *pdev; 215 int msi_enabled; 216 int msix_enabled; 217 struct msix_entry *msix_vectors; 218#ifdef CONFIG_MYRI10GE_DCA 219 int dca_enabled; 220 int relaxed_order; 221#endif 222 u32 link_state; 223 unsigned int rdma_tags_available; 224 int intr_coal_delay; 225 __be32 __iomem *intr_coal_delay_ptr; 226 int wc_cookie; 227 int down_cnt; 228 wait_queue_head_t down_wq; 229 struct work_struct watchdog_work; 230 struct timer_list watchdog_timer; 231 int watchdog_resets; 232 int watchdog_pause; 233 int pause; 234 bool fw_name_allocated; 235 char *fw_name; 236 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 237 char *product_code_string; 238 char fw_version[128]; 239 int fw_ver_major; 240 int fw_ver_minor; 241 int fw_ver_tiny; 242 int adopted_rx_filter_bug; 243 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 244 unsigned long serial_number; 245 int vendor_specific_offset; 246 int fw_multicast_support; 247 u32 features; 248 u32 max_tso6; 249 u32 read_dma; 250 u32 write_dma; 251 u32 read_write_dma; 252 u32 link_changes; 253 u32 msg_enable; 254 unsigned int board_number; 255 int rebooted; 256}; 257 258static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 259static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 260static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 261static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 262MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 263MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 264MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 265MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 266 267/* Careful: must be accessed under kernel_param_lock() */ 268static char *myri10ge_fw_name = NULL; 269module_param(myri10ge_fw_name, charp, 0644); 270MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 271 272#define MYRI10GE_MAX_BOARDS 8 273static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 274 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 275module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 276 0444); 277MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 278 279static int myri10ge_ecrc_enable = 1; 280module_param(myri10ge_ecrc_enable, int, 0444); 281MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 282 283static int myri10ge_small_bytes = -1; /* -1 == auto */ 284module_param(myri10ge_small_bytes, int, 0644); 285MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 286 287static int myri10ge_msi = 1; /* enable msi by default */ 288module_param(myri10ge_msi, int, 0644); 289MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 290 291static int myri10ge_intr_coal_delay = 75; 292module_param(myri10ge_intr_coal_delay, int, 0444); 293MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 294 295static int myri10ge_flow_control = 1; 296module_param(myri10ge_flow_control, int, 0444); 297MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 298 299static int myri10ge_deassert_wait = 1; 300module_param(myri10ge_deassert_wait, int, 0644); 301MODULE_PARM_DESC(myri10ge_deassert_wait, 302 "Wait when deasserting legacy interrupts"); 303 304static int myri10ge_force_firmware = 0; 305module_param(myri10ge_force_firmware, int, 0444); 306MODULE_PARM_DESC(myri10ge_force_firmware, 307 "Force firmware to assume aligned completions"); 308 309static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 310module_param(myri10ge_initial_mtu, int, 0444); 311MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 312 313static int myri10ge_napi_weight = 64; 314module_param(myri10ge_napi_weight, int, 0444); 315MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 316 317static int myri10ge_watchdog_timeout = 1; 318module_param(myri10ge_watchdog_timeout, int, 0444); 319MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 320 321static int myri10ge_max_irq_loops = 1048576; 322module_param(myri10ge_max_irq_loops, int, 0444); 323MODULE_PARM_DESC(myri10ge_max_irq_loops, 324 "Set stuck legacy IRQ detection threshold"); 325 326#define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 327 328static int myri10ge_debug = -1; /* defaults above */ 329module_param(myri10ge_debug, int, 0); 330MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 331 332static int myri10ge_fill_thresh = 256; 333module_param(myri10ge_fill_thresh, int, 0644); 334MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 335 336static int myri10ge_reset_recover = 1; 337 338static int myri10ge_max_slices = 1; 339module_param(myri10ge_max_slices, int, 0444); 340MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 341 342static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 343module_param(myri10ge_rss_hash, int, 0444); 344MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 345 346static int myri10ge_dca = 1; 347module_param(myri10ge_dca, int, 0444); 348MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 349 350#define MYRI10GE_FW_OFFSET 1024*1024 351#define MYRI10GE_HIGHPART_TO_U32(X) \ 352(sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 353#define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 354 355#define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 356 357static void myri10ge_set_multicast_list(struct net_device *dev); 358static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 359 struct net_device *dev); 360 361static inline void put_be32(__be32 val, __be32 __iomem * p) 362{ 363 __raw_writel((__force __u32) val, (__force void __iomem *)p); 364} 365 366static void myri10ge_get_stats(struct net_device *dev, 367 struct rtnl_link_stats64 *stats); 368 369static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 370{ 371 if (mgp->fw_name_allocated) 372 kfree(mgp->fw_name); 373 mgp->fw_name = name; 374 mgp->fw_name_allocated = allocated; 375} 376 377static int 378myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 379 struct myri10ge_cmd *data, int atomic) 380{ 381 struct mcp_cmd *buf; 382 char buf_bytes[sizeof(*buf) + 8]; 383 struct mcp_cmd_response *response = mgp->cmd; 384 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 385 u32 dma_low, dma_high, result, value; 386 int sleep_total = 0; 387 388 /* ensure buf is aligned to 8 bytes */ 389 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 390 391 buf->data0 = htonl(data->data0); 392 buf->data1 = htonl(data->data1); 393 buf->data2 = htonl(data->data2); 394 buf->cmd = htonl(cmd); 395 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 396 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 397 398 buf->response_addr.low = htonl(dma_low); 399 buf->response_addr.high = htonl(dma_high); 400 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 401 mb(); 402 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 403 404 /* wait up to 15ms. Longest command is the DMA benchmark, 405 * which is capped at 5ms, but runs from a timeout handler 406 * that runs every 7.8ms. So a 15ms timeout leaves us with 407 * a 2.2ms margin 408 */ 409 if (atomic) { 410 /* if atomic is set, do not sleep, 411 * and try to get the completion quickly 412 * (1ms will be enough for those commands) */ 413 for (sleep_total = 0; 414 sleep_total < 1000 && 415 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 416 sleep_total += 10) { 417 udelay(10); 418 mb(); 419 } 420 } else { 421 /* use msleep for most command */ 422 for (sleep_total = 0; 423 sleep_total < 15 && 424 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 425 sleep_total++) 426 msleep(1); 427 } 428 429 result = ntohl(response->result); 430 value = ntohl(response->data); 431 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 432 if (result == 0) { 433 data->data0 = value; 434 return 0; 435 } else if (result == MXGEFW_CMD_UNKNOWN) { 436 return -ENOSYS; 437 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 438 return -E2BIG; 439 } else if (result == MXGEFW_CMD_ERROR_RANGE && 440 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 441 (data-> 442 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 443 0) { 444 return -ERANGE; 445 } else { 446 dev_err(&mgp->pdev->dev, 447 "command %d failed, result = %d\n", 448 cmd, result); 449 return -ENXIO; 450 } 451 } 452 453 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 454 cmd, result); 455 return -EAGAIN; 456} 457 458/* 459 * The eeprom strings on the lanaiX have the format 460 * SN=x\0 461 * MAC=x:x:x:x:x:x\0 462 * PT:ddd mmm xx xx:xx:xx xx\0 463 * PV:ddd mmm xx xx:xx:xx xx\0 464 */ 465static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 466{ 467 char *ptr, *limit; 468 int i; 469 470 ptr = mgp->eeprom_strings; 471 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 472 473 while (*ptr != '\0' && ptr < limit) { 474 if (memcmp(ptr, "MAC=", 4) == 0) { 475 ptr += 4; 476 mgp->mac_addr_string = ptr; 477 for (i = 0; i < 6; i++) { 478 if ((ptr + 2) > limit) 479 goto abort; 480 mgp->mac_addr[i] = 481 simple_strtoul(ptr, &ptr, 16); 482 ptr += 1; 483 } 484 } 485 if (memcmp(ptr, "PC=", 3) == 0) { 486 ptr += 3; 487 mgp->product_code_string = ptr; 488 } 489 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 490 ptr += 3; 491 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 492 } 493 while (ptr < limit && *ptr++) ; 494 } 495 496 return 0; 497 498abort: 499 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 500 return -ENXIO; 501} 502 503/* 504 * Enable or disable periodic RDMAs from the host to make certain 505 * chipsets resend dropped PCIe messages 506 */ 507 508static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 509{ 510 char __iomem *submit; 511 __be32 buf[16] __attribute__ ((__aligned__(8))); 512 u32 dma_low, dma_high; 513 int i; 514 515 /* clear confirmation addr */ 516 mgp->cmd->data = 0; 517 mb(); 518 519 /* send a rdma command to the PCIe engine, and wait for the 520 * response in the confirmation address. The firmware should 521 * write a -1 there to indicate it is alive and well 522 */ 523 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 524 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 525 526 buf[0] = htonl(dma_high); /* confirm addr MSW */ 527 buf[1] = htonl(dma_low); /* confirm addr LSW */ 528 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 529 buf[3] = htonl(dma_high); /* dummy addr MSW */ 530 buf[4] = htonl(dma_low); /* dummy addr LSW */ 531 buf[5] = htonl(enable); /* enable? */ 532 533 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 534 535 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 536 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 537 msleep(1); 538 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 539 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 540 (enable ? "enable" : "disable")); 541} 542 543static int 544myri10ge_validate_firmware(struct myri10ge_priv *mgp, 545 struct mcp_gen_header *hdr) 546{ 547 struct device *dev = &mgp->pdev->dev; 548 549 /* check firmware type */ 550 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 551 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 552 return -EINVAL; 553 } 554 555 /* save firmware version for ethtool */ 556 strscpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 557 558 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 559 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 560 561 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 562 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 563 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 564 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 565 MXGEFW_VERSION_MINOR); 566 return -EINVAL; 567 } 568 return 0; 569} 570 571static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 572{ 573 unsigned crc, reread_crc; 574 const struct firmware *fw; 575 struct device *dev = &mgp->pdev->dev; 576 unsigned char *fw_readback; 577 struct mcp_gen_header *hdr; 578 size_t hdr_offset; 579 int status; 580 unsigned i; 581 582 if (request_firmware(&fw, mgp->fw_name, dev) < 0) { 583 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 584 mgp->fw_name); 585 status = -EINVAL; 586 goto abort_with_nothing; 587 } 588 589 /* check size */ 590 591 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 592 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 593 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 594 status = -EINVAL; 595 goto abort_with_fw; 596 } 597 598 /* check id */ 599 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 600 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 601 dev_err(dev, "Bad firmware file\n"); 602 status = -EINVAL; 603 goto abort_with_fw; 604 } 605 hdr = (void *)(fw->data + hdr_offset); 606 607 status = myri10ge_validate_firmware(mgp, hdr); 608 if (status != 0) 609 goto abort_with_fw; 610 611 crc = crc32(~0, fw->data, fw->size); 612 for (i = 0; i < fw->size; i += 256) { 613 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 614 fw->data + i, 615 min(256U, (unsigned)(fw->size - i))); 616 mb(); 617 readb(mgp->sram); 618 } 619 fw_readback = vmalloc(fw->size); 620 if (!fw_readback) { 621 status = -ENOMEM; 622 goto abort_with_fw; 623 } 624 /* corruption checking is good for parity recovery and buggy chipset */ 625 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 626 reread_crc = crc32(~0, fw_readback, fw->size); 627 vfree(fw_readback); 628 if (crc != reread_crc) { 629 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 630 (unsigned)fw->size, reread_crc, crc); 631 status = -EIO; 632 goto abort_with_fw; 633 } 634 *size = (u32) fw->size; 635 636abort_with_fw: 637 release_firmware(fw); 638 639abort_with_nothing: 640 return status; 641} 642 643static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 644{ 645 struct mcp_gen_header *hdr; 646 struct device *dev = &mgp->pdev->dev; 647 const size_t bytes = sizeof(struct mcp_gen_header); 648 size_t hdr_offset; 649 int status; 650 651 /* find running firmware header */ 652 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 653 654 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 655 dev_err(dev, "Running firmware has bad header offset (%d)\n", 656 (int)hdr_offset); 657 return -EIO; 658 } 659 660 /* copy header of running firmware from SRAM to host memory to 661 * validate firmware */ 662 hdr = kmalloc(bytes, GFP_KERNEL); 663 if (hdr == NULL) 664 return -ENOMEM; 665 666 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 667 status = myri10ge_validate_firmware(mgp, hdr); 668 kfree(hdr); 669 670 /* check to see if adopted firmware has bug where adopting 671 * it will cause broadcasts to be filtered unless the NIC 672 * is kept in ALLMULTI mode */ 673 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 674 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 675 mgp->adopted_rx_filter_bug = 1; 676 dev_warn(dev, "Adopting fw %d.%d.%d: " 677 "working around rx filter bug\n", 678 mgp->fw_ver_major, mgp->fw_ver_minor, 679 mgp->fw_ver_tiny); 680 } 681 return status; 682} 683 684static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 685{ 686 struct myri10ge_cmd cmd; 687 int status; 688 689 /* probe for IPv6 TSO support */ 690 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 691 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 692 &cmd, 0); 693 if (status == 0) { 694 mgp->max_tso6 = cmd.data0; 695 mgp->features |= NETIF_F_TSO6; 696 } 697 698 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 699 if (status != 0) { 700 dev_err(&mgp->pdev->dev, 701 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 702 return -ENXIO; 703 } 704 705 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 706 707 return 0; 708} 709 710static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 711{ 712 char __iomem *submit; 713 __be32 buf[16] __attribute__ ((__aligned__(8))); 714 u32 dma_low, dma_high, size; 715 int status, i; 716 717 size = 0; 718 status = myri10ge_load_hotplug_firmware(mgp, &size); 719 if (status) { 720 if (!adopt) 721 return status; 722 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 723 724 /* Do not attempt to adopt firmware if there 725 * was a bad crc */ 726 if (status == -EIO) 727 return status; 728 729 status = myri10ge_adopt_running_firmware(mgp); 730 if (status != 0) { 731 dev_err(&mgp->pdev->dev, 732 "failed to adopt running firmware\n"); 733 return status; 734 } 735 dev_info(&mgp->pdev->dev, 736 "Successfully adopted running firmware\n"); 737 if (mgp->tx_boundary == 4096) { 738 dev_warn(&mgp->pdev->dev, 739 "Using firmware currently running on NIC" 740 ". For optimal\n"); 741 dev_warn(&mgp->pdev->dev, 742 "performance consider loading optimized " 743 "firmware\n"); 744 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 745 } 746 747 set_fw_name(mgp, "adopted", false); 748 mgp->tx_boundary = 2048; 749 myri10ge_dummy_rdma(mgp, 1); 750 status = myri10ge_get_firmware_capabilities(mgp); 751 return status; 752 } 753 754 /* clear confirmation addr */ 755 mgp->cmd->data = 0; 756 mb(); 757 758 /* send a reload command to the bootstrap MCP, and wait for the 759 * response in the confirmation address. The firmware should 760 * write a -1 there to indicate it is alive and well 761 */ 762 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 763 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 764 765 buf[0] = htonl(dma_high); /* confirm addr MSW */ 766 buf[1] = htonl(dma_low); /* confirm addr LSW */ 767 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 768 769 /* FIX: All newest firmware should un-protect the bottom of 770 * the sram before handoff. However, the very first interfaces 771 * do not. Therefore the handoff copy must skip the first 8 bytes 772 */ 773 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 774 buf[4] = htonl(size - 8); /* length of code */ 775 buf[5] = htonl(8); /* where to copy to */ 776 buf[6] = htonl(0); /* where to jump to */ 777 778 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 779 780 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 781 mb(); 782 msleep(1); 783 mb(); 784 i = 0; 785 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 786 msleep(1 << i); 787 i++; 788 } 789 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 790 dev_err(&mgp->pdev->dev, "handoff failed\n"); 791 return -ENXIO; 792 } 793 myri10ge_dummy_rdma(mgp, 1); 794 status = myri10ge_get_firmware_capabilities(mgp); 795 796 return status; 797} 798 799static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, 800 const u8 * addr) 801{ 802 struct myri10ge_cmd cmd; 803 int status; 804 805 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 806 | (addr[2] << 8) | addr[3]); 807 808 cmd.data1 = ((addr[4] << 8) | (addr[5])); 809 810 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 811 return status; 812} 813 814static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 815{ 816 struct myri10ge_cmd cmd; 817 int status, ctl; 818 819 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 820 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 821 822 if (status) { 823 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 824 return status; 825 } 826 mgp->pause = pause; 827 return 0; 828} 829 830static void 831myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 832{ 833 struct myri10ge_cmd cmd; 834 int status, ctl; 835 836 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 837 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 838 if (status) 839 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 840} 841 842static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 843{ 844 struct myri10ge_cmd cmd; 845 int status; 846 u32 len; 847 struct page *dmatest_page; 848 dma_addr_t dmatest_bus; 849 char *test = " "; 850 851 dmatest_page = alloc_page(GFP_KERNEL); 852 if (!dmatest_page) 853 return -ENOMEM; 854 dmatest_bus = dma_map_page(&mgp->pdev->dev, dmatest_page, 0, 855 PAGE_SIZE, DMA_BIDIRECTIONAL); 856 if (unlikely(dma_mapping_error(&mgp->pdev->dev, dmatest_bus))) { 857 __free_page(dmatest_page); 858 return -ENOMEM; 859 } 860 861 /* Run a small DMA test. 862 * The magic multipliers to the length tell the firmware 863 * to do DMA read, write, or read+write tests. The 864 * results are returned in cmd.data0. The upper 16 865 * bits or the return is the number of transfers completed. 866 * The lower 16 bits is the time in 0.5us ticks that the 867 * transfers took to complete. 868 */ 869 870 len = mgp->tx_boundary; 871 872 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 873 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 874 cmd.data2 = len * 0x10000; 875 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 876 if (status != 0) { 877 test = "read"; 878 goto abort; 879 } 880 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 881 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 882 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 883 cmd.data2 = len * 0x1; 884 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 885 if (status != 0) { 886 test = "write"; 887 goto abort; 888 } 889 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 890 891 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 892 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 893 cmd.data2 = len * 0x10001; 894 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 895 if (status != 0) { 896 test = "read/write"; 897 goto abort; 898 } 899 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 900 (cmd.data0 & 0xffff); 901 902abort: 903 dma_unmap_page(&mgp->pdev->dev, dmatest_bus, PAGE_SIZE, 904 DMA_BIDIRECTIONAL); 905 put_page(dmatest_page); 906 907 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 908 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 909 test, status); 910 911 return status; 912} 913 914static int myri10ge_reset(struct myri10ge_priv *mgp) 915{ 916 struct myri10ge_cmd cmd; 917 struct myri10ge_slice_state *ss; 918 int i, status; 919 size_t bytes; 920#ifdef CONFIG_MYRI10GE_DCA 921 unsigned long dca_tag_off; 922#endif 923 924 /* try to send a reset command to the card to see if it 925 * is alive */ 926 memset(&cmd, 0, sizeof(cmd)); 927 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 928 if (status != 0) { 929 dev_err(&mgp->pdev->dev, "failed reset\n"); 930 return -ENXIO; 931 } 932 933 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 934 /* 935 * Use non-ndis mcp_slot (eg, 4 bytes total, 936 * no toeplitz hash value returned. Older firmware will 937 * not understand this command, but will use the correct 938 * sized mcp_slot, so we ignore error returns 939 */ 940 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 941 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 942 943 /* Now exchange information about interrupts */ 944 945 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 946 cmd.data0 = (u32) bytes; 947 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 948 949 /* 950 * Even though we already know how many slices are supported 951 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 952 * has magic side effects, and must be called after a reset. 953 * It must be called prior to calling any RSS related cmds, 954 * including assigning an interrupt queue for anything but 955 * slice 0. It must also be called *after* 956 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 957 * the firmware to compute offsets. 958 */ 959 960 if (mgp->num_slices > 1) { 961 962 /* ask the maximum number of slices it supports */ 963 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 964 &cmd, 0); 965 if (status != 0) { 966 dev_err(&mgp->pdev->dev, 967 "failed to get number of slices\n"); 968 } 969 970 /* 971 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 972 * to setting up the interrupt queue DMA 973 */ 974 975 cmd.data0 = mgp->num_slices; 976 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 977 if (mgp->dev->real_num_tx_queues > 1) 978 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 979 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 980 &cmd, 0); 981 982 /* Firmware older than 1.4.32 only supports multiple 983 * RX queues, so if we get an error, first retry using a 984 * single TX queue before giving up */ 985 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 986 netif_set_real_num_tx_queues(mgp->dev, 1); 987 cmd.data0 = mgp->num_slices; 988 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 989 status = myri10ge_send_cmd(mgp, 990 MXGEFW_CMD_ENABLE_RSS_QUEUES, 991 &cmd, 0); 992 } 993 994 if (status != 0) { 995 dev_err(&mgp->pdev->dev, 996 "failed to set number of slices\n"); 997 998 return status; 999 } 1000 } 1001 for (i = 0; i < mgp->num_slices; i++) { 1002 ss = &mgp->ss[i]; 1003 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1004 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1005 cmd.data2 = i; 1006 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1007 &cmd, 0); 1008 } 1009 1010 status |= 1011 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1012 for (i = 0; i < mgp->num_slices; i++) { 1013 ss = &mgp->ss[i]; 1014 ss->irq_claim = 1015 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1016 } 1017 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1018 &cmd, 0); 1019 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1020 1021 status |= myri10ge_send_cmd 1022 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1023 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1024 if (status != 0) { 1025 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1026 return status; 1027 } 1028 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1029 1030#ifdef CONFIG_MYRI10GE_DCA 1031 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1032 dca_tag_off = cmd.data0; 1033 for (i = 0; i < mgp->num_slices; i++) { 1034 ss = &mgp->ss[i]; 1035 if (status == 0) { 1036 ss->dca_tag = (__iomem __be32 *) 1037 (mgp->sram + dca_tag_off + 4 * i); 1038 } else { 1039 ss->dca_tag = NULL; 1040 } 1041 } 1042#endif /* CONFIG_MYRI10GE_DCA */ 1043 1044 /* reset mcp/driver shared state back to 0 */ 1045 1046 mgp->link_changes = 0; 1047 for (i = 0; i < mgp->num_slices; i++) { 1048 ss = &mgp->ss[i]; 1049 1050 memset(ss->rx_done.entry, 0, bytes); 1051 ss->tx.req = 0; 1052 ss->tx.done = 0; 1053 ss->tx.pkt_start = 0; 1054 ss->tx.pkt_done = 0; 1055 ss->rx_big.cnt = 0; 1056 ss->rx_small.cnt = 0; 1057 ss->rx_done.idx = 0; 1058 ss->rx_done.cnt = 0; 1059 ss->tx.wake_queue = 0; 1060 ss->tx.stop_queue = 0; 1061 } 1062 1063 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1064 myri10ge_change_pause(mgp, mgp->pause); 1065 myri10ge_set_multicast_list(mgp->dev); 1066 return status; 1067} 1068 1069#ifdef CONFIG_MYRI10GE_DCA 1070static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1071{ 1072 int ret; 1073 u16 ctl; 1074 1075 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1076 1077 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1078 if (ret != on) { 1079 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1080 ctl |= (on << 4); 1081 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1082 } 1083 return ret; 1084} 1085 1086static void 1087myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1088{ 1089 ss->cached_dca_tag = tag; 1090 put_be32(htonl(tag), ss->dca_tag); 1091} 1092 1093static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1094{ 1095 int cpu = get_cpu(); 1096 int tag; 1097 1098 if (cpu != ss->cpu) { 1099 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1100 if (ss->cached_dca_tag != tag) 1101 myri10ge_write_dca(ss, cpu, tag); 1102 ss->cpu = cpu; 1103 } 1104 put_cpu(); 1105} 1106 1107static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1108{ 1109 int err, i; 1110 struct pci_dev *pdev = mgp->pdev; 1111 1112 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1113 return; 1114 if (!myri10ge_dca) { 1115 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1116 return; 1117 } 1118 err = dca_add_requester(&pdev->dev); 1119 if (err) { 1120 if (err != -ENODEV) 1121 dev_err(&pdev->dev, 1122 "dca_add_requester() failed, err=%d\n", err); 1123 return; 1124 } 1125 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1126 mgp->dca_enabled = 1; 1127 for (i = 0; i < mgp->num_slices; i++) { 1128 mgp->ss[i].cpu = -1; 1129 mgp->ss[i].cached_dca_tag = -1; 1130 myri10ge_update_dca(&mgp->ss[i]); 1131 } 1132} 1133 1134static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1135{ 1136 struct pci_dev *pdev = mgp->pdev; 1137 1138 if (!mgp->dca_enabled) 1139 return; 1140 mgp->dca_enabled = 0; 1141 if (mgp->relaxed_order) 1142 myri10ge_toggle_relaxed(pdev, 1); 1143 dca_remove_requester(&pdev->dev); 1144} 1145 1146static int myri10ge_notify_dca_device(struct device *dev, void *data) 1147{ 1148 struct myri10ge_priv *mgp; 1149 unsigned long event; 1150 1151 mgp = dev_get_drvdata(dev); 1152 event = *(unsigned long *)data; 1153 1154 if (event == DCA_PROVIDER_ADD) 1155 myri10ge_setup_dca(mgp); 1156 else if (event == DCA_PROVIDER_REMOVE) 1157 myri10ge_teardown_dca(mgp); 1158 return 0; 1159} 1160#endif /* CONFIG_MYRI10GE_DCA */ 1161 1162static inline void 1163myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1164 struct mcp_kreq_ether_recv *src) 1165{ 1166 __be32 low; 1167 1168 low = src->addr_low; 1169 src->addr_low = htonl(DMA_BIT_MASK(32)); 1170 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1171 mb(); 1172 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1173 mb(); 1174 src->addr_low = low; 1175 put_be32(low, &dst->addr_low); 1176 mb(); 1177} 1178 1179static void 1180myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1181 int bytes, int watchdog) 1182{ 1183 struct page *page; 1184 dma_addr_t bus; 1185 int idx; 1186#if MYRI10GE_ALLOC_SIZE > 4096 1187 int end_offset; 1188#endif 1189 1190 if (unlikely(rx->watchdog_needed && !watchdog)) 1191 return; 1192 1193 /* try to refill entire ring */ 1194 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1195 idx = rx->fill_cnt & rx->mask; 1196 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1197 /* we can use part of previous page */ 1198 get_page(rx->page); 1199 } else { 1200 /* we need a new page */ 1201 page = 1202 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1203 MYRI10GE_ALLOC_ORDER); 1204 if (unlikely(page == NULL)) { 1205 if (rx->fill_cnt - rx->cnt < 16) 1206 rx->watchdog_needed = 1; 1207 return; 1208 } 1209 1210 bus = dma_map_page(&mgp->pdev->dev, page, 0, 1211 MYRI10GE_ALLOC_SIZE, 1212 DMA_FROM_DEVICE); 1213 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 1214 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1215 if (rx->fill_cnt - rx->cnt < 16) 1216 rx->watchdog_needed = 1; 1217 return; 1218 } 1219 1220 rx->page = page; 1221 rx->page_offset = 0; 1222 rx->bus = bus; 1223 1224 } 1225 rx->info[idx].page = rx->page; 1226 rx->info[idx].page_offset = rx->page_offset; 1227 /* note that this is the address of the start of the 1228 * page */ 1229 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1230 rx->shadow[idx].addr_low = 1231 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1232 rx->shadow[idx].addr_high = 1233 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1234 1235 /* start next packet on a cacheline boundary */ 1236 rx->page_offset += SKB_DATA_ALIGN(bytes); 1237 1238#if MYRI10GE_ALLOC_SIZE > 4096 1239 /* don't cross a 4KB boundary */ 1240 end_offset = rx->page_offset + bytes - 1; 1241 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1242 rx->page_offset = end_offset & ~4095; 1243#endif 1244 rx->fill_cnt++; 1245 1246 /* copy 8 descriptors to the firmware at a time */ 1247 if ((idx & 7) == 7) { 1248 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1249 &rx->shadow[idx - 7]); 1250 } 1251 } 1252} 1253 1254static inline void 1255myri10ge_unmap_rx_page(struct pci_dev *pdev, 1256 struct myri10ge_rx_buffer_state *info, int bytes) 1257{ 1258 /* unmap the recvd page if we're the only or last user of it */ 1259 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1260 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1261 dma_unmap_page(&pdev->dev, (dma_unmap_addr(info, bus) 1262 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1263 MYRI10GE_ALLOC_SIZE, DMA_FROM_DEVICE); 1264 } 1265} 1266 1267/* 1268 * GRO does not support acceleration of tagged vlan frames, and 1269 * this NIC does not support vlan tag offload, so we must pop 1270 * the tag ourselves to be able to achieve GRO performance that 1271 * is comparable to LRO. 1272 */ 1273 1274static inline void 1275myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1276{ 1277 u8 *va; 1278 struct vlan_ethhdr *veh; 1279 skb_frag_t *frag; 1280 __wsum vsum; 1281 1282 va = addr; 1283 va += MXGEFW_PAD; 1284 veh = (struct vlan_ethhdr *)va; 1285 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1286 NETIF_F_HW_VLAN_CTAG_RX && 1287 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1288 /* fixup csum if needed */ 1289 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1290 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1291 skb->csum = csum_sub(skb->csum, vsum); 1292 } 1293 /* pop tag */ 1294 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1295 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1296 skb->len -= VLAN_HLEN; 1297 skb->data_len -= VLAN_HLEN; 1298 frag = skb_shinfo(skb)->frags; 1299 skb_frag_off_add(frag, VLAN_HLEN); 1300 skb_frag_size_sub(frag, VLAN_HLEN); 1301 } 1302} 1303 1304#define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1305 1306static inline int 1307myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1308{ 1309 struct myri10ge_priv *mgp = ss->mgp; 1310 struct sk_buff *skb; 1311 skb_frag_t *rx_frags; 1312 struct myri10ge_rx_buf *rx; 1313 int i, idx, remainder, bytes; 1314 struct pci_dev *pdev = mgp->pdev; 1315 struct net_device *dev = mgp->dev; 1316 u8 *va; 1317 1318 if (len <= mgp->small_bytes) { 1319 rx = &ss->rx_small; 1320 bytes = mgp->small_bytes; 1321 } else { 1322 rx = &ss->rx_big; 1323 bytes = mgp->big_bytes; 1324 } 1325 1326 len += MXGEFW_PAD; 1327 idx = rx->cnt & rx->mask; 1328 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1329 prefetch(va); 1330 1331 skb = napi_get_frags(&ss->napi); 1332 if (unlikely(skb == NULL)) { 1333 ss->stats.rx_dropped++; 1334 for (i = 0, remainder = len; remainder > 0; i++) { 1335 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1336 put_page(rx->info[idx].page); 1337 rx->cnt++; 1338 idx = rx->cnt & rx->mask; 1339 remainder -= MYRI10GE_ALLOC_SIZE; 1340 } 1341 return 0; 1342 } 1343 rx_frags = skb_shinfo(skb)->frags; 1344 /* Fill skb_frag_t(s) with data from our receive */ 1345 for (i = 0, remainder = len; remainder > 0; i++) { 1346 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1347 skb_fill_page_desc(skb, i, rx->info[idx].page, 1348 rx->info[idx].page_offset, 1349 remainder < MYRI10GE_ALLOC_SIZE ? 1350 remainder : MYRI10GE_ALLOC_SIZE); 1351 rx->cnt++; 1352 idx = rx->cnt & rx->mask; 1353 remainder -= MYRI10GE_ALLOC_SIZE; 1354 } 1355 1356 /* remove padding */ 1357 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1358 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1359 len -= MXGEFW_PAD; 1360 1361 skb->len = len; 1362 skb->data_len = len; 1363 skb->truesize += len; 1364 if (dev->features & NETIF_F_RXCSUM) { 1365 skb->ip_summed = CHECKSUM_COMPLETE; 1366 skb->csum = csum; 1367 } 1368 myri10ge_vlan_rx(mgp->dev, va, skb); 1369 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1370 1371 napi_gro_frags(&ss->napi); 1372 1373 return 1; 1374} 1375 1376static inline void 1377myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1378{ 1379 struct pci_dev *pdev = ss->mgp->pdev; 1380 struct myri10ge_tx_buf *tx = &ss->tx; 1381 struct netdev_queue *dev_queue; 1382 struct sk_buff *skb; 1383 int idx, len; 1384 1385 while (tx->pkt_done != mcp_index) { 1386 idx = tx->done & tx->mask; 1387 skb = tx->info[idx].skb; 1388 1389 /* Mark as free */ 1390 tx->info[idx].skb = NULL; 1391 if (tx->info[idx].last) { 1392 tx->pkt_done++; 1393 tx->info[idx].last = 0; 1394 } 1395 tx->done++; 1396 len = dma_unmap_len(&tx->info[idx], len); 1397 dma_unmap_len_set(&tx->info[idx], len, 0); 1398 if (skb) { 1399 ss->stats.tx_bytes += skb->len; 1400 ss->stats.tx_packets++; 1401 dev_consume_skb_irq(skb); 1402 if (len) 1403 dma_unmap_single(&pdev->dev, 1404 dma_unmap_addr(&tx->info[idx], 1405 bus), len, 1406 DMA_TO_DEVICE); 1407 } else { 1408 if (len) 1409 dma_unmap_page(&pdev->dev, 1410 dma_unmap_addr(&tx->info[idx], 1411 bus), len, 1412 DMA_TO_DEVICE); 1413 } 1414 } 1415 1416 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1417 /* 1418 * Make a minimal effort to prevent the NIC from polling an 1419 * idle tx queue. If we can't get the lock we leave the queue 1420 * active. In this case, either a thread was about to start 1421 * using the queue anyway, or we lost a race and the NIC will 1422 * waste some of its resources polling an inactive queue for a 1423 * while. 1424 */ 1425 1426 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1427 __netif_tx_trylock(dev_queue)) { 1428 if (tx->req == tx->done) { 1429 tx->queue_active = 0; 1430 put_be32(htonl(1), tx->send_stop); 1431 mb(); 1432 } 1433 __netif_tx_unlock(dev_queue); 1434 } 1435 1436 /* start the queue if we've stopped it */ 1437 if (netif_tx_queue_stopped(dev_queue) && 1438 tx->req - tx->done < (tx->mask >> 1) && 1439 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1440 tx->wake_queue++; 1441 netif_tx_wake_queue(dev_queue); 1442 } 1443} 1444 1445static inline int 1446myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1447{ 1448 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1449 struct myri10ge_priv *mgp = ss->mgp; 1450 unsigned long rx_bytes = 0; 1451 unsigned long rx_packets = 0; 1452 unsigned long rx_ok; 1453 int idx = rx_done->idx; 1454 int cnt = rx_done->cnt; 1455 int work_done = 0; 1456 u16 length; 1457 __wsum checksum; 1458 1459 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1460 length = ntohs(rx_done->entry[idx].length); 1461 rx_done->entry[idx].length = 0; 1462 checksum = csum_unfold(rx_done->entry[idx].checksum); 1463 rx_ok = myri10ge_rx_done(ss, length, checksum); 1464 rx_packets += rx_ok; 1465 rx_bytes += rx_ok * (unsigned long)length; 1466 cnt++; 1467 idx = cnt & (mgp->max_intr_slots - 1); 1468 work_done++; 1469 } 1470 rx_done->idx = idx; 1471 rx_done->cnt = cnt; 1472 ss->stats.rx_packets += rx_packets; 1473 ss->stats.rx_bytes += rx_bytes; 1474 1475 /* restock receive rings if needed */ 1476 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1477 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1478 mgp->small_bytes + MXGEFW_PAD, 0); 1479 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1480 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1481 1482 return work_done; 1483} 1484 1485static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1486{ 1487 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1488 1489 if (unlikely(stats->stats_updated)) { 1490 unsigned link_up = ntohl(stats->link_up); 1491 if (mgp->link_state != link_up) { 1492 mgp->link_state = link_up; 1493 1494 if (mgp->link_state == MXGEFW_LINK_UP) { 1495 netif_info(mgp, link, mgp->dev, "link up\n"); 1496 netif_carrier_on(mgp->dev); 1497 mgp->link_changes++; 1498 } else { 1499 netif_info(mgp, link, mgp->dev, "link %s\n", 1500 (link_up == MXGEFW_LINK_MYRINET ? 1501 "mismatch (Myrinet detected)" : 1502 "down")); 1503 netif_carrier_off(mgp->dev); 1504 mgp->link_changes++; 1505 } 1506 } 1507 if (mgp->rdma_tags_available != 1508 ntohl(stats->rdma_tags_available)) { 1509 mgp->rdma_tags_available = 1510 ntohl(stats->rdma_tags_available); 1511 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1512 mgp->rdma_tags_available); 1513 } 1514 mgp->down_cnt += stats->link_down; 1515 if (stats->link_down) 1516 wake_up(&mgp->down_wq); 1517 } 1518} 1519 1520static int myri10ge_poll(struct napi_struct *napi, int budget) 1521{ 1522 struct myri10ge_slice_state *ss = 1523 container_of(napi, struct myri10ge_slice_state, napi); 1524 int work_done; 1525 1526#ifdef CONFIG_MYRI10GE_DCA 1527 if (ss->mgp->dca_enabled) 1528 myri10ge_update_dca(ss); 1529#endif 1530 /* process as many rx events as NAPI will allow */ 1531 work_done = myri10ge_clean_rx_done(ss, budget); 1532 1533 if (work_done < budget) { 1534 napi_complete_done(napi, work_done); 1535 put_be32(htonl(3), ss->irq_claim); 1536 } 1537 return work_done; 1538} 1539 1540static irqreturn_t myri10ge_intr(int irq, void *arg) 1541{ 1542 struct myri10ge_slice_state *ss = arg; 1543 struct myri10ge_priv *mgp = ss->mgp; 1544 struct mcp_irq_data *stats = ss->fw_stats; 1545 struct myri10ge_tx_buf *tx = &ss->tx; 1546 u32 send_done_count; 1547 int i; 1548 1549 /* an interrupt on a non-zero receive-only slice is implicitly 1550 * valid since MSI-X irqs are not shared */ 1551 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1552 napi_schedule(&ss->napi); 1553 return IRQ_HANDLED; 1554 } 1555 1556 /* make sure it is our IRQ, and that the DMA has finished */ 1557 if (unlikely(!stats->valid)) 1558 return IRQ_NONE; 1559 1560 /* low bit indicates receives are present, so schedule 1561 * napi poll handler */ 1562 if (stats->valid & 1) 1563 napi_schedule(&ss->napi); 1564 1565 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1566 put_be32(0, mgp->irq_deassert); 1567 if (!myri10ge_deassert_wait) 1568 stats->valid = 0; 1569 mb(); 1570 } else 1571 stats->valid = 0; 1572 1573 /* Wait for IRQ line to go low, if using INTx */ 1574 i = 0; 1575 while (1) { 1576 i++; 1577 /* check for transmit completes and receives */ 1578 send_done_count = ntohl(stats->send_done_count); 1579 if (send_done_count != tx->pkt_done) 1580 myri10ge_tx_done(ss, (int)send_done_count); 1581 if (unlikely(i > myri10ge_max_irq_loops)) { 1582 netdev_warn(mgp->dev, "irq stuck?\n"); 1583 stats->valid = 0; 1584 schedule_work(&mgp->watchdog_work); 1585 } 1586 if (likely(stats->valid == 0)) 1587 break; 1588 cpu_relax(); 1589 barrier(); 1590 } 1591 1592 /* Only slice 0 updates stats */ 1593 if (ss == mgp->ss) 1594 myri10ge_check_statblock(mgp); 1595 1596 put_be32(htonl(3), ss->irq_claim + 1); 1597 return IRQ_HANDLED; 1598} 1599 1600static int 1601myri10ge_get_link_ksettings(struct net_device *netdev, 1602 struct ethtool_link_ksettings *cmd) 1603{ 1604 struct myri10ge_priv *mgp = netdev_priv(netdev); 1605 char *ptr; 1606 int i; 1607 1608 cmd->base.autoneg = AUTONEG_DISABLE; 1609 cmd->base.speed = SPEED_10000; 1610 cmd->base.duplex = DUPLEX_FULL; 1611 1612 /* 1613 * parse the product code to deterimine the interface type 1614 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1615 * after the 3rd dash in the driver's cached copy of the 1616 * EEPROM's product code string. 1617 */ 1618 ptr = mgp->product_code_string; 1619 if (ptr == NULL) { 1620 netdev_err(netdev, "Missing product code\n"); 1621 return 0; 1622 } 1623 for (i = 0; i < 3; i++, ptr++) { 1624 ptr = strchr(ptr, '-'); 1625 if (ptr == NULL) { 1626 netdev_err(netdev, "Invalid product code %s\n", 1627 mgp->product_code_string); 1628 return 0; 1629 } 1630 } 1631 if (*ptr == '2') 1632 ptr++; 1633 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1634 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1635 cmd->base.port = PORT_FIBRE; 1636 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1637 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1638 } else { 1639 cmd->base.port = PORT_OTHER; 1640 } 1641 1642 return 0; 1643} 1644 1645static void 1646myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1647{ 1648 struct myri10ge_priv *mgp = netdev_priv(netdev); 1649 1650 strscpy(info->driver, "myri10ge", sizeof(info->driver)); 1651 strscpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1652 strscpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1653 strscpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1654} 1655 1656static int myri10ge_get_coalesce(struct net_device *netdev, 1657 struct ethtool_coalesce *coal, 1658 struct kernel_ethtool_coalesce *kernel_coal, 1659 struct netlink_ext_ack *extack) 1660{ 1661 struct myri10ge_priv *mgp = netdev_priv(netdev); 1662 1663 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1664 return 0; 1665} 1666 1667static int myri10ge_set_coalesce(struct net_device *netdev, 1668 struct ethtool_coalesce *coal, 1669 struct kernel_ethtool_coalesce *kernel_coal, 1670 struct netlink_ext_ack *extack) 1671{ 1672 struct myri10ge_priv *mgp = netdev_priv(netdev); 1673 1674 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1675 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1676 return 0; 1677} 1678 1679static void 1680myri10ge_get_pauseparam(struct net_device *netdev, 1681 struct ethtool_pauseparam *pause) 1682{ 1683 struct myri10ge_priv *mgp = netdev_priv(netdev); 1684 1685 pause->autoneg = 0; 1686 pause->rx_pause = mgp->pause; 1687 pause->tx_pause = mgp->pause; 1688} 1689 1690static int 1691myri10ge_set_pauseparam(struct net_device *netdev, 1692 struct ethtool_pauseparam *pause) 1693{ 1694 struct myri10ge_priv *mgp = netdev_priv(netdev); 1695 1696 if (pause->tx_pause != mgp->pause) 1697 return myri10ge_change_pause(mgp, pause->tx_pause); 1698 if (pause->rx_pause != mgp->pause) 1699 return myri10ge_change_pause(mgp, pause->rx_pause); 1700 if (pause->autoneg != 0) 1701 return -EINVAL; 1702 return 0; 1703} 1704 1705static void 1706myri10ge_get_ringparam(struct net_device *netdev, 1707 struct ethtool_ringparam *ring, 1708 struct kernel_ethtool_ringparam *kernel_ring, 1709 struct netlink_ext_ack *extack) 1710{ 1711 struct myri10ge_priv *mgp = netdev_priv(netdev); 1712 1713 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1714 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1715 ring->rx_jumbo_max_pending = 0; 1716 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1717 ring->rx_mini_pending = ring->rx_mini_max_pending; 1718 ring->rx_pending = ring->rx_max_pending; 1719 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1720 ring->tx_pending = ring->tx_max_pending; 1721} 1722 1723static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1724 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1725 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1726 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1727 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1728 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1729 "tx_heartbeat_errors", "tx_window_errors", 1730 /* device-specific stats */ 1731 "tx_boundary", "irq", "MSI", "MSIX", 1732 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1733 "serial_number", "watchdog_resets", 1734#ifdef CONFIG_MYRI10GE_DCA 1735 "dca_capable_firmware", "dca_device_present", 1736#endif 1737 "link_changes", "link_up", "dropped_link_overflow", 1738 "dropped_link_error_or_filtered", 1739 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1740 "dropped_unicast_filtered", "dropped_multicast_filtered", 1741 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1742 "dropped_no_big_buffer" 1743}; 1744 1745static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1746 "----------- slice ---------", 1747 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1748 "rx_small_cnt", "rx_big_cnt", 1749 "wake_queue", "stop_queue", "tx_linearized", 1750}; 1751 1752#define MYRI10GE_NET_STATS_LEN 21 1753#define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1754#define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1755 1756static void 1757myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1758{ 1759 struct myri10ge_priv *mgp = netdev_priv(netdev); 1760 int i; 1761 1762 switch (stringset) { 1763 case ETH_SS_STATS: 1764 memcpy(data, *myri10ge_gstrings_main_stats, 1765 sizeof(myri10ge_gstrings_main_stats)); 1766 data += sizeof(myri10ge_gstrings_main_stats); 1767 for (i = 0; i < mgp->num_slices; i++) { 1768 memcpy(data, *myri10ge_gstrings_slice_stats, 1769 sizeof(myri10ge_gstrings_slice_stats)); 1770 data += sizeof(myri10ge_gstrings_slice_stats); 1771 } 1772 break; 1773 } 1774} 1775 1776static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1777{ 1778 struct myri10ge_priv *mgp = netdev_priv(netdev); 1779 1780 switch (sset) { 1781 case ETH_SS_STATS: 1782 return MYRI10GE_MAIN_STATS_LEN + 1783 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1784 default: 1785 return -EOPNOTSUPP; 1786 } 1787} 1788 1789static void 1790myri10ge_get_ethtool_stats(struct net_device *netdev, 1791 struct ethtool_stats *stats, u64 * data) 1792{ 1793 struct myri10ge_priv *mgp = netdev_priv(netdev); 1794 struct myri10ge_slice_state *ss; 1795 struct rtnl_link_stats64 link_stats; 1796 int slice; 1797 int i; 1798 1799 /* force stats update */ 1800 memset(&link_stats, 0, sizeof(link_stats)); 1801 (void)myri10ge_get_stats(netdev, &link_stats); 1802 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1803 data[i] = ((u64 *)&link_stats)[i]; 1804 1805 data[i++] = (unsigned int)mgp->tx_boundary; 1806 data[i++] = (unsigned int)mgp->pdev->irq; 1807 data[i++] = (unsigned int)mgp->msi_enabled; 1808 data[i++] = (unsigned int)mgp->msix_enabled; 1809 data[i++] = (unsigned int)mgp->read_dma; 1810 data[i++] = (unsigned int)mgp->write_dma; 1811 data[i++] = (unsigned int)mgp->read_write_dma; 1812 data[i++] = (unsigned int)mgp->serial_number; 1813 data[i++] = (unsigned int)mgp->watchdog_resets; 1814#ifdef CONFIG_MYRI10GE_DCA 1815 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1816 data[i++] = (unsigned int)(mgp->dca_enabled); 1817#endif 1818 data[i++] = (unsigned int)mgp->link_changes; 1819 1820 /* firmware stats are useful only in the first slice */ 1821 ss = &mgp->ss[0]; 1822 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1823 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1824 data[i++] = 1825 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1826 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1827 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1828 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1829 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1830 data[i++] = 1831 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1832 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1833 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1834 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1835 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1836 1837 for (slice = 0; slice < mgp->num_slices; slice++) { 1838 ss = &mgp->ss[slice]; 1839 data[i++] = slice; 1840 data[i++] = (unsigned int)ss->tx.pkt_start; 1841 data[i++] = (unsigned int)ss->tx.pkt_done; 1842 data[i++] = (unsigned int)ss->tx.req; 1843 data[i++] = (unsigned int)ss->tx.done; 1844 data[i++] = (unsigned int)ss->rx_small.cnt; 1845 data[i++] = (unsigned int)ss->rx_big.cnt; 1846 data[i++] = (unsigned int)ss->tx.wake_queue; 1847 data[i++] = (unsigned int)ss->tx.stop_queue; 1848 data[i++] = (unsigned int)ss->tx.linearized; 1849 } 1850} 1851 1852static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1853{ 1854 struct myri10ge_priv *mgp = netdev_priv(netdev); 1855 mgp->msg_enable = value; 1856} 1857 1858static u32 myri10ge_get_msglevel(struct net_device *netdev) 1859{ 1860 struct myri10ge_priv *mgp = netdev_priv(netdev); 1861 return mgp->msg_enable; 1862} 1863 1864/* 1865 * Use a low-level command to change the LED behavior. Rather than 1866 * blinking (which is the normal case), when identify is used, the 1867 * yellow LED turns solid. 1868 */ 1869static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1870{ 1871 struct mcp_gen_header *hdr; 1872 struct device *dev = &mgp->pdev->dev; 1873 size_t hdr_off, pattern_off, hdr_len; 1874 u32 pattern = 0xfffffffe; 1875 1876 /* find running firmware header */ 1877 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1878 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1879 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1880 (int)hdr_off); 1881 return -EIO; 1882 } 1883 hdr_len = swab32(readl(mgp->sram + hdr_off + 1884 offsetof(struct mcp_gen_header, header_length))); 1885 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1886 if (pattern_off >= (hdr_len + hdr_off)) { 1887 dev_info(dev, "Firmware does not support LED identification\n"); 1888 return -EINVAL; 1889 } 1890 if (!on) 1891 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1892 writel(swab32(pattern), mgp->sram + pattern_off); 1893 return 0; 1894} 1895 1896static int 1897myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1898{ 1899 struct myri10ge_priv *mgp = netdev_priv(netdev); 1900 int rc; 1901 1902 switch (state) { 1903 case ETHTOOL_ID_ACTIVE: 1904 rc = myri10ge_led(mgp, 1); 1905 break; 1906 1907 case ETHTOOL_ID_INACTIVE: 1908 rc = myri10ge_led(mgp, 0); 1909 break; 1910 1911 default: 1912 rc = -EINVAL; 1913 } 1914 1915 return rc; 1916} 1917 1918static const struct ethtool_ops myri10ge_ethtool_ops = { 1919 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1920 .get_drvinfo = myri10ge_get_drvinfo, 1921 .get_coalesce = myri10ge_get_coalesce, 1922 .set_coalesce = myri10ge_set_coalesce, 1923 .get_pauseparam = myri10ge_get_pauseparam, 1924 .set_pauseparam = myri10ge_set_pauseparam, 1925 .get_ringparam = myri10ge_get_ringparam, 1926 .get_link = ethtool_op_get_link, 1927 .get_strings = myri10ge_get_strings, 1928 .get_sset_count = myri10ge_get_sset_count, 1929 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1930 .set_msglevel = myri10ge_set_msglevel, 1931 .get_msglevel = myri10ge_get_msglevel, 1932 .set_phys_id = myri10ge_phys_id, 1933 .get_link_ksettings = myri10ge_get_link_ksettings, 1934}; 1935 1936static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1937{ 1938 struct myri10ge_priv *mgp = ss->mgp; 1939 struct myri10ge_cmd cmd; 1940 struct net_device *dev = mgp->dev; 1941 int tx_ring_size, rx_ring_size; 1942 int tx_ring_entries, rx_ring_entries; 1943 int i, slice, status; 1944 size_t bytes; 1945 1946 /* get ring sizes */ 1947 slice = ss - mgp->ss; 1948 cmd.data0 = slice; 1949 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1950 tx_ring_size = cmd.data0; 1951 cmd.data0 = slice; 1952 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1953 if (status != 0) 1954 return status; 1955 rx_ring_size = cmd.data0; 1956 1957 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1958 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1959 ss->tx.mask = tx_ring_entries - 1; 1960 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1961 1962 status = -ENOMEM; 1963 1964 /* allocate the host shadow rings */ 1965 1966 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1967 * sizeof(*ss->tx.req_list); 1968 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1969 if (ss->tx.req_bytes == NULL) 1970 goto abort_with_nothing; 1971 1972 /* ensure req_list entries are aligned to 8 bytes */ 1973 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1974 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1975 ss->tx.queue_active = 0; 1976 1977 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1978 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1979 if (ss->rx_small.shadow == NULL) 1980 goto abort_with_tx_req_bytes; 1981 1982 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1983 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1984 if (ss->rx_big.shadow == NULL) 1985 goto abort_with_rx_small_shadow; 1986 1987 /* allocate the host info rings */ 1988 1989 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1990 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1991 if (ss->tx.info == NULL) 1992 goto abort_with_rx_big_shadow; 1993 1994 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1995 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1996 if (ss->rx_small.info == NULL) 1997 goto abort_with_tx_info; 1998 1999 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 2000 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 2001 if (ss->rx_big.info == NULL) 2002 goto abort_with_rx_small_info; 2003 2004 /* Fill the receive rings */ 2005 ss->rx_big.cnt = 0; 2006 ss->rx_small.cnt = 0; 2007 ss->rx_big.fill_cnt = 0; 2008 ss->rx_small.fill_cnt = 0; 2009 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2010 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2011 ss->rx_small.watchdog_needed = 0; 2012 ss->rx_big.watchdog_needed = 0; 2013 if (mgp->small_bytes == 0) { 2014 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2015 } else { 2016 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2017 mgp->small_bytes + MXGEFW_PAD, 0); 2018 } 2019 2020 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2021 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2022 slice, ss->rx_small.fill_cnt); 2023 goto abort_with_rx_small_ring; 2024 } 2025 2026 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2027 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2028 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2029 slice, ss->rx_big.fill_cnt); 2030 goto abort_with_rx_big_ring; 2031 } 2032 2033 return 0; 2034 2035abort_with_rx_big_ring: 2036 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2037 int idx = i & ss->rx_big.mask; 2038 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2039 mgp->big_bytes); 2040 put_page(ss->rx_big.info[idx].page); 2041 } 2042 2043abort_with_rx_small_ring: 2044 if (mgp->small_bytes == 0) 2045 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2046 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2047 int idx = i & ss->rx_small.mask; 2048 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2049 mgp->small_bytes + MXGEFW_PAD); 2050 put_page(ss->rx_small.info[idx].page); 2051 } 2052 2053 kfree(ss->rx_big.info); 2054 2055abort_with_rx_small_info: 2056 kfree(ss->rx_small.info); 2057 2058abort_with_tx_info: 2059 kfree(ss->tx.info); 2060 2061abort_with_rx_big_shadow: 2062 kfree(ss->rx_big.shadow); 2063 2064abort_with_rx_small_shadow: 2065 kfree(ss->rx_small.shadow); 2066 2067abort_with_tx_req_bytes: 2068 kfree(ss->tx.req_bytes); 2069 ss->tx.req_bytes = NULL; 2070 ss->tx.req_list = NULL; 2071 2072abort_with_nothing: 2073 return status; 2074} 2075 2076static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2077{ 2078 struct myri10ge_priv *mgp = ss->mgp; 2079 struct sk_buff *skb; 2080 struct myri10ge_tx_buf *tx; 2081 int i, len, idx; 2082 2083 /* If not allocated, skip it */ 2084 if (ss->tx.req_list == NULL) 2085 return; 2086 2087 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2088 idx = i & ss->rx_big.mask; 2089 if (i == ss->rx_big.fill_cnt - 1) 2090 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2091 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2092 mgp->big_bytes); 2093 put_page(ss->rx_big.info[idx].page); 2094 } 2095 2096 if (mgp->small_bytes == 0) 2097 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2098 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2099 idx = i & ss->rx_small.mask; 2100 if (i == ss->rx_small.fill_cnt - 1) 2101 ss->rx_small.info[idx].page_offset = 2102 MYRI10GE_ALLOC_SIZE; 2103 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2104 mgp->small_bytes + MXGEFW_PAD); 2105 put_page(ss->rx_small.info[idx].page); 2106 } 2107 tx = &ss->tx; 2108 while (tx->done != tx->req) { 2109 idx = tx->done & tx->mask; 2110 skb = tx->info[idx].skb; 2111 2112 /* Mark as free */ 2113 tx->info[idx].skb = NULL; 2114 tx->done++; 2115 len = dma_unmap_len(&tx->info[idx], len); 2116 dma_unmap_len_set(&tx->info[idx], len, 0); 2117 if (skb) { 2118 ss->stats.tx_dropped++; 2119 dev_kfree_skb_any(skb); 2120 if (len) 2121 dma_unmap_single(&mgp->pdev->dev, 2122 dma_unmap_addr(&tx->info[idx], 2123 bus), len, 2124 DMA_TO_DEVICE); 2125 } else { 2126 if (len) 2127 dma_unmap_page(&mgp->pdev->dev, 2128 dma_unmap_addr(&tx->info[idx], 2129 bus), len, 2130 DMA_TO_DEVICE); 2131 } 2132 } 2133 kfree(ss->rx_big.info); 2134 2135 kfree(ss->rx_small.info); 2136 2137 kfree(ss->tx.info); 2138 2139 kfree(ss->rx_big.shadow); 2140 2141 kfree(ss->rx_small.shadow); 2142 2143 kfree(ss->tx.req_bytes); 2144 ss->tx.req_bytes = NULL; 2145 ss->tx.req_list = NULL; 2146} 2147 2148static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2149{ 2150 struct pci_dev *pdev = mgp->pdev; 2151 struct myri10ge_slice_state *ss; 2152 struct net_device *netdev = mgp->dev; 2153 int i; 2154 int status; 2155 2156 mgp->msi_enabled = 0; 2157 mgp->msix_enabled = 0; 2158 status = 0; 2159 if (myri10ge_msi) { 2160 if (mgp->num_slices > 1) { 2161 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2162 mgp->num_slices, mgp->num_slices); 2163 if (status < 0) { 2164 dev_err(&pdev->dev, 2165 "Error %d setting up MSI-X\n", status); 2166 return status; 2167 } 2168 mgp->msix_enabled = 1; 2169 } 2170 if (mgp->msix_enabled == 0) { 2171 status = pci_enable_msi(pdev); 2172 if (status != 0) { 2173 dev_err(&pdev->dev, 2174 "Error %d setting up MSI; falling back to xPIC\n", 2175 status); 2176 } else { 2177 mgp->msi_enabled = 1; 2178 } 2179 } 2180 } 2181 if (mgp->msix_enabled) { 2182 for (i = 0; i < mgp->num_slices; i++) { 2183 ss = &mgp->ss[i]; 2184 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2185 "%s:slice-%d", netdev->name, i); 2186 status = request_irq(mgp->msix_vectors[i].vector, 2187 myri10ge_intr, 0, ss->irq_desc, 2188 ss); 2189 if (status != 0) { 2190 dev_err(&pdev->dev, 2191 "slice %d failed to allocate IRQ\n", i); 2192 i--; 2193 while (i >= 0) { 2194 free_irq(mgp->msix_vectors[i].vector, 2195 &mgp->ss[i]); 2196 i--; 2197 } 2198 pci_disable_msix(pdev); 2199 return status; 2200 } 2201 } 2202 } else { 2203 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2204 mgp->dev->name, &mgp->ss[0]); 2205 if (status != 0) { 2206 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2207 if (mgp->msi_enabled) 2208 pci_disable_msi(pdev); 2209 } 2210 } 2211 return status; 2212} 2213 2214static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2215{ 2216 struct pci_dev *pdev = mgp->pdev; 2217 int i; 2218 2219 if (mgp->msix_enabled) { 2220 for (i = 0; i < mgp->num_slices; i++) 2221 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2222 } else { 2223 free_irq(pdev->irq, &mgp->ss[0]); 2224 } 2225 if (mgp->msi_enabled) 2226 pci_disable_msi(pdev); 2227 if (mgp->msix_enabled) 2228 pci_disable_msix(pdev); 2229} 2230 2231static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2232{ 2233 struct myri10ge_cmd cmd; 2234 struct myri10ge_slice_state *ss; 2235 int status; 2236 2237 ss = &mgp->ss[slice]; 2238 status = 0; 2239 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2240 cmd.data0 = slice; 2241 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2242 &cmd, 0); 2243 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2244 (mgp->sram + cmd.data0); 2245 } 2246 cmd.data0 = slice; 2247 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2248 &cmd, 0); 2249 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2250 (mgp->sram + cmd.data0); 2251 2252 cmd.data0 = slice; 2253 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2254 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2255 (mgp->sram + cmd.data0); 2256 2257 ss->tx.send_go = (__iomem __be32 *) 2258 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2259 ss->tx.send_stop = (__iomem __be32 *) 2260 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2261 return status; 2262 2263} 2264 2265static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2266{ 2267 struct myri10ge_cmd cmd; 2268 struct myri10ge_slice_state *ss; 2269 int status; 2270 2271 ss = &mgp->ss[slice]; 2272 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2273 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2274 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2275 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2276 if (status == -ENOSYS) { 2277 dma_addr_t bus = ss->fw_stats_bus; 2278 if (slice != 0) 2279 return -EINVAL; 2280 bus += offsetof(struct mcp_irq_data, send_done_count); 2281 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2282 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2283 status = myri10ge_send_cmd(mgp, 2284 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2285 &cmd, 0); 2286 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2287 mgp->fw_multicast_support = 0; 2288 } else { 2289 mgp->fw_multicast_support = 1; 2290 } 2291 return 0; 2292} 2293 2294static int myri10ge_open(struct net_device *dev) 2295{ 2296 struct myri10ge_slice_state *ss; 2297 struct myri10ge_priv *mgp = netdev_priv(dev); 2298 struct myri10ge_cmd cmd; 2299 int i, status, big_pow2, slice; 2300 u8 __iomem *itable; 2301 2302 if (mgp->running != MYRI10GE_ETH_STOPPED) 2303 return -EBUSY; 2304 2305 mgp->running = MYRI10GE_ETH_STARTING; 2306 status = myri10ge_reset(mgp); 2307 if (status != 0) { 2308 netdev_err(dev, "failed reset\n"); 2309 goto abort_with_nothing; 2310 } 2311 2312 if (mgp->num_slices > 1) { 2313 cmd.data0 = mgp->num_slices; 2314 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2315 if (mgp->dev->real_num_tx_queues > 1) 2316 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2317 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2318 &cmd, 0); 2319 if (status != 0) { 2320 netdev_err(dev, "failed to set number of slices\n"); 2321 goto abort_with_nothing; 2322 } 2323 /* setup the indirection table */ 2324 cmd.data0 = mgp->num_slices; 2325 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2326 &cmd, 0); 2327 2328 status |= myri10ge_send_cmd(mgp, 2329 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2330 &cmd, 0); 2331 if (status != 0) { 2332 netdev_err(dev, "failed to setup rss tables\n"); 2333 goto abort_with_nothing; 2334 } 2335 2336 /* just enable an identity mapping */ 2337 itable = mgp->sram + cmd.data0; 2338 for (i = 0; i < mgp->num_slices; i++) 2339 __raw_writeb(i, &itable[i]); 2340 2341 cmd.data0 = 1; 2342 cmd.data1 = myri10ge_rss_hash; 2343 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2344 &cmd, 0); 2345 if (status != 0) { 2346 netdev_err(dev, "failed to enable slices\n"); 2347 goto abort_with_nothing; 2348 } 2349 } 2350 2351 status = myri10ge_request_irq(mgp); 2352 if (status != 0) 2353 goto abort_with_nothing; 2354 2355 /* decide what small buffer size to use. For good TCP rx 2356 * performance, it is important to not receive 1514 byte 2357 * frames into jumbo buffers, as it confuses the socket buffer 2358 * accounting code, leading to drops and erratic performance. 2359 */ 2360 2361 if (dev->mtu <= ETH_DATA_LEN) 2362 /* enough for a TCP header */ 2363 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2364 ? (128 - MXGEFW_PAD) 2365 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2366 else 2367 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2368 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2369 2370 /* Override the small buffer size? */ 2371 if (myri10ge_small_bytes >= 0) 2372 mgp->small_bytes = myri10ge_small_bytes; 2373 2374 /* Firmware needs the big buff size as a power of 2. Lie and 2375 * tell him the buffer is larger, because we only use 1 2376 * buffer/pkt, and the mtu will prevent overruns. 2377 */ 2378 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2379 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2380 while (!is_power_of_2(big_pow2)) 2381 big_pow2++; 2382 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2383 } else { 2384 big_pow2 = MYRI10GE_ALLOC_SIZE; 2385 mgp->big_bytes = big_pow2; 2386 } 2387 2388 /* setup the per-slice data structures */ 2389 for (slice = 0; slice < mgp->num_slices; slice++) { 2390 ss = &mgp->ss[slice]; 2391 2392 status = myri10ge_get_txrx(mgp, slice); 2393 if (status != 0) { 2394 netdev_err(dev, "failed to get ring sizes or locations\n"); 2395 goto abort_with_rings; 2396 } 2397 status = myri10ge_allocate_rings(ss); 2398 if (status != 0) 2399 goto abort_with_rings; 2400 2401 /* only firmware which supports multiple TX queues 2402 * supports setting up the tx stats on non-zero 2403 * slices */ 2404 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2405 status = myri10ge_set_stats(mgp, slice); 2406 if (status) { 2407 netdev_err(dev, "Couldn't set stats DMA\n"); 2408 goto abort_with_rings; 2409 } 2410 2411 /* must happen prior to any irq */ 2412 napi_enable(&(ss)->napi); 2413 } 2414 2415 /* now give firmware buffers sizes, and MTU */ 2416 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2417 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2418 cmd.data0 = mgp->small_bytes; 2419 status |= 2420 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2421 cmd.data0 = big_pow2; 2422 status |= 2423 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2424 if (status) { 2425 netdev_err(dev, "Couldn't set buffer sizes\n"); 2426 goto abort_with_rings; 2427 } 2428 2429 /* 2430 * Set Linux style TSO mode; this is needed only on newer 2431 * firmware versions. Older versions default to Linux 2432 * style TSO 2433 */ 2434 cmd.data0 = 0; 2435 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2436 if (status && status != -ENOSYS) { 2437 netdev_err(dev, "Couldn't set TSO mode\n"); 2438 goto abort_with_rings; 2439 } 2440 2441 mgp->link_state = ~0U; 2442 mgp->rdma_tags_available = 15; 2443 2444 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2445 if (status) { 2446 netdev_err(dev, "Couldn't bring up link\n"); 2447 goto abort_with_rings; 2448 } 2449 2450 mgp->running = MYRI10GE_ETH_RUNNING; 2451 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2452 add_timer(&mgp->watchdog_timer); 2453 netif_tx_wake_all_queues(dev); 2454 2455 return 0; 2456 2457abort_with_rings: 2458 while (slice) { 2459 slice--; 2460 napi_disable(&mgp->ss[slice].napi); 2461 } 2462 for (i = 0; i < mgp->num_slices; i++) 2463 myri10ge_free_rings(&mgp->ss[i]); 2464 2465 myri10ge_free_irq(mgp); 2466 2467abort_with_nothing: 2468 mgp->running = MYRI10GE_ETH_STOPPED; 2469 return -ENOMEM; 2470} 2471 2472static int myri10ge_close(struct net_device *dev) 2473{ 2474 struct myri10ge_priv *mgp = netdev_priv(dev); 2475 struct myri10ge_cmd cmd; 2476 int status, old_down_cnt; 2477 int i; 2478 2479 if (mgp->running != MYRI10GE_ETH_RUNNING) 2480 return 0; 2481 2482 if (mgp->ss[0].tx.req_bytes == NULL) 2483 return 0; 2484 2485 del_timer_sync(&mgp->watchdog_timer); 2486 mgp->running = MYRI10GE_ETH_STOPPING; 2487 for (i = 0; i < mgp->num_slices; i++) 2488 napi_disable(&mgp->ss[i].napi); 2489 2490 netif_carrier_off(dev); 2491 2492 netif_tx_stop_all_queues(dev); 2493 if (mgp->rebooted == 0) { 2494 old_down_cnt = mgp->down_cnt; 2495 mb(); 2496 status = 2497 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2498 if (status) 2499 netdev_err(dev, "Couldn't bring down link\n"); 2500 2501 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2502 HZ); 2503 if (old_down_cnt == mgp->down_cnt) 2504 netdev_err(dev, "never got down irq\n"); 2505 } 2506 netif_tx_disable(dev); 2507 myri10ge_free_irq(mgp); 2508 for (i = 0; i < mgp->num_slices; i++) 2509 myri10ge_free_rings(&mgp->ss[i]); 2510 2511 mgp->running = MYRI10GE_ETH_STOPPED; 2512 return 0; 2513} 2514 2515/* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2516 * backwards one at a time and handle ring wraps */ 2517 2518static inline void 2519myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2520 struct mcp_kreq_ether_send *src, int cnt) 2521{ 2522 int idx, starting_slot; 2523 starting_slot = tx->req; 2524 while (cnt > 1) { 2525 cnt--; 2526 idx = (starting_slot + cnt) & tx->mask; 2527 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2528 mb(); 2529 } 2530} 2531 2532/* 2533 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2534 * at most 32 bytes at a time, so as to avoid involving the software 2535 * pio handler in the nic. We re-write the first segment's flags 2536 * to mark them valid only after writing the entire chain. 2537 */ 2538 2539static inline void 2540myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2541 int cnt) 2542{ 2543 int idx, i; 2544 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2545 struct mcp_kreq_ether_send *srcp; 2546 u8 last_flags; 2547 2548 idx = tx->req & tx->mask; 2549 2550 last_flags = src->flags; 2551 src->flags = 0; 2552 mb(); 2553 dst = dstp = &tx->lanai[idx]; 2554 srcp = src; 2555 2556 if ((idx + cnt) < tx->mask) { 2557 for (i = 0; i < (cnt - 1); i += 2) { 2558 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2559 mb(); /* force write every 32 bytes */ 2560 srcp += 2; 2561 dstp += 2; 2562 } 2563 } else { 2564 /* submit all but the first request, and ensure 2565 * that it is submitted below */ 2566 myri10ge_submit_req_backwards(tx, src, cnt); 2567 i = 0; 2568 } 2569 if (i < cnt) { 2570 /* submit the first request */ 2571 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2572 mb(); /* barrier before setting valid flag */ 2573 } 2574 2575 /* re-write the last 32-bits with the valid flags */ 2576 src->flags = last_flags; 2577 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2578 tx->req += cnt; 2579 mb(); 2580} 2581 2582static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2583 struct myri10ge_tx_buf *tx, int idx) 2584{ 2585 unsigned int len; 2586 int last_idx; 2587 2588 /* Free any DMA resources we've alloced and clear out the skb slot */ 2589 last_idx = (idx + 1) & tx->mask; 2590 idx = tx->req & tx->mask; 2591 do { 2592 len = dma_unmap_len(&tx->info[idx], len); 2593 if (len) { 2594 if (tx->info[idx].skb != NULL) 2595 dma_unmap_single(&mgp->pdev->dev, 2596 dma_unmap_addr(&tx->info[idx], 2597 bus), len, 2598 DMA_TO_DEVICE); 2599 else 2600 dma_unmap_page(&mgp->pdev->dev, 2601 dma_unmap_addr(&tx->info[idx], 2602 bus), len, 2603 DMA_TO_DEVICE); 2604 dma_unmap_len_set(&tx->info[idx], len, 0); 2605 tx->info[idx].skb = NULL; 2606 } 2607 idx = (idx + 1) & tx->mask; 2608 } while (idx != last_idx); 2609} 2610 2611/* 2612 * Transmit a packet. We need to split the packet so that a single 2613 * segment does not cross myri10ge->tx_boundary, so this makes segment 2614 * counting tricky. So rather than try to count segments up front, we 2615 * just give up if there are too few segments to hold a reasonably 2616 * fragmented packet currently available. If we run 2617 * out of segments while preparing a packet for DMA, we just linearize 2618 * it and try again. 2619 */ 2620 2621static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2622 struct net_device *dev) 2623{ 2624 struct myri10ge_priv *mgp = netdev_priv(dev); 2625 struct myri10ge_slice_state *ss; 2626 struct mcp_kreq_ether_send *req; 2627 struct myri10ge_tx_buf *tx; 2628 skb_frag_t *frag; 2629 struct netdev_queue *netdev_queue; 2630 dma_addr_t bus; 2631 u32 low; 2632 __be32 high_swapped; 2633 unsigned int len; 2634 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2635 u16 pseudo_hdr_offset, cksum_offset, queue; 2636 int cum_len, seglen, boundary, rdma_count; 2637 u8 flags, odd_flag; 2638 2639 queue = skb_get_queue_mapping(skb); 2640 ss = &mgp->ss[queue]; 2641 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2642 tx = &ss->tx; 2643 2644again: 2645 req = tx->req_list; 2646 avail = tx->mask - 1 - (tx->req - tx->done); 2647 2648 mss = 0; 2649 max_segments = MXGEFW_MAX_SEND_DESC; 2650 2651 if (skb_is_gso(skb)) { 2652 mss = skb_shinfo(skb)->gso_size; 2653 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2654 } 2655 2656 if ((unlikely(avail < max_segments))) { 2657 /* we are out of transmit resources */ 2658 tx->stop_queue++; 2659 netif_tx_stop_queue(netdev_queue); 2660 return NETDEV_TX_BUSY; 2661 } 2662 2663 /* Setup checksum offloading, if needed */ 2664 cksum_offset = 0; 2665 pseudo_hdr_offset = 0; 2666 odd_flag = 0; 2667 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2668 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2669 cksum_offset = skb_checksum_start_offset(skb); 2670 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2671 /* If the headers are excessively large, then we must 2672 * fall back to a software checksum */ 2673 if (unlikely(!mss && (cksum_offset > 255 || 2674 pseudo_hdr_offset > 127))) { 2675 if (skb_checksum_help(skb)) 2676 goto drop; 2677 cksum_offset = 0; 2678 pseudo_hdr_offset = 0; 2679 } else { 2680 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2681 flags |= MXGEFW_FLAGS_CKSUM; 2682 } 2683 } 2684 2685 cum_len = 0; 2686 2687 if (mss) { /* TSO */ 2688 /* this removes any CKSUM flag from before */ 2689 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2690 2691 /* negative cum_len signifies to the 2692 * send loop that we are still in the 2693 * header portion of the TSO packet. 2694 * TSO header can be at most 1KB long */ 2695 cum_len = -skb_tcp_all_headers(skb); 2696 2697 /* for IPv6 TSO, the checksum offset stores the 2698 * TCP header length, to save the firmware from 2699 * the need to parse the headers */ 2700 if (skb_is_gso_v6(skb)) { 2701 cksum_offset = tcp_hdrlen(skb); 2702 /* Can only handle headers <= max_tso6 long */ 2703 if (unlikely(-cum_len > mgp->max_tso6)) 2704 return myri10ge_sw_tso(skb, dev); 2705 } 2706 /* for TSO, pseudo_hdr_offset holds mss. 2707 * The firmware figures out where to put 2708 * the checksum by parsing the header. */ 2709 pseudo_hdr_offset = mss; 2710 } else 2711 /* Mark small packets, and pad out tiny packets */ 2712 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2713 flags |= MXGEFW_FLAGS_SMALL; 2714 2715 /* pad frames to at least ETH_ZLEN bytes */ 2716 if (eth_skb_pad(skb)) { 2717 /* The packet is gone, so we must 2718 * return 0 */ 2719 ss->stats.tx_dropped += 1; 2720 return NETDEV_TX_OK; 2721 } 2722 } 2723 2724 /* map the skb for DMA */ 2725 len = skb_headlen(skb); 2726 bus = dma_map_single(&mgp->pdev->dev, skb->data, len, DMA_TO_DEVICE); 2727 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) 2728 goto drop; 2729 2730 idx = tx->req & tx->mask; 2731 tx->info[idx].skb = skb; 2732 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2733 dma_unmap_len_set(&tx->info[idx], len, len); 2734 2735 frag_cnt = skb_shinfo(skb)->nr_frags; 2736 frag_idx = 0; 2737 count = 0; 2738 rdma_count = 0; 2739 2740 /* "rdma_count" is the number of RDMAs belonging to the 2741 * current packet BEFORE the current send request. For 2742 * non-TSO packets, this is equal to "count". 2743 * For TSO packets, rdma_count needs to be reset 2744 * to 0 after a segment cut. 2745 * 2746 * The rdma_count field of the send request is 2747 * the number of RDMAs of the packet starting at 2748 * that request. For TSO send requests with one ore more cuts 2749 * in the middle, this is the number of RDMAs starting 2750 * after the last cut in the request. All previous 2751 * segments before the last cut implicitly have 1 RDMA. 2752 * 2753 * Since the number of RDMAs is not known beforehand, 2754 * it must be filled-in retroactively - after each 2755 * segmentation cut or at the end of the entire packet. 2756 */ 2757 2758 while (1) { 2759 /* Break the SKB or Fragment up into pieces which 2760 * do not cross mgp->tx_boundary */ 2761 low = MYRI10GE_LOWPART_TO_U32(bus); 2762 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2763 while (len) { 2764 u8 flags_next; 2765 int cum_len_next; 2766 2767 if (unlikely(count == max_segments)) 2768 goto abort_linearize; 2769 2770 boundary = 2771 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2772 seglen = boundary - low; 2773 if (seglen > len) 2774 seglen = len; 2775 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2776 cum_len_next = cum_len + seglen; 2777 if (mss) { /* TSO */ 2778 (req - rdma_count)->rdma_count = rdma_count + 1; 2779 2780 if (likely(cum_len >= 0)) { /* payload */ 2781 int next_is_first, chop; 2782 2783 chop = (cum_len_next > mss); 2784 cum_len_next = cum_len_next % mss; 2785 next_is_first = (cum_len_next == 0); 2786 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2787 flags_next |= next_is_first * 2788 MXGEFW_FLAGS_FIRST; 2789 rdma_count |= -(chop | next_is_first); 2790 rdma_count += chop & ~next_is_first; 2791 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2792 int small; 2793 2794 rdma_count = -1; 2795 cum_len_next = 0; 2796 seglen = -cum_len; 2797 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2798 flags_next = MXGEFW_FLAGS_TSO_PLD | 2799 MXGEFW_FLAGS_FIRST | 2800 (small * MXGEFW_FLAGS_SMALL); 2801 } 2802 } 2803 req->addr_high = high_swapped; 2804 req->addr_low = htonl(low); 2805 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2806 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2807 req->rdma_count = 1; 2808 req->length = htons(seglen); 2809 req->cksum_offset = cksum_offset; 2810 req->flags = flags | ((cum_len & 1) * odd_flag); 2811 2812 low += seglen; 2813 len -= seglen; 2814 cum_len = cum_len_next; 2815 flags = flags_next; 2816 req++; 2817 count++; 2818 rdma_count++; 2819 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2820 if (unlikely(cksum_offset > seglen)) 2821 cksum_offset -= seglen; 2822 else 2823 cksum_offset = 0; 2824 } 2825 } 2826 if (frag_idx == frag_cnt) 2827 break; 2828 2829 /* map next fragment for DMA */ 2830 frag = &skb_shinfo(skb)->frags[frag_idx]; 2831 frag_idx++; 2832 len = skb_frag_size(frag); 2833 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2834 DMA_TO_DEVICE); 2835 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 2836 myri10ge_unmap_tx_dma(mgp, tx, idx); 2837 goto drop; 2838 } 2839 idx = (count + tx->req) & tx->mask; 2840 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2841 dma_unmap_len_set(&tx->info[idx], len, len); 2842 } 2843 2844 (req - rdma_count)->rdma_count = rdma_count; 2845 if (mss) 2846 do { 2847 req--; 2848 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2849 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2850 MXGEFW_FLAGS_FIRST))); 2851 idx = ((count - 1) + tx->req) & tx->mask; 2852 tx->info[idx].last = 1; 2853 myri10ge_submit_req(tx, tx->req_list, count); 2854 /* if using multiple tx queues, make sure NIC polls the 2855 * current slice */ 2856 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2857 tx->queue_active = 1; 2858 put_be32(htonl(1), tx->send_go); 2859 mb(); 2860 } 2861 tx->pkt_start++; 2862 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2863 tx->stop_queue++; 2864 netif_tx_stop_queue(netdev_queue); 2865 } 2866 return NETDEV_TX_OK; 2867 2868abort_linearize: 2869 myri10ge_unmap_tx_dma(mgp, tx, idx); 2870 2871 if (skb_is_gso(skb)) { 2872 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2873 goto drop; 2874 } 2875 2876 if (skb_linearize(skb)) 2877 goto drop; 2878 2879 tx->linearized++; 2880 goto again; 2881 2882drop: 2883 dev_kfree_skb_any(skb); 2884 ss->stats.tx_dropped += 1; 2885 return NETDEV_TX_OK; 2886 2887} 2888 2889static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2890 struct net_device *dev) 2891{ 2892 struct sk_buff *segs, *curr, *next; 2893 struct myri10ge_priv *mgp = netdev_priv(dev); 2894 struct myri10ge_slice_state *ss; 2895 netdev_tx_t status; 2896 2897 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2898 if (IS_ERR(segs)) 2899 goto drop; 2900 2901 skb_list_walk_safe(segs, curr, next) { 2902 skb_mark_not_on_list(curr); 2903 status = myri10ge_xmit(curr, dev); 2904 if (status != 0) { 2905 dev_kfree_skb_any(curr); 2906 skb_list_walk_safe(next, curr, next) { 2907 curr->next = NULL; 2908 dev_kfree_skb_any(curr); 2909 } 2910 goto drop; 2911 } 2912 } 2913 dev_kfree_skb_any(skb); 2914 return NETDEV_TX_OK; 2915 2916drop: 2917 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2918 dev_kfree_skb_any(skb); 2919 ss->stats.tx_dropped += 1; 2920 return NETDEV_TX_OK; 2921} 2922 2923static void myri10ge_get_stats(struct net_device *dev, 2924 struct rtnl_link_stats64 *stats) 2925{ 2926 const struct myri10ge_priv *mgp = netdev_priv(dev); 2927 const struct myri10ge_slice_netstats *slice_stats; 2928 int i; 2929 2930 for (i = 0; i < mgp->num_slices; i++) { 2931 slice_stats = &mgp->ss[i].stats; 2932 stats->rx_packets += slice_stats->rx_packets; 2933 stats->tx_packets += slice_stats->tx_packets; 2934 stats->rx_bytes += slice_stats->rx_bytes; 2935 stats->tx_bytes += slice_stats->tx_bytes; 2936 stats->rx_dropped += slice_stats->rx_dropped; 2937 stats->tx_dropped += slice_stats->tx_dropped; 2938 } 2939} 2940 2941static void myri10ge_set_multicast_list(struct net_device *dev) 2942{ 2943 struct myri10ge_priv *mgp = netdev_priv(dev); 2944 struct myri10ge_cmd cmd; 2945 struct netdev_hw_addr *ha; 2946 __be32 data[2] = { 0, 0 }; 2947 int err; 2948 2949 /* can be called from atomic contexts, 2950 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2951 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2952 2953 /* This firmware is known to not support multicast */ 2954 if (!mgp->fw_multicast_support) 2955 return; 2956 2957 /* Disable multicast filtering */ 2958 2959 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2960 if (err != 0) { 2961 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2962 err); 2963 goto abort; 2964 } 2965 2966 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2967 /* request to disable multicast filtering, so quit here */ 2968 return; 2969 } 2970 2971 /* Flush the filters */ 2972 2973 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2974 &cmd, 1); 2975 if (err != 0) { 2976 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2977 err); 2978 goto abort; 2979 } 2980 2981 /* Walk the multicast list, and add each address */ 2982 netdev_for_each_mc_addr(ha, dev) { 2983 memcpy(data, &ha->addr, ETH_ALEN); 2984 cmd.data0 = ntohl(data[0]); 2985 cmd.data1 = ntohl(data[1]); 2986 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2987 &cmd, 1); 2988 2989 if (err != 0) { 2990 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2991 err, ha->addr); 2992 goto abort; 2993 } 2994 } 2995 /* Enable multicast filtering */ 2996 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2997 if (err != 0) { 2998 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 2999 err); 3000 goto abort; 3001 } 3002 3003 return; 3004 3005abort: 3006 return; 3007} 3008 3009static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3010{ 3011 struct sockaddr *sa = addr; 3012 struct myri10ge_priv *mgp = netdev_priv(dev); 3013 int status; 3014 3015 if (!is_valid_ether_addr(sa->sa_data)) 3016 return -EADDRNOTAVAIL; 3017 3018 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3019 if (status != 0) { 3020 netdev_err(dev, "changing mac address failed with %d\n", 3021 status); 3022 return status; 3023 } 3024 3025 /* change the dev structure */ 3026 eth_hw_addr_set(dev, sa->sa_data); 3027 return 0; 3028} 3029 3030static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3031{ 3032 struct myri10ge_priv *mgp = netdev_priv(dev); 3033 3034 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3035 if (mgp->running) { 3036 /* if we change the mtu on an active device, we must 3037 * reset the device so the firmware sees the change */ 3038 myri10ge_close(dev); 3039 dev->mtu = new_mtu; 3040 myri10ge_open(dev); 3041 } else 3042 dev->mtu = new_mtu; 3043 3044 return 0; 3045} 3046 3047/* 3048 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3049 * Only do it if the bridge is a root port since we don't want to disturb 3050 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3051 */ 3052 3053static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3054{ 3055 struct pci_dev *bridge = mgp->pdev->bus->self; 3056 struct device *dev = &mgp->pdev->dev; 3057 int cap; 3058 unsigned err_cap; 3059 int ret; 3060 3061 if (!myri10ge_ecrc_enable || !bridge) 3062 return; 3063 3064 /* check that the bridge is a root port */ 3065 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3066 if (myri10ge_ecrc_enable > 1) { 3067 struct pci_dev *prev_bridge, *old_bridge = bridge; 3068 3069 /* Walk the hierarchy up to the root port 3070 * where ECRC has to be enabled */ 3071 do { 3072 prev_bridge = bridge; 3073 bridge = bridge->bus->self; 3074 if (!bridge || prev_bridge == bridge) { 3075 dev_err(dev, 3076 "Failed to find root port" 3077 " to force ECRC\n"); 3078 return; 3079 } 3080 } while (pci_pcie_type(bridge) != 3081 PCI_EXP_TYPE_ROOT_PORT); 3082 3083 dev_info(dev, 3084 "Forcing ECRC on non-root port %s" 3085 " (enabling on root port %s)\n", 3086 pci_name(old_bridge), pci_name(bridge)); 3087 } else { 3088 dev_err(dev, 3089 "Not enabling ECRC on non-root port %s\n", 3090 pci_name(bridge)); 3091 return; 3092 } 3093 } 3094 3095 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3096 if (!cap) 3097 return; 3098 3099 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3100 if (ret) { 3101 dev_err(dev, "failed reading ext-conf-space of %s\n", 3102 pci_name(bridge)); 3103 dev_err(dev, "\t pci=nommconf in use? " 3104 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3105 return; 3106 } 3107 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3108 return; 3109 3110 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3111 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3112 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3113} 3114 3115/* 3116 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3117 * when the PCI-E Completion packets are aligned on an 8-byte 3118 * boundary. Some PCI-E chip sets always align Completion packets; on 3119 * the ones that do not, the alignment can be enforced by enabling 3120 * ECRC generation (if supported). 3121 * 3122 * When PCI-E Completion packets are not aligned, it is actually more 3123 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3124 * 3125 * If the driver can neither enable ECRC nor verify that it has 3126 * already been enabled, then it must use a firmware image which works 3127 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3128 * should also ensure that it never gives the device a Read-DMA which is 3129 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3130 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3131 * firmware image, and set tx_boundary to 4KB. 3132 */ 3133 3134static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3135{ 3136 struct pci_dev *pdev = mgp->pdev; 3137 struct device *dev = &pdev->dev; 3138 int status; 3139 3140 mgp->tx_boundary = 4096; 3141 /* 3142 * Verify the max read request size was set to 4KB 3143 * before trying the test with 4KB. 3144 */ 3145 status = pcie_get_readrq(pdev); 3146 if (status < 0) { 3147 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3148 goto abort; 3149 } 3150 if (status != 4096) { 3151 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3152 mgp->tx_boundary = 2048; 3153 } 3154 /* 3155 * load the optimized firmware (which assumes aligned PCIe 3156 * completions) in order to see if it works on this host. 3157 */ 3158 set_fw_name(mgp, myri10ge_fw_aligned, false); 3159 status = myri10ge_load_firmware(mgp, 1); 3160 if (status != 0) { 3161 goto abort; 3162 } 3163 3164 /* 3165 * Enable ECRC if possible 3166 */ 3167 myri10ge_enable_ecrc(mgp); 3168 3169 /* 3170 * Run a DMA test which watches for unaligned completions and 3171 * aborts on the first one seen. 3172 */ 3173 3174 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3175 if (status == 0) 3176 return; /* keep the aligned firmware */ 3177 3178 if (status != -E2BIG) 3179 dev_warn(dev, "DMA test failed: %d\n", status); 3180 if (status == -ENOSYS) 3181 dev_warn(dev, "Falling back to ethp! " 3182 "Please install up to date fw\n"); 3183abort: 3184 /* fall back to using the unaligned firmware */ 3185 mgp->tx_boundary = 2048; 3186 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3187} 3188 3189static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3190{ 3191 int overridden = 0; 3192 3193 if (myri10ge_force_firmware == 0) { 3194 int link_width; 3195 u16 lnk; 3196 3197 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3198 link_width = (lnk >> 4) & 0x3f; 3199 3200 /* Check to see if Link is less than 8 or if the 3201 * upstream bridge is known to provide aligned 3202 * completions */ 3203 if (link_width < 8) { 3204 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3205 link_width); 3206 mgp->tx_boundary = 4096; 3207 set_fw_name(mgp, myri10ge_fw_aligned, false); 3208 } else { 3209 myri10ge_firmware_probe(mgp); 3210 } 3211 } else { 3212 if (myri10ge_force_firmware == 1) { 3213 dev_info(&mgp->pdev->dev, 3214 "Assuming aligned completions (forced)\n"); 3215 mgp->tx_boundary = 4096; 3216 set_fw_name(mgp, myri10ge_fw_aligned, false); 3217 } else { 3218 dev_info(&mgp->pdev->dev, 3219 "Assuming unaligned completions (forced)\n"); 3220 mgp->tx_boundary = 2048; 3221 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3222 } 3223 } 3224 3225 kernel_param_lock(THIS_MODULE); 3226 if (myri10ge_fw_name != NULL) { 3227 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3228 if (fw_name) { 3229 overridden = 1; 3230 set_fw_name(mgp, fw_name, true); 3231 } 3232 } 3233 kernel_param_unlock(THIS_MODULE); 3234 3235 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3236 myri10ge_fw_names[mgp->board_number] != NULL && 3237 strlen(myri10ge_fw_names[mgp->board_number])) { 3238 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3239 overridden = 1; 3240 } 3241 if (overridden) 3242 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3243 mgp->fw_name); 3244} 3245 3246static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3247{ 3248 struct pci_dev *bridge = pdev->bus->self; 3249 int cap; 3250 u32 mask; 3251 3252 if (bridge == NULL) 3253 return; 3254 3255 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3256 if (cap) { 3257 /* a sram parity error can cause a surprise link 3258 * down; since we expect and can recover from sram 3259 * parity errors, mask surprise link down events */ 3260 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3261 mask |= 0x20; 3262 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3263 } 3264} 3265 3266static int __maybe_unused myri10ge_suspend(struct device *dev) 3267{ 3268 struct myri10ge_priv *mgp; 3269 struct net_device *netdev; 3270 3271 mgp = dev_get_drvdata(dev); 3272 if (mgp == NULL) 3273 return -EINVAL; 3274 netdev = mgp->dev; 3275 3276 netif_device_detach(netdev); 3277 if (netif_running(netdev)) { 3278 netdev_info(netdev, "closing\n"); 3279 rtnl_lock(); 3280 myri10ge_close(netdev); 3281 rtnl_unlock(); 3282 } 3283 myri10ge_dummy_rdma(mgp, 0); 3284 3285 return 0; 3286} 3287 3288static int __maybe_unused myri10ge_resume(struct device *dev) 3289{ 3290 struct pci_dev *pdev = to_pci_dev(dev); 3291 struct myri10ge_priv *mgp; 3292 struct net_device *netdev; 3293 int status; 3294 u16 vendor; 3295 3296 mgp = pci_get_drvdata(pdev); 3297 if (mgp == NULL) 3298 return -EINVAL; 3299 netdev = mgp->dev; 3300 msleep(5); /* give card time to respond */ 3301 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3302 if (vendor == 0xffff) { 3303 netdev_err(mgp->dev, "device disappeared!\n"); 3304 return -EIO; 3305 } 3306 3307 myri10ge_reset(mgp); 3308 myri10ge_dummy_rdma(mgp, 1); 3309 3310 if (netif_running(netdev)) { 3311 rtnl_lock(); 3312 status = myri10ge_open(netdev); 3313 rtnl_unlock(); 3314 if (status != 0) 3315 goto abort_with_enabled; 3316 3317 } 3318 netif_device_attach(netdev); 3319 3320 return 0; 3321 3322abort_with_enabled: 3323 return -EIO; 3324} 3325 3326static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3327{ 3328 struct pci_dev *pdev = mgp->pdev; 3329 int vs = mgp->vendor_specific_offset; 3330 u32 reboot; 3331 3332 /*enter read32 mode */ 3333 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3334 3335 /*read REBOOT_STATUS (0xfffffff0) */ 3336 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3337 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3338 return reboot; 3339} 3340 3341static void 3342myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3343 int *busy_slice_cnt, u32 rx_pause_cnt) 3344{ 3345 struct myri10ge_priv *mgp = ss->mgp; 3346 int slice = ss - mgp->ss; 3347 3348 if (ss->tx.req != ss->tx.done && 3349 ss->tx.done == ss->watchdog_tx_done && 3350 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3351 /* nic seems like it might be stuck.. */ 3352 if (rx_pause_cnt != mgp->watchdog_pause) { 3353 if (net_ratelimit()) 3354 netdev_warn(mgp->dev, "slice %d: TX paused, " 3355 "check link partner\n", slice); 3356 } else { 3357 netdev_warn(mgp->dev, 3358 "slice %d: TX stuck %d %d %d %d %d %d\n", 3359 slice, ss->tx.queue_active, ss->tx.req, 3360 ss->tx.done, ss->tx.pkt_start, 3361 ss->tx.pkt_done, 3362 (int)ntohl(mgp->ss[slice].fw_stats-> 3363 send_done_count)); 3364 *reset_needed = 1; 3365 ss->stuck = 1; 3366 } 3367 } 3368 if (ss->watchdog_tx_done != ss->tx.done || 3369 ss->watchdog_rx_done != ss->rx_done.cnt) { 3370 *busy_slice_cnt += 1; 3371 } 3372 ss->watchdog_tx_done = ss->tx.done; 3373 ss->watchdog_tx_req = ss->tx.req; 3374 ss->watchdog_rx_done = ss->rx_done.cnt; 3375} 3376 3377/* 3378 * This watchdog is used to check whether the board has suffered 3379 * from a parity error and needs to be recovered. 3380 */ 3381static void myri10ge_watchdog(struct work_struct *work) 3382{ 3383 struct myri10ge_priv *mgp = 3384 container_of(work, struct myri10ge_priv, watchdog_work); 3385 struct myri10ge_slice_state *ss; 3386 u32 reboot, rx_pause_cnt; 3387 int status, rebooted; 3388 int i; 3389 int reset_needed = 0; 3390 int busy_slice_cnt = 0; 3391 u16 cmd, vendor; 3392 3393 mgp->watchdog_resets++; 3394 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3395 rebooted = 0; 3396 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3397 /* Bus master DMA disabled? Check to see 3398 * if the card rebooted due to a parity error 3399 * For now, just report it */ 3400 reboot = myri10ge_read_reboot(mgp); 3401 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3402 reboot, myri10ge_reset_recover ? "" : " not"); 3403 if (myri10ge_reset_recover == 0) 3404 return; 3405 rtnl_lock(); 3406 mgp->rebooted = 1; 3407 rebooted = 1; 3408 myri10ge_close(mgp->dev); 3409 myri10ge_reset_recover--; 3410 mgp->rebooted = 0; 3411 /* 3412 * A rebooted nic will come back with config space as 3413 * it was after power was applied to PCIe bus. 3414 * Attempt to restore config space which was saved 3415 * when the driver was loaded, or the last time the 3416 * nic was resumed from power saving mode. 3417 */ 3418 pci_restore_state(mgp->pdev); 3419 3420 /* save state again for accounting reasons */ 3421 pci_save_state(mgp->pdev); 3422 3423 } else { 3424 /* if we get back -1's from our slot, perhaps somebody 3425 * powered off our card. Don't try to reset it in 3426 * this case */ 3427 if (cmd == 0xffff) { 3428 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3429 if (vendor == 0xffff) { 3430 netdev_err(mgp->dev, "device disappeared!\n"); 3431 return; 3432 } 3433 } 3434 /* Perhaps it is a software error. See if stuck slice 3435 * has recovered, reset if not */ 3436 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3437 for (i = 0; i < mgp->num_slices; i++) { 3438 ss = mgp->ss; 3439 if (ss->stuck) { 3440 myri10ge_check_slice(ss, &reset_needed, 3441 &busy_slice_cnt, 3442 rx_pause_cnt); 3443 ss->stuck = 0; 3444 } 3445 } 3446 if (!reset_needed) { 3447 netdev_dbg(mgp->dev, "not resetting\n"); 3448 return; 3449 } 3450 3451 netdev_err(mgp->dev, "device timeout, resetting\n"); 3452 } 3453 3454 if (!rebooted) { 3455 rtnl_lock(); 3456 myri10ge_close(mgp->dev); 3457 } 3458 status = myri10ge_load_firmware(mgp, 1); 3459 if (status != 0) 3460 netdev_err(mgp->dev, "failed to load firmware\n"); 3461 else 3462 myri10ge_open(mgp->dev); 3463 rtnl_unlock(); 3464} 3465 3466/* 3467 * We use our own timer routine rather than relying upon 3468 * netdev->tx_timeout because we have a very large hardware transmit 3469 * queue. Due to the large queue, the netdev->tx_timeout function 3470 * cannot detect a NIC with a parity error in a timely fashion if the 3471 * NIC is lightly loaded. 3472 */ 3473static void myri10ge_watchdog_timer(struct timer_list *t) 3474{ 3475 struct myri10ge_priv *mgp; 3476 struct myri10ge_slice_state *ss; 3477 int i, reset_needed, busy_slice_cnt; 3478 u32 rx_pause_cnt; 3479 u16 cmd; 3480 3481 mgp = from_timer(mgp, t, watchdog_timer); 3482 3483 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3484 busy_slice_cnt = 0; 3485 for (i = 0, reset_needed = 0; 3486 i < mgp->num_slices && reset_needed == 0; ++i) { 3487 3488 ss = &mgp->ss[i]; 3489 if (ss->rx_small.watchdog_needed) { 3490 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3491 mgp->small_bytes + MXGEFW_PAD, 3492 1); 3493 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3494 myri10ge_fill_thresh) 3495 ss->rx_small.watchdog_needed = 0; 3496 } 3497 if (ss->rx_big.watchdog_needed) { 3498 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3499 mgp->big_bytes, 1); 3500 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3501 myri10ge_fill_thresh) 3502 ss->rx_big.watchdog_needed = 0; 3503 } 3504 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3505 rx_pause_cnt); 3506 } 3507 /* if we've sent or received no traffic, poll the NIC to 3508 * ensure it is still there. Otherwise, we risk not noticing 3509 * an error in a timely fashion */ 3510 if (busy_slice_cnt == 0) { 3511 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3512 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3513 reset_needed = 1; 3514 } 3515 } 3516 mgp->watchdog_pause = rx_pause_cnt; 3517 3518 if (reset_needed) { 3519 schedule_work(&mgp->watchdog_work); 3520 } else { 3521 /* rearm timer */ 3522 mod_timer(&mgp->watchdog_timer, 3523 jiffies + myri10ge_watchdog_timeout * HZ); 3524 } 3525} 3526 3527static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3528{ 3529 struct myri10ge_slice_state *ss; 3530 struct pci_dev *pdev = mgp->pdev; 3531 size_t bytes; 3532 int i; 3533 3534 if (mgp->ss == NULL) 3535 return; 3536 3537 for (i = 0; i < mgp->num_slices; i++) { 3538 ss = &mgp->ss[i]; 3539 if (ss->rx_done.entry != NULL) { 3540 bytes = mgp->max_intr_slots * 3541 sizeof(*ss->rx_done.entry); 3542 dma_free_coherent(&pdev->dev, bytes, 3543 ss->rx_done.entry, ss->rx_done.bus); 3544 ss->rx_done.entry = NULL; 3545 } 3546 if (ss->fw_stats != NULL) { 3547 bytes = sizeof(*ss->fw_stats); 3548 dma_free_coherent(&pdev->dev, bytes, 3549 ss->fw_stats, ss->fw_stats_bus); 3550 ss->fw_stats = NULL; 3551 } 3552 __netif_napi_del(&ss->napi); 3553 } 3554 /* Wait till napi structs are no longer used, and then free ss. */ 3555 synchronize_net(); 3556 kfree(mgp->ss); 3557 mgp->ss = NULL; 3558} 3559 3560static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3561{ 3562 struct myri10ge_slice_state *ss; 3563 struct pci_dev *pdev = mgp->pdev; 3564 size_t bytes; 3565 int i; 3566 3567 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3568 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3569 if (mgp->ss == NULL) { 3570 return -ENOMEM; 3571 } 3572 3573 for (i = 0; i < mgp->num_slices; i++) { 3574 ss = &mgp->ss[i]; 3575 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3576 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3577 &ss->rx_done.bus, 3578 GFP_KERNEL); 3579 if (ss->rx_done.entry == NULL) 3580 goto abort; 3581 bytes = sizeof(*ss->fw_stats); 3582 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3583 &ss->fw_stats_bus, 3584 GFP_KERNEL); 3585 if (ss->fw_stats == NULL) 3586 goto abort; 3587 ss->mgp = mgp; 3588 ss->dev = mgp->dev; 3589 netif_napi_add_weight(ss->dev, &ss->napi, myri10ge_poll, 3590 myri10ge_napi_weight); 3591 } 3592 return 0; 3593abort: 3594 myri10ge_free_slices(mgp); 3595 return -ENOMEM; 3596} 3597 3598/* 3599 * This function determines the number of slices supported. 3600 * The number slices is the minimum of the number of CPUS, 3601 * the number of MSI-X irqs supported, the number of slices 3602 * supported by the firmware 3603 */ 3604static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3605{ 3606 struct myri10ge_cmd cmd; 3607 struct pci_dev *pdev = mgp->pdev; 3608 char *old_fw; 3609 bool old_allocated; 3610 int i, status, ncpus; 3611 3612 mgp->num_slices = 1; 3613 ncpus = netif_get_num_default_rss_queues(); 3614 3615 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3616 (myri10ge_max_slices == -1 && ncpus < 2)) 3617 return; 3618 3619 /* try to load the slice aware rss firmware */ 3620 old_fw = mgp->fw_name; 3621 old_allocated = mgp->fw_name_allocated; 3622 /* don't free old_fw if we override it. */ 3623 mgp->fw_name_allocated = false; 3624 3625 if (myri10ge_fw_name != NULL) { 3626 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3627 myri10ge_fw_name); 3628 set_fw_name(mgp, myri10ge_fw_name, false); 3629 } else if (old_fw == myri10ge_fw_aligned) 3630 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3631 else 3632 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3633 status = myri10ge_load_firmware(mgp, 0); 3634 if (status != 0) { 3635 dev_info(&pdev->dev, "Rss firmware not found\n"); 3636 if (old_allocated) 3637 kfree(old_fw); 3638 return; 3639 } 3640 3641 /* hit the board with a reset to ensure it is alive */ 3642 memset(&cmd, 0, sizeof(cmd)); 3643 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3644 if (status != 0) { 3645 dev_err(&mgp->pdev->dev, "failed reset\n"); 3646 goto abort_with_fw; 3647 } 3648 3649 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3650 3651 /* tell it the size of the interrupt queues */ 3652 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3653 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3654 if (status != 0) { 3655 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3656 goto abort_with_fw; 3657 } 3658 3659 /* ask the maximum number of slices it supports */ 3660 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3661 if (status != 0) 3662 goto abort_with_fw; 3663 else 3664 mgp->num_slices = cmd.data0; 3665 3666 /* Only allow multiple slices if MSI-X is usable */ 3667 if (!myri10ge_msi) { 3668 goto abort_with_fw; 3669 } 3670 3671 /* if the admin did not specify a limit to how many 3672 * slices we should use, cap it automatically to the 3673 * number of CPUs currently online */ 3674 if (myri10ge_max_slices == -1) 3675 myri10ge_max_slices = ncpus; 3676 3677 if (mgp->num_slices > myri10ge_max_slices) 3678 mgp->num_slices = myri10ge_max_slices; 3679 3680 /* Now try to allocate as many MSI-X vectors as we have 3681 * slices. We give up on MSI-X if we can only get a single 3682 * vector. */ 3683 3684 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3685 GFP_KERNEL); 3686 if (mgp->msix_vectors == NULL) 3687 goto no_msix; 3688 for (i = 0; i < mgp->num_slices; i++) { 3689 mgp->msix_vectors[i].entry = i; 3690 } 3691 3692 while (mgp->num_slices > 1) { 3693 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3694 if (mgp->num_slices == 1) 3695 goto no_msix; 3696 status = pci_enable_msix_range(pdev, 3697 mgp->msix_vectors, 3698 mgp->num_slices, 3699 mgp->num_slices); 3700 if (status < 0) 3701 goto no_msix; 3702 3703 pci_disable_msix(pdev); 3704 3705 if (status == mgp->num_slices) { 3706 if (old_allocated) 3707 kfree(old_fw); 3708 return; 3709 } else { 3710 mgp->num_slices = status; 3711 } 3712 } 3713 3714no_msix: 3715 if (mgp->msix_vectors != NULL) { 3716 kfree(mgp->msix_vectors); 3717 mgp->msix_vectors = NULL; 3718 } 3719 3720abort_with_fw: 3721 mgp->num_slices = 1; 3722 set_fw_name(mgp, old_fw, old_allocated); 3723 myri10ge_load_firmware(mgp, 0); 3724} 3725 3726static const struct net_device_ops myri10ge_netdev_ops = { 3727 .ndo_open = myri10ge_open, 3728 .ndo_stop = myri10ge_close, 3729 .ndo_start_xmit = myri10ge_xmit, 3730 .ndo_get_stats64 = myri10ge_get_stats, 3731 .ndo_validate_addr = eth_validate_addr, 3732 .ndo_change_mtu = myri10ge_change_mtu, 3733 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3734 .ndo_set_mac_address = myri10ge_set_mac_address, 3735}; 3736 3737static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3738{ 3739 struct net_device *netdev; 3740 struct myri10ge_priv *mgp; 3741 struct device *dev = &pdev->dev; 3742 int status = -ENXIO; 3743 unsigned hdr_offset, ss_offset; 3744 static int board_number; 3745 3746 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3747 if (netdev == NULL) 3748 return -ENOMEM; 3749 3750 SET_NETDEV_DEV(netdev, &pdev->dev); 3751 3752 mgp = netdev_priv(netdev); 3753 mgp->dev = netdev; 3754 mgp->pdev = pdev; 3755 mgp->pause = myri10ge_flow_control; 3756 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3757 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3758 mgp->board_number = board_number; 3759 init_waitqueue_head(&mgp->down_wq); 3760 3761 if (pci_enable_device(pdev)) { 3762 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3763 status = -ENODEV; 3764 goto abort_with_netdev; 3765 } 3766 3767 /* Find the vendor-specific cap so we can check 3768 * the reboot register later on */ 3769 mgp->vendor_specific_offset 3770 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3771 3772 /* Set our max read request to 4KB */ 3773 status = pcie_set_readrq(pdev, 4096); 3774 if (status != 0) { 3775 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3776 status); 3777 goto abort_with_enabled; 3778 } 3779 3780 myri10ge_mask_surprise_down(pdev); 3781 pci_set_master(pdev); 3782 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3783 if (status != 0) { 3784 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3785 goto abort_with_enabled; 3786 } 3787 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3788 &mgp->cmd_bus, GFP_KERNEL); 3789 if (!mgp->cmd) { 3790 status = -ENOMEM; 3791 goto abort_with_enabled; 3792 } 3793 3794 mgp->board_span = pci_resource_len(pdev, 0); 3795 mgp->iomem_base = pci_resource_start(pdev, 0); 3796 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3797 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3798 if (mgp->sram == NULL) { 3799 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3800 mgp->board_span, mgp->iomem_base); 3801 status = -ENXIO; 3802 goto abort_with_mtrr; 3803 } 3804 hdr_offset = 3805 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3806 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3807 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3808 if (mgp->sram_size > mgp->board_span || 3809 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3810 dev_err(&pdev->dev, 3811 "invalid sram_size %dB or board span %ldB\n", 3812 mgp->sram_size, mgp->board_span); 3813 status = -EINVAL; 3814 goto abort_with_ioremap; 3815 } 3816 memcpy_fromio(mgp->eeprom_strings, 3817 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3818 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3819 status = myri10ge_read_mac_addr(mgp); 3820 if (status) 3821 goto abort_with_ioremap; 3822 3823 eth_hw_addr_set(netdev, mgp->mac_addr); 3824 3825 myri10ge_select_firmware(mgp); 3826 3827 status = myri10ge_load_firmware(mgp, 1); 3828 if (status != 0) { 3829 dev_err(&pdev->dev, "failed to load firmware\n"); 3830 goto abort_with_ioremap; 3831 } 3832 myri10ge_probe_slices(mgp); 3833 status = myri10ge_alloc_slices(mgp); 3834 if (status != 0) { 3835 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3836 goto abort_with_firmware; 3837 } 3838 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3839 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3840 status = myri10ge_reset(mgp); 3841 if (status != 0) { 3842 dev_err(&pdev->dev, "failed reset\n"); 3843 goto abort_with_slices; 3844 } 3845#ifdef CONFIG_MYRI10GE_DCA 3846 myri10ge_setup_dca(mgp); 3847#endif 3848 pci_set_drvdata(pdev, mgp); 3849 3850 /* MTU range: 68 - 9000 */ 3851 netdev->min_mtu = ETH_MIN_MTU; 3852 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3853 3854 if (myri10ge_initial_mtu > netdev->max_mtu) 3855 myri10ge_initial_mtu = netdev->max_mtu; 3856 if (myri10ge_initial_mtu < netdev->min_mtu) 3857 myri10ge_initial_mtu = netdev->min_mtu; 3858 3859 netdev->mtu = myri10ge_initial_mtu; 3860 3861 netdev->netdev_ops = &myri10ge_netdev_ops; 3862 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3863 3864 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3865 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3866 3867 netdev->features = netdev->hw_features | NETIF_F_HIGHDMA; 3868 3869 netdev->vlan_features |= mgp->features; 3870 if (mgp->fw_ver_tiny < 37) 3871 netdev->vlan_features &= ~NETIF_F_TSO6; 3872 if (mgp->fw_ver_tiny < 32) 3873 netdev->vlan_features &= ~NETIF_F_TSO; 3874 3875 /* make sure we can get an irq, and that MSI can be 3876 * setup (if available). */ 3877 status = myri10ge_request_irq(mgp); 3878 if (status != 0) 3879 goto abort_with_slices; 3880 myri10ge_free_irq(mgp); 3881 3882 /* Save configuration space to be restored if the 3883 * nic resets due to a parity error */ 3884 pci_save_state(pdev); 3885 3886 /* Setup the watchdog timer */ 3887 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3888 3889 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3890 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3891 status = register_netdev(netdev); 3892 if (status != 0) { 3893 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3894 goto abort_with_state; 3895 } 3896 if (mgp->msix_enabled) 3897 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3898 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3899 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3900 else 3901 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3902 mgp->msi_enabled ? "MSI" : "xPIC", 3903 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3904 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3905 3906 board_number++; 3907 return 0; 3908 3909abort_with_state: 3910 pci_restore_state(pdev); 3911 3912abort_with_slices: 3913 myri10ge_free_slices(mgp); 3914 3915abort_with_firmware: 3916 kfree(mgp->msix_vectors); 3917 myri10ge_dummy_rdma(mgp, 0); 3918 3919abort_with_ioremap: 3920 if (mgp->mac_addr_string != NULL) 3921 dev_err(&pdev->dev, 3922 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3923 mgp->mac_addr_string, mgp->serial_number); 3924 iounmap(mgp->sram); 3925 3926abort_with_mtrr: 3927 arch_phys_wc_del(mgp->wc_cookie); 3928 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3929 mgp->cmd, mgp->cmd_bus); 3930 3931abort_with_enabled: 3932 pci_disable_device(pdev); 3933 3934abort_with_netdev: 3935 set_fw_name(mgp, NULL, false); 3936 free_netdev(netdev); 3937 return status; 3938} 3939 3940/* 3941 * myri10ge_remove 3942 * 3943 * Does what is necessary to shutdown one Myrinet device. Called 3944 * once for each Myrinet card by the kernel when a module is 3945 * unloaded. 3946 */ 3947static void myri10ge_remove(struct pci_dev *pdev) 3948{ 3949 struct myri10ge_priv *mgp; 3950 struct net_device *netdev; 3951 3952 mgp = pci_get_drvdata(pdev); 3953 if (mgp == NULL) 3954 return; 3955 3956 cancel_work_sync(&mgp->watchdog_work); 3957 netdev = mgp->dev; 3958 unregister_netdev(netdev); 3959 3960#ifdef CONFIG_MYRI10GE_DCA 3961 myri10ge_teardown_dca(mgp); 3962#endif 3963 myri10ge_dummy_rdma(mgp, 0); 3964 3965 /* avoid a memory leak */ 3966 pci_restore_state(pdev); 3967 3968 iounmap(mgp->sram); 3969 arch_phys_wc_del(mgp->wc_cookie); 3970 myri10ge_free_slices(mgp); 3971 kfree(mgp->msix_vectors); 3972 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3973 mgp->cmd, mgp->cmd_bus); 3974 3975 set_fw_name(mgp, NULL, false); 3976 free_netdev(netdev); 3977 pci_disable_device(pdev); 3978} 3979 3980#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 3981#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 3982 3983static const struct pci_device_id myri10ge_pci_tbl[] = { 3984 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 3985 {PCI_DEVICE 3986 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 3987 {0}, 3988}; 3989 3990MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 3991 3992static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 3993 3994static struct pci_driver myri10ge_driver = { 3995 .name = "myri10ge", 3996 .probe = myri10ge_probe, 3997 .remove = myri10ge_remove, 3998 .id_table = myri10ge_pci_tbl, 3999 .driver.pm = &myri10ge_pm_ops, 4000}; 4001 4002#ifdef CONFIG_MYRI10GE_DCA 4003static int 4004myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4005{ 4006 int err = driver_for_each_device(&myri10ge_driver.driver, 4007 NULL, &event, 4008 myri10ge_notify_dca_device); 4009 4010 if (err) 4011 return NOTIFY_BAD; 4012 return NOTIFY_DONE; 4013} 4014 4015static struct notifier_block myri10ge_dca_notifier = { 4016 .notifier_call = myri10ge_notify_dca, 4017 .next = NULL, 4018 .priority = 0, 4019}; 4020#endif /* CONFIG_MYRI10GE_DCA */ 4021 4022static __init int myri10ge_init_module(void) 4023{ 4024 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4025 4026 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4027 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4028 myri10ge_rss_hash); 4029 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4030 } 4031#ifdef CONFIG_MYRI10GE_DCA 4032 dca_register_notify(&myri10ge_dca_notifier); 4033#endif 4034 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4035 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4036 4037 return pci_register_driver(&myri10ge_driver); 4038} 4039 4040module_init(myri10ge_init_module); 4041 4042static __exit void myri10ge_cleanup_module(void) 4043{ 4044#ifdef CONFIG_MYRI10GE_DCA 4045 dca_unregister_notify(&myri10ge_dca_notifier); 4046#endif 4047 pci_unregister_driver(&myri10ge_driver); 4048} 4049 4050module_exit(myri10ge_cleanup_module); 4051