1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29/*$FreeBSD$*/ 30#include "opt_ddb.h" 31#include "opt_inet.h" 32#include "opt_inet6.h" 33#include "opt_rss.h" 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#endif 38 39#include <sys/param.h> 40#include <sys/systm.h> 41#ifdef DDB 42#include <sys/types.h> 43#include <ddb/ddb.h> 44#endif 45#if __FreeBSD_version >= 800000 46#include <sys/buf_ring.h> 47#endif 48#include <sys/bus.h> 49#include <sys/endian.h> 50#include <sys/kernel.h> 51#include <sys/kthread.h> 52#include <sys/malloc.h> 53#include <sys/mbuf.h> 54#include <sys/module.h> 55#include <sys/rman.h> 56#include <sys/smp.h> 57#include <sys/socket.h> 58#include <sys/sockio.h> 59#include <sys/sysctl.h> 60#include <sys/taskqueue.h> 61#include <sys/eventhandler.h> 62#include <machine/bus.h> 63#include <machine/resource.h> 64 65#include <net/bpf.h> 66#include <net/ethernet.h> 67#include <net/if.h> 68#include <net/if_var.h> 69#include <net/if_arp.h> 70#include <net/if_dl.h> 71#include <net/if_media.h> 72#include <net/iflib.h> 73#ifdef RSS 74#include <net/rss_config.h> 75#include <netinet/in_rss.h> 76#endif 77 78#include <net/if_types.h> 79#include <net/if_vlan_var.h> 80 81#include <netinet/in_systm.h> 82#include <netinet/in.h> 83#include <netinet/if_ether.h> 84#include <netinet/ip.h> 85#include <netinet/ip6.h> 86#include <netinet/tcp.h> 87#include <netinet/udp.h> 88 89#include <machine/in_cksum.h> 90#include <dev/led/led.h> 91#include <dev/pci/pcivar.h> 92#include <dev/pci/pcireg.h> 93 94#include "e1000_api.h" 95#include "e1000_82571.h" 96#include "ifdi_if.h" 97 98 99#ifndef _EM_H_DEFINED_ 100#define _EM_H_DEFINED_ 101 102 103/* Tunables */ 104 105/* 106 * EM_MAX_TXD: Maximum number of Transmit Descriptors 107 * Valid Range: 80-256 for 82542 and 82543-based adapters 108 * 80-4096 for others 109 * Default Value: 1024 110 * This value is the number of transmit descriptors allocated by the driver. 111 * Increasing this value allows the driver to queue more transmits. Each 112 * descriptor is 16 bytes. 113 * Since TDLEN should be multiple of 128bytes, the number of transmit 114 * desscriptors should meet the following condition. 115 * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 116 */ 117#define EM_MIN_TXD 128 118#define EM_MAX_TXD 4096 119#define EM_DEFAULT_TXD 1024 120#define EM_DEFAULT_MULTI_TXD 4096 121#define IGB_MAX_TXD 4096 122 123/* 124 * EM_MAX_RXD - Maximum number of receive Descriptors 125 * Valid Range: 80-256 for 82542 and 82543-based adapters 126 * 80-4096 for others 127 * Default Value: 1024 128 * This value is the number of receive descriptors allocated by the driver. 129 * Increasing this value allows the driver to buffer more incoming packets. 130 * Each descriptor is 16 bytes. A receive buffer is also allocated for each 131 * descriptor. The maximum MTU size is 16110. 132 * Since TDLEN should be multiple of 128bytes, the number of transmit 133 * desscriptors should meet the following condition. 134 * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 135 */ 136#define EM_MIN_RXD 128 137#define EM_MAX_RXD 4096 138#define EM_DEFAULT_RXD 1024 139#define EM_DEFAULT_MULTI_RXD 4096 140#define IGB_MAX_RXD 4096 141 142/* 143 * EM_TIDV - Transmit Interrupt Delay Value 144 * Valid Range: 0-65535 (0=off) 145 * Default Value: 64 146 * This value delays the generation of transmit interrupts in units of 147 * 1.024 microseconds. Transmit interrupt reduction can improve CPU 148 * efficiency if properly tuned for specific network traffic. If the 149 * system is reporting dropped transmits, this value may be set too high 150 * causing the driver to run out of available transmit descriptors. 151 */ 152#define EM_TIDV 64 153 154/* 155 * EM_TADV - Transmit Absolute Interrupt Delay Value 156 * (Not valid for 82542/82543/82544) 157 * Valid Range: 0-65535 (0=off) 158 * Default Value: 64 159 * This value, in units of 1.024 microseconds, limits the delay in which a 160 * transmit interrupt is generated. Useful only if EM_TIDV is non-zero, 161 * this value ensures that an interrupt is generated after the initial 162 * packet is sent on the wire within the set amount of time. Proper tuning, 163 * along with EM_TIDV, may improve traffic throughput in specific 164 * network conditions. 165 */ 166#define EM_TADV 64 167 168/* 169 * EM_RDTR - Receive Interrupt Delay Timer (Packet Timer) 170 * Valid Range: 0-65535 (0=off) 171 * Default Value: 0 172 * This value delays the generation of receive interrupts in units of 1.024 173 * microseconds. Receive interrupt reduction can improve CPU efficiency if 174 * properly tuned for specific network traffic. Increasing this value adds 175 * extra latency to frame reception and can end up decreasing the throughput 176 * of TCP traffic. If the system is reporting dropped receives, this value 177 * may be set too high, causing the driver to run out of available receive 178 * descriptors. 179 * 180 * CAUTION: When setting EM_RDTR to a value other than 0, adapters 181 * may hang (stop transmitting) under certain network conditions. 182 * If this occurs a WATCHDOG message is logged in the system 183 * event log. In addition, the controller is automatically reset, 184 * restoring the network connection. To eliminate the potential 185 * for the hang ensure that EM_RDTR is set to 0. 186 */ 187#define EM_RDTR 0 188 189/* 190 * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) 191 * Valid Range: 0-65535 (0=off) 192 * Default Value: 64 193 * This value, in units of 1.024 microseconds, limits the delay in which a 194 * receive interrupt is generated. Useful only if EM_RDTR is non-zero, 195 * this value ensures that an interrupt is generated after the initial 196 * packet is received within the set amount of time. Proper tuning, 197 * along with EM_RDTR, may improve traffic throughput in specific network 198 * conditions. 199 */ 200#define EM_RADV 64 201 202/* 203 * This parameter controls whether or not autonegotation is enabled. 204 * 0 - Disable autonegotiation 205 * 1 - Enable autonegotiation 206 */ 207#define DO_AUTO_NEG 1 208 209/* 210 * This parameter control whether or not the driver will wait for 211 * autonegotiation to complete. 212 * 1 - Wait for autonegotiation to complete 213 * 0 - Don't wait for autonegotiation to complete 214 */ 215#define WAIT_FOR_AUTO_NEG_DEFAULT 0 216 217/* Tunables -- End */ 218 219#define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ 220 ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ 221 ADVERTISE_1000_FULL) 222 223#define AUTO_ALL_MODES 0 224 225/* PHY master/slave setting */ 226#define EM_MASTER_SLAVE e1000_ms_hw_default 227 228/* 229 * Micellaneous constants 230 */ 231#define EM_VENDOR_ID 0x8086 232#define EM_FLASH 0x0014 233 234#define EM_JUMBO_PBA 0x00000028 235#define EM_DEFAULT_PBA 0x00000030 236#define EM_SMARTSPEED_DOWNSHIFT 3 237#define EM_SMARTSPEED_MAX 15 238#define EM_MAX_LOOP 10 239 240#define MAX_NUM_MULTICAST_ADDRESSES 128 241#define PCI_ANY_ID (~0U) 242#define ETHER_ALIGN 2 243#define EM_FC_PAUSE_TIME 0x0680 244#define EM_EEPROM_APME 0x400; 245#define EM_82544_APME 0x0004; 246 247 248/* Support AutoMediaDetect for Marvell M88 PHY in i354 */ 249#define IGB_MEDIA_RESET (1 << 0) 250 251/* Define the starting Interrupt rate per Queue */ 252#define IGB_INTS_PER_SEC 8000 253#define IGB_DEFAULT_ITR ((1000000/IGB_INTS_PER_SEC) << 2) 254 255#define IGB_LINK_ITR 2000 256#define I210_LINK_DELAY 1000 257 258#define IGB_TXPBSIZE 20408 259#define IGB_HDR_BUF 128 260#define IGB_PKTTYPE_MASK 0x0000FFF0 261#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ 262 263/* 264 * Driver state logic for the detection of a hung state 265 * in hardware. Set TX_HUNG whenever a TX packet is used 266 * (data is sent) and clear it when txeof() is invoked if 267 * any descriptors from the ring are cleaned/reclaimed. 268 * Increment internal counter if no descriptors are cleaned 269 * and compare to TX_MAXTRIES. When counter > TX_MAXTRIES, 270 * reset adapter. 271 */ 272#define EM_TX_IDLE 0x00000000 273#define EM_TX_BUSY 0x00000001 274#define EM_TX_HUNG 0x80000000 275#define EM_TX_MAXTRIES 10 276 277#define PCICFG_DESC_RING_STATUS 0xe4 278#define FLUSH_DESC_REQUIRED 0x100 279 280 281#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ 282 ((hw->mac.type <= e1000_82576) ? 16 : 8)) 283#define IGB_RX_HTHRESH 8 284#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ 285 (adapter->intr_type == IFLIB_INTR_MSIX)) ? 1 : 4) 286 287#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 288#define IGB_TX_HTHRESH 1 289#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ 290 (adapter->intr_type == IFLIB_INTR_MSIX) ? 1 : 16) 291 292/* 293 * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be 294 * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will 295 * also optimize cache line size effect. H/W supports up to cache line size 128. 296 */ 297#define EM_DBA_ALIGN 128 298 299/* 300 * See Intel 82574 Driver Programming Interface Manual, Section 10.2.6.9 301 */ 302#define TARC_COMPENSATION_MODE (1 << 7) /* Compensation Mode */ 303#define TARC_SPEED_MODE_BIT (1 << 21) /* On PCI-E MACs only */ 304#define TARC_MQ_FIX (1 << 23) | \ 305 (1 << 24) | \ 306 (1 << 25) /* Handle errata in MQ mode */ 307#define TARC_ERRATA_BIT (1 << 26) /* Note from errata on 82574 */ 308 309/* PCI Config defines */ 310#define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) 311#define EM_BAR_TYPE_MASK 0x00000001 312#define EM_BAR_TYPE_MMEM 0x00000000 313#define EM_BAR_TYPE_IO 0x00000001 314#define EM_BAR_TYPE_FLASH 0x0014 315#define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK) 316#define EM_BAR_MEM_TYPE_MASK 0x00000006 317#define EM_BAR_MEM_TYPE_32BIT 0x00000000 318#define EM_BAR_MEM_TYPE_64BIT 0x00000004 319#define EM_MSIX_BAR 3 /* On 82575 */ 320 321/* More backward compatibility */ 322#if __FreeBSD_version < 900000 323#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 324#endif 325 326/* Defines for printing debug information */ 327#define DEBUG_INIT 0 328#define DEBUG_IOCTL 0 329#define DEBUG_HW 0 330 331#define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") 332#define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) 333#define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) 334#define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") 335#define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) 336#define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) 337#define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") 338#define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) 339#define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) 340 341#define EM_MAX_SCATTER 40 342#define EM_VFTA_SIZE 128 343#define EM_TSO_SIZE 65535 344#define EM_TSO_SEG_SIZE 4096 /* Max dma segment size */ 345#define ETH_ZLEN 60 346#define ETH_ADDR_LEN 6 347#define EM_CSUM_OFFLOAD (CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP) /* Offload bits in mbuf flag */ 348#define IGB_CSUM_OFFLOAD (CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP | \ 349 CSUM_IP_SCTP | CSUM_IP6_UDP | CSUM_IP6_TCP | \ 350 CSUM_IP6_SCTP) /* Offload bits in mbuf flag */ 351 352 353#define IGB_PKTTYPE_MASK 0x0000FFF0 354#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ 355 356/* 357 * 82574 has a nonstandard address for EIAC 358 * and since its only used in MSI-X, and in 359 * the em driver only 82574 uses MSI-X we can 360 * solve it just using this define. 361 */ 362#define EM_EIAC 0x000DC 363/* 364 * 82574 only reports 3 MSI-X vectors by default; 365 * defines assisting with making it report 5 are 366 * located here. 367 */ 368#define EM_NVM_PCIE_CTRL 0x1B 369#define EM_NVM_MSIX_N_MASK (0x7 << EM_NVM_MSIX_N_SHIFT) 370#define EM_NVM_MSIX_N_SHIFT 7 371 372struct adapter; 373 374struct em_int_delay_info { 375 struct adapter *adapter; /* Back-pointer to the adapter struct */ 376 int offset; /* Register offset to read/write */ 377 int value; /* Current value in usecs */ 378}; 379 380/* 381 * The transmit ring, one per tx queue 382 */ 383struct tx_ring { 384 struct adapter *adapter; 385 struct e1000_tx_desc *tx_base; 386 uint64_t tx_paddr; 387 qidx_t *tx_rsq; 388 bool tx_tso; /* last tx was tso */ 389 uint8_t me; 390 qidx_t tx_rs_cidx; 391 qidx_t tx_rs_pidx; 392 qidx_t tx_cidx_processed; 393 /* Interrupt resources */ 394 void *tag; 395 struct resource *res; 396 unsigned long tx_irq; 397 398 /* Saved csum offloading context information */ 399 int csum_flags; 400 int csum_lhlen; 401 int csum_iphlen; 402 403 int csum_thlen; 404 int csum_mss; 405 int csum_pktlen; 406 407 uint32_t csum_txd_upper; 408 uint32_t csum_txd_lower; /* last field */ 409}; 410 411/* 412 * The Receive ring, one per rx queue 413 */ 414struct rx_ring { 415 struct adapter *adapter; 416 struct em_rx_queue *que; 417 u32 me; 418 u32 payload; 419 union e1000_rx_desc_extended *rx_base; 420 uint64_t rx_paddr; 421 422 /* Interrupt resources */ 423 void *tag; 424 struct resource *res; 425 bool discard; 426 427 /* Soft stats */ 428 unsigned long rx_irq; 429 unsigned long rx_discarded; 430 unsigned long rx_packets; 431 unsigned long rx_bytes; 432}; 433 434struct em_tx_queue { 435 struct adapter *adapter; 436 u32 msix; 437 u32 eims; /* This queue's EIMS bit */ 438 u32 me; 439 struct tx_ring txr; 440}; 441 442struct em_rx_queue { 443 struct adapter *adapter; 444 u32 me; 445 u32 msix; 446 u32 eims; 447 struct rx_ring rxr; 448 u64 irqs; 449 struct if_irq que_irq; 450}; 451 452/* Our adapter structure */ 453struct adapter { 454 struct ifnet *ifp; 455 struct e1000_hw hw; 456 457 if_softc_ctx_t shared; 458 if_ctx_t ctx; 459#define tx_num_queues shared->isc_ntxqsets 460#define rx_num_queues shared->isc_nrxqsets 461#define intr_type shared->isc_intr 462 /* FreeBSD operating-system-specific structures. */ 463 struct e1000_osdep osdep; 464 device_t dev; 465 struct cdev *led_dev; 466 467 struct em_tx_queue *tx_queues; 468 struct em_rx_queue *rx_queues; 469 struct if_irq irq; 470 471 struct resource *memory; 472 struct resource *flash; 473 struct resource *ioport; 474 475 struct resource *res; 476 void *tag; 477 u32 linkvec; 478 u32 ivars; 479 480 struct ifmedia *media; 481 int msix; 482 int if_flags; 483 int em_insert_vlan_header; 484 u32 ims; 485 bool in_detach; 486 487 u32 flags; 488 /* Task for FAST handling */ 489 struct grouptask link_task; 490 491 u16 num_vlans; 492 u32 txd_cmd; 493 494 u32 tx_process_limit; 495 u32 rx_process_limit; 496 u32 rx_mbuf_sz; 497 498 /* Management and WOL features */ 499 u32 wol; 500 bool has_manage; 501 bool has_amt; 502 503 /* Multicast array memory */ 504 u8 *mta; 505 506 /* 507 ** Shadow VFTA table, this is needed because 508 ** the real vlan filter table gets cleared during 509 ** a soft reset and the driver needs to be able 510 ** to repopulate it. 511 */ 512 u32 shadow_vfta[EM_VFTA_SIZE]; 513 514 /* Info about the interface */ 515 u16 link_active; 516 u16 fc; 517 u16 link_speed; 518 u16 link_duplex; 519 u32 smartspeed; 520 u32 dmac; 521 int link_mask; 522 523 u64 que_mask; 524 525 struct em_int_delay_info tx_int_delay; 526 struct em_int_delay_info tx_abs_int_delay; 527 struct em_int_delay_info rx_int_delay; 528 struct em_int_delay_info rx_abs_int_delay; 529 struct em_int_delay_info tx_itr; 530 531 /* Misc stats maintained by the driver */ 532 unsigned long dropped_pkts; 533 unsigned long link_irq; 534 unsigned long rx_overruns; 535 unsigned long watchdog_events; 536 537 struct e1000_hw_stats stats; 538 u16 vf_ifp; 539}; 540 541/******************************************************************************** 542 * vendor_info_array 543 * 544 * This array contains the list of Subvendor/Subdevice IDs on which the driver 545 * should load. 546 * 547 ********************************************************************************/ 548typedef struct _em_vendor_info_t { 549 unsigned int vendor_id; 550 unsigned int device_id; 551 unsigned int subvendor_id; 552 unsigned int subdevice_id; 553 unsigned int index; 554} em_vendor_info_t; 555 556void em_dump_rs(struct adapter *); 557 558#define EM_RSSRK_SIZE 4 559#define EM_RSSRK_VAL(key, i) (key[(i) * EM_RSSRK_SIZE] | \ 560 key[(i) * EM_RSSRK_SIZE + 1] << 8 | \ 561 key[(i) * EM_RSSRK_SIZE + 2] << 16 | \ 562 key[(i) * EM_RSSRK_SIZE + 3] << 24) 563#endif /* _EM_H_DEFINED_ */ 564