1/****************************************************************************** 2 3 Copyright (c) 2001-2008, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/
| 1/****************************************************************************** 2 3 Copyright (c) 2001-2008, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/
|
33/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 181003 2008-07-30 18:15:18Z jfv $*/
| 33/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 185352 2008-11-26 23:41:18Z jfv $*/
|
34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#endif 38
| 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#endif 38
|
39/* Undefine this if not using CURRENT */ 40#define IXGBE_VLAN_EVENTS 41
| |
42#include "ixgbe.h" 43 44/********************************************************************* 45 * Set this to one to display debug statistics 46 *********************************************************************/ 47int ixgbe_display_debug_stats = 0; 48 49/********************************************************************* 50 * Driver version 51 *********************************************************************/
| 39#include "ixgbe.h" 40 41/********************************************************************* 42 * Set this to one to display debug statistics 43 *********************************************************************/ 44int ixgbe_display_debug_stats = 0; 45 46/********************************************************************* 47 * Driver version 48 *********************************************************************/
|
52char ixgbe_driver_version[] = "1.4.7";
| 49char ixgbe_driver_version[] = "1.6.2";
|
53 54/********************************************************************* 55 * PCI Device ID Table 56 * 57 * Used by probe to select devices to load on 58 * Last field stores an index into ixgbe_strings 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 62 *********************************************************************/ 63 64static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 65{ 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
| 50 51/********************************************************************* 52 * PCI Device ID Table 53 * 54 * Used by probe to select devices to load on 55 * Last field stores an index into ixgbe_strings 56 * Last entry must be all 0s 57 * 58 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 59 *********************************************************************/ 60 61static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 62{ 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
|
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT_DUAL_PORT, 0, 0, 0},
| |
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
| 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
|
| 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
|
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
| 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
|
| 72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
|
73 /* required last entry */ 74 {0, 0, 0, 0, 0} 75}; 76 77/********************************************************************* 78 * Table of branding strings 79 *********************************************************************/ 80 81static char *ixgbe_strings[] = { 82 "Intel(R) PRO/10GbE PCI-Express Network Driver" 83}; 84 85/********************************************************************* 86 * Function prototypes 87 *********************************************************************/ 88static int ixgbe_probe(device_t); 89static int ixgbe_attach(device_t); 90static int ixgbe_detach(device_t); 91static int ixgbe_shutdown(device_t); 92static void ixgbe_start(struct ifnet *); 93static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 94static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 95static void ixgbe_watchdog(struct adapter *); 96static void ixgbe_init(void *); 97static void ixgbe_init_locked(struct adapter *); 98static void ixgbe_stop(void *); 99static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 100static int ixgbe_media_change(struct ifnet *); 101static void ixgbe_identify_hardware(struct adapter *); 102static int ixgbe_allocate_pci_resources(struct adapter *); 103static int ixgbe_allocate_msix(struct adapter *); 104static int ixgbe_allocate_legacy(struct adapter *); 105static int ixgbe_allocate_queues(struct adapter *); 106static int ixgbe_setup_msix(struct adapter *); 107static void ixgbe_free_pci_resources(struct adapter *); 108static void ixgbe_local_timer(void *); 109static int ixgbe_hardware_init(struct adapter *); 110static void ixgbe_setup_interface(device_t, struct adapter *); 111 112static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 113static int ixgbe_setup_transmit_structures(struct adapter *); 114static void ixgbe_setup_transmit_ring(struct tx_ring *); 115static void ixgbe_initialize_transmit_units(struct adapter *); 116static void ixgbe_free_transmit_structures(struct adapter *); 117static void ixgbe_free_transmit_buffers(struct tx_ring *); 118 119static int ixgbe_allocate_receive_buffers(struct rx_ring *); 120static int ixgbe_setup_receive_structures(struct adapter *); 121static int ixgbe_setup_receive_ring(struct rx_ring *); 122static void ixgbe_initialize_receive_units(struct adapter *); 123static void ixgbe_free_receive_structures(struct adapter *); 124static void ixgbe_free_receive_buffers(struct rx_ring *); 125 126static void ixgbe_enable_intr(struct adapter *); 127static void ixgbe_disable_intr(struct adapter *); 128static void ixgbe_update_stats_counters(struct adapter *); 129static bool ixgbe_txeof(struct tx_ring *); 130static bool ixgbe_rxeof(struct rx_ring *, int);
| 74 /* required last entry */ 75 {0, 0, 0, 0, 0} 76}; 77 78/********************************************************************* 79 * Table of branding strings 80 *********************************************************************/ 81 82static char *ixgbe_strings[] = { 83 "Intel(R) PRO/10GbE PCI-Express Network Driver" 84}; 85 86/********************************************************************* 87 * Function prototypes 88 *********************************************************************/ 89static int ixgbe_probe(device_t); 90static int ixgbe_attach(device_t); 91static int ixgbe_detach(device_t); 92static int ixgbe_shutdown(device_t); 93static void ixgbe_start(struct ifnet *); 94static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 95static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 96static void ixgbe_watchdog(struct adapter *); 97static void ixgbe_init(void *); 98static void ixgbe_init_locked(struct adapter *); 99static void ixgbe_stop(void *); 100static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 101static int ixgbe_media_change(struct ifnet *); 102static void ixgbe_identify_hardware(struct adapter *); 103static int ixgbe_allocate_pci_resources(struct adapter *); 104static int ixgbe_allocate_msix(struct adapter *); 105static int ixgbe_allocate_legacy(struct adapter *); 106static int ixgbe_allocate_queues(struct adapter *); 107static int ixgbe_setup_msix(struct adapter *); 108static void ixgbe_free_pci_resources(struct adapter *); 109static void ixgbe_local_timer(void *); 110static int ixgbe_hardware_init(struct adapter *); 111static void ixgbe_setup_interface(device_t, struct adapter *); 112 113static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 114static int ixgbe_setup_transmit_structures(struct adapter *); 115static void ixgbe_setup_transmit_ring(struct tx_ring *); 116static void ixgbe_initialize_transmit_units(struct adapter *); 117static void ixgbe_free_transmit_structures(struct adapter *); 118static void ixgbe_free_transmit_buffers(struct tx_ring *); 119 120static int ixgbe_allocate_receive_buffers(struct rx_ring *); 121static int ixgbe_setup_receive_structures(struct adapter *); 122static int ixgbe_setup_receive_ring(struct rx_ring *); 123static void ixgbe_initialize_receive_units(struct adapter *); 124static void ixgbe_free_receive_structures(struct adapter *); 125static void ixgbe_free_receive_buffers(struct rx_ring *); 126 127static void ixgbe_enable_intr(struct adapter *); 128static void ixgbe_disable_intr(struct adapter *); 129static void ixgbe_update_stats_counters(struct adapter *); 130static bool ixgbe_txeof(struct tx_ring *); 131static bool ixgbe_rxeof(struct rx_ring *, int);
|
131static void ixgbe_rx_checksum(struct adapter *, u32, struct mbuf *);
| 132static void ixgbe_rx_checksum(u32, struct mbuf *);
|
132static void ixgbe_set_promisc(struct adapter *); 133static void ixgbe_disable_promisc(struct adapter *); 134static void ixgbe_set_multi(struct adapter *); 135static void ixgbe_print_hw_stats(struct adapter *); 136static void ixgbe_print_debug_info(struct adapter *); 137static void ixgbe_update_link_status(struct adapter *);
| 133static void ixgbe_set_promisc(struct adapter *); 134static void ixgbe_disable_promisc(struct adapter *); 135static void ixgbe_set_multi(struct adapter *); 136static void ixgbe_print_hw_stats(struct adapter *); 137static void ixgbe_print_debug_info(struct adapter *); 138static void ixgbe_update_link_status(struct adapter *);
|
138static int ixgbe_get_buf(struct rx_ring *, int);
| 139static int ixgbe_get_buf(struct rx_ring *, int, u8);
|
139static int ixgbe_xmit(struct tx_ring *, struct mbuf **); 140static int ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS); 141static int ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS); 142static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS); 143static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 144 struct ixgbe_dma_alloc *, int); 145static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 146static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 147 const char *, int *, int); 148static boolean_t ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 149static boolean_t ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
| 140static int ixgbe_xmit(struct tx_ring *, struct mbuf **); 141static int ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS); 142static int ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS); 143static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS); 144static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 145 struct ixgbe_dma_alloc *, int); 146static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 147static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 148 const char *, int *, int); 149static boolean_t ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 150static boolean_t ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
|
150static void ixgbe_set_ivar(struct adapter *, u16, u8);
| 151static void ixgbe_set_ivar(struct adapter *, u16, u8, s8);
|
151static void ixgbe_configure_ivars(struct adapter *); 152static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 153
| 152static void ixgbe_configure_ivars(struct adapter *); 153static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 154
|
154#ifdef IXGBE_VLAN_EVENTS
| 155#ifdef IXGBE_HW_VLAN_SUPPORT
|
155static void ixgbe_register_vlan(void *, struct ifnet *, u16); 156static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 157#endif 158
| 156static void ixgbe_register_vlan(void *, struct ifnet *, u16); 157static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 158#endif 159
|
| 160static void ixgbe_update_aim(struct rx_ring *); 161 162/* Support for pluggable optic modules */ 163static bool ixgbe_sfp_probe(struct adapter *); 164
|
159/* Legacy (single vector interrupt handler */ 160static void ixgbe_legacy_irq(void *); 161 162/* The MSI/X Interrupt handlers */ 163static void ixgbe_msix_tx(void *); 164static void ixgbe_msix_rx(void *); 165static void ixgbe_msix_link(void *); 166 167/* Legacy interrupts use deferred handlers */ 168static void ixgbe_handle_tx(void *context, int pending); 169static void ixgbe_handle_rx(void *context, int pending); 170
| 165/* Legacy (single vector interrupt handler */ 166static void ixgbe_legacy_irq(void *); 167 168/* The MSI/X Interrupt handlers */ 169static void ixgbe_msix_tx(void *); 170static void ixgbe_msix_rx(void *); 171static void ixgbe_msix_link(void *); 172 173/* Legacy interrupts use deferred handlers */ 174static void ixgbe_handle_tx(void *context, int pending); 175static void ixgbe_handle_rx(void *context, int pending); 176
|
171#ifndef NO_82598_A0_SUPPORT 172static void desc_flip(void *); 173#endif
| |
174 175/********************************************************************* 176 * FreeBSD Device Interface Entry Points 177 *********************************************************************/ 178 179static device_method_t ixgbe_methods[] = { 180 /* Device interface */ 181 DEVMETHOD(device_probe, ixgbe_probe), 182 DEVMETHOD(device_attach, ixgbe_attach), 183 DEVMETHOD(device_detach, ixgbe_detach), 184 DEVMETHOD(device_shutdown, ixgbe_shutdown), 185 {0, 0} 186}; 187 188static driver_t ixgbe_driver = { 189 "ix", ixgbe_methods, sizeof(struct adapter), 190}; 191 192static devclass_t ixgbe_devclass; 193DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 194 195MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 196MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 197 198/* 199** TUNEABLE PARAMETERS: 200*/ 201
| 177 178/********************************************************************* 179 * FreeBSD Device Interface Entry Points 180 *********************************************************************/ 181 182static device_method_t ixgbe_methods[] = { 183 /* Device interface */ 184 DEVMETHOD(device_probe, ixgbe_probe), 185 DEVMETHOD(device_attach, ixgbe_attach), 186 DEVMETHOD(device_detach, ixgbe_detach), 187 DEVMETHOD(device_shutdown, ixgbe_shutdown), 188 {0, 0} 189}; 190 191static driver_t ixgbe_driver = { 192 "ix", ixgbe_methods, sizeof(struct adapter), 193}; 194 195static devclass_t ixgbe_devclass; 196DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 197 198MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 199MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 200 201/* 202** TUNEABLE PARAMETERS: 203*/ 204
|
| 205/* 206** These parameters are used in Adaptive 207** Interrupt Moderation. The value is set 208** into EITR and controls the interrupt 209** frequency. They can be modified but 210** be careful in tuning them. 211*/ 212static int ixgbe_enable_aim = TRUE; 213TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 214static int ixgbe_low_latency = IXGBE_LOW_LATENCY; 215TUNABLE_INT("hw.ixgbe.low_latency", &ixgbe_low_latency); 216static int ixgbe_ave_latency = IXGBE_LOW_LATENCY; 217TUNABLE_INT("hw.ixgbe.ave_latency", &ixgbe_low_latency); 218static int ixgbe_bulk_latency = IXGBE_BULK_LATENCY; 219TUNABLE_INT("hw.ixgbe.bulk_latency", &ixgbe_bulk_latency); 220
|
202/* How many packets rxeof tries to clean at a time */ 203static int ixgbe_rx_process_limit = 100; 204TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 205 206/* Flow control setting, default to full */
| 221/* How many packets rxeof tries to clean at a time */ 222static int ixgbe_rx_process_limit = 100; 223TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 224 225/* Flow control setting, default to full */
|
207static int ixgbe_flow_control = 3;
| 226static int ixgbe_flow_control = ixgbe_fc_none;
|
208TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 209 210/* 211 * Should the driver do LRO on the RX end 212 * this can be toggled on the fly, but the 213 * interface must be reset (down/up) for it 214 * to take effect. 215 */
| 227TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 228 229/* 230 * Should the driver do LRO on the RX end 231 * this can be toggled on the fly, but the 232 * interface must be reset (down/up) for it 233 * to take effect. 234 */
|
216static int ixgbe_enable_lro = 0;
| 235static int ixgbe_enable_lro = 1;
|
217TUNABLE_INT("hw.ixgbe.enable_lro", &ixgbe_enable_lro); 218 219/* 220 * MSIX should be the default for best performance, 221 * but this allows it to be forced off for testing. 222 */ 223static int ixgbe_enable_msix = 1; 224TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 225 226/*
| 236TUNABLE_INT("hw.ixgbe.enable_lro", &ixgbe_enable_lro); 237 238/* 239 * MSIX should be the default for best performance, 240 * but this allows it to be forced off for testing. 241 */ 242static int ixgbe_enable_msix = 1; 243TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 244 245/*
|
| 246 * Enable RX Header Split 247 */ 248static int ixgbe_rx_hdr_split = 1; 249TUNABLE_INT("hw.ixgbe.rx_hdr_split", &ixgbe_rx_hdr_split); 250 251/*
|
227 * Number of TX/RX Queues, with 0 setting 228 * it autoconfigures to the number of cpus. 229 */ 230static int ixgbe_tx_queues = 1; 231TUNABLE_INT("hw.ixgbe.tx_queues", &ixgbe_tx_queues);
| 252 * Number of TX/RX Queues, with 0 setting 253 * it autoconfigures to the number of cpus. 254 */ 255static int ixgbe_tx_queues = 1; 256TUNABLE_INT("hw.ixgbe.tx_queues", &ixgbe_tx_queues);
|
232static int ixgbe_rx_queues = 4;
| 257static int ixgbe_rx_queues = 1;
|
233TUNABLE_INT("hw.ixgbe.rx_queues", &ixgbe_rx_queues); 234 235/* Number of TX descriptors per ring */ 236static int ixgbe_txd = DEFAULT_TXD; 237TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 238 239/* Number of RX descriptors per ring */ 240static int ixgbe_rxd = DEFAULT_RXD; 241TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 242 243/* Total number of Interfaces - need for config sanity check */ 244static int ixgbe_total_ports; 245
| 258TUNABLE_INT("hw.ixgbe.rx_queues", &ixgbe_rx_queues); 259 260/* Number of TX descriptors per ring */ 261static int ixgbe_txd = DEFAULT_TXD; 262TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 263 264/* Number of RX descriptors per ring */ 265static int ixgbe_rxd = DEFAULT_RXD; 266TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 267 268/* Total number of Interfaces - need for config sanity check */ 269static int ixgbe_total_ports; 270
|
246/* Optics type of this interface */ 247static int ixgbe_optics; 248
| |
249/********************************************************************* 250 * Device identification routine 251 * 252 * ixgbe_probe determines if the driver should be loaded on 253 * adapter based on PCI vendor/device id of the adapter. 254 * 255 * return 0 on success, positive on failure 256 *********************************************************************/ 257 258static int 259ixgbe_probe(device_t dev) 260{ 261 ixgbe_vendor_info_t *ent; 262
| 271/********************************************************************* 272 * Device identification routine 273 * 274 * ixgbe_probe determines if the driver should be loaded on 275 * adapter based on PCI vendor/device id of the adapter. 276 * 277 * return 0 on success, positive on failure 278 *********************************************************************/ 279 280static int 281ixgbe_probe(device_t dev) 282{ 283 ixgbe_vendor_info_t *ent; 284
|
263 u_int16_t pci_vendor_id = 0; 264 u_int16_t pci_device_id = 0; 265 u_int16_t pci_subvendor_id = 0; 266 u_int16_t pci_subdevice_id = 0; 267 char adapter_name[128];
| 285 u16 pci_vendor_id = 0; 286 u16 pci_device_id = 0; 287 u16 pci_subvendor_id = 0; 288 u16 pci_subdevice_id = 0; 289 char adapter_name[256];
|
268 269 INIT_DEBUGOUT("ixgbe_probe: begin"); 270 271 pci_vendor_id = pci_get_vendor(dev); 272 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 273 return (ENXIO); 274 275 pci_device_id = pci_get_device(dev); 276 pci_subvendor_id = pci_get_subvendor(dev); 277 pci_subdevice_id = pci_get_subdevice(dev); 278 279 ent = ixgbe_vendor_info_array; 280 while (ent->vendor_id != 0) { 281 if ((pci_vendor_id == ent->vendor_id) && 282 (pci_device_id == ent->device_id) && 283 284 ((pci_subvendor_id == ent->subvendor_id) || 285 (ent->subvendor_id == 0)) && 286 287 ((pci_subdevice_id == ent->subdevice_id) || 288 (ent->subdevice_id == 0))) { 289 sprintf(adapter_name, "%s, Version - %s", 290 ixgbe_strings[ent->index], 291 ixgbe_driver_version);
| 290 291 INIT_DEBUGOUT("ixgbe_probe: begin"); 292 293 pci_vendor_id = pci_get_vendor(dev); 294 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 295 return (ENXIO); 296 297 pci_device_id = pci_get_device(dev); 298 pci_subvendor_id = pci_get_subvendor(dev); 299 pci_subdevice_id = pci_get_subdevice(dev); 300 301 ent = ixgbe_vendor_info_array; 302 while (ent->vendor_id != 0) { 303 if ((pci_vendor_id == ent->vendor_id) && 304 (pci_device_id == ent->device_id) && 305 306 ((pci_subvendor_id == ent->subvendor_id) || 307 (ent->subvendor_id == 0)) && 308 309 ((pci_subdevice_id == ent->subdevice_id) || 310 (ent->subdevice_id == 0))) { 311 sprintf(adapter_name, "%s, Version - %s", 312 ixgbe_strings[ent->index], 313 ixgbe_driver_version);
|
292 switch (pci_device_id) { 293 case IXGBE_DEV_ID_82598AT_DUAL_PORT : 294 ixgbe_total_ports += 2; 295 break; 296 case IXGBE_DEV_ID_82598_CX4_DUAL_PORT : 297 ixgbe_optics = IFM_10G_CX4; 298 ixgbe_total_ports += 2; 299 break; 300 case IXGBE_DEV_ID_82598AF_DUAL_PORT : 301 ixgbe_optics = IFM_10G_SR; 302 ixgbe_total_ports += 2; 303 break; 304 case IXGBE_DEV_ID_82598AF_SINGLE_PORT : 305 ixgbe_optics = IFM_10G_SR; 306 ixgbe_total_ports += 1; 307 break; 308 case IXGBE_DEV_ID_82598EB_XF_LR : 309 ixgbe_optics = IFM_10G_LR; 310 ixgbe_total_ports += 1; 311 break; 312 case IXGBE_DEV_ID_82598EB_CX4 : 313 ixgbe_optics = IFM_10G_CX4; 314 ixgbe_total_ports += 1; 315 break; 316 case IXGBE_DEV_ID_82598AT : 317 ixgbe_total_ports += 1; 318 default: 319 break; 320 }
| |
321 device_set_desc_copy(dev, adapter_name); 322 return (0); 323 } 324 ent++; 325 }
| 314 device_set_desc_copy(dev, adapter_name); 315 return (0); 316 } 317 ent++; 318 }
|
326
| |
327 return (ENXIO); 328} 329 330/********************************************************************* 331 * Device initialization routine 332 * 333 * The attach entry point is called when the driver is being loaded. 334 * This routine identifies the type of hardware, allocates all resources 335 * and initializes the hardware. 336 * 337 * return 0 on success, positive on failure 338 *********************************************************************/ 339 340static int 341ixgbe_attach(device_t dev) 342{ 343 struct adapter *adapter; 344 int error = 0;
| 319 return (ENXIO); 320} 321 322/********************************************************************* 323 * Device initialization routine 324 * 325 * The attach entry point is called when the driver is being loaded. 326 * This routine identifies the type of hardware, allocates all resources 327 * and initializes the hardware. 328 * 329 * return 0 on success, positive on failure 330 *********************************************************************/ 331 332static int 333ixgbe_attach(device_t dev) 334{ 335 struct adapter *adapter; 336 int error = 0;
|
345 u32 ctrl_ext;
| 337 u16 pci_device_id; 338 u32 ctrl_ext;
|
346 347 INIT_DEBUGOUT("ixgbe_attach: begin"); 348 349 /* Allocate, clear, and link in our adapter structure */ 350 adapter = device_get_softc(dev); 351 adapter->dev = adapter->osdep.dev = dev; 352 353 /* Core Lock Init*/ 354 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 355
| 339 340 INIT_DEBUGOUT("ixgbe_attach: begin"); 341 342 /* Allocate, clear, and link in our adapter structure */ 343 adapter = device_get_softc(dev); 344 adapter->dev = adapter->osdep.dev = dev; 345 346 /* Core Lock Init*/ 347 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 348
|
| 349 /* Keep track of number of ports and optics */ 350 pci_device_id = pci_get_device(dev); 351 switch (pci_device_id) { 352 case IXGBE_DEV_ID_82598_CX4_DUAL_PORT : 353 adapter->optics = IFM_10G_CX4; 354 ixgbe_total_ports += 2; 355 break; 356 case IXGBE_DEV_ID_82598AF_DUAL_PORT : 357 adapter->optics = IFM_10G_SR; 358 ixgbe_total_ports += 2; 359 break; 360 case IXGBE_DEV_ID_82598AF_SINGLE_PORT : 361 adapter->optics = IFM_10G_SR; 362 ixgbe_total_ports += 1; 363 break; 364 case IXGBE_DEV_ID_82598EB_XF_LR : 365 adapter->optics = IFM_10G_LR; 366 ixgbe_total_ports += 1; 367 break; 368 case IXGBE_DEV_ID_82598EB_CX4 : 369 adapter->optics = IFM_10G_CX4; 370 ixgbe_total_ports += 1; 371 break; 372 case IXGBE_DEV_ID_82598AT : 373 ixgbe_total_ports += 1; 374 case IXGBE_DEV_ID_82598_DA_DUAL_PORT : 375 ixgbe_total_ports += 2; 376 default: 377 break; 378 } 379
|
356 /* SYSCTL APIs */ 357 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 358 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 359 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 360 adapter, 0, ixgbe_sysctl_stats, "I", "Statistics"); 361 362 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 363 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 364 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 365 adapter, 0, ixgbe_sysctl_debug, "I", "Debug Info"); 366 367 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 368 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 369 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 370 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control"); 371 372 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 373 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 374 OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, 375 &ixgbe_enable_lro, 1, "Large Receive Offload"); 376
| 380 /* SYSCTL APIs */ 381 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 382 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 383 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 384 adapter, 0, ixgbe_sysctl_stats, "I", "Statistics"); 385 386 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 387 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 388 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 389 adapter, 0, ixgbe_sysctl_debug, "I", "Debug Info"); 390 391 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 392 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 393 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 394 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control"); 395 396 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 397 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 398 OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, 399 &ixgbe_enable_lro, 1, "Large Receive Offload"); 400
|
| 401 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 402 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 403 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 404 &ixgbe_enable_aim, 1, "Interrupt Moderation"); 405 406 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 407 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 408 OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW, 409 &ixgbe_low_latency, 1, "Low Latency"); 410 411 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 412 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 413 OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW, 414 &ixgbe_ave_latency, 1, "Average Latency"); 415 416 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 417 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 418 OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW, 419 &ixgbe_bulk_latency, 1, "Bulk Latency"); 420 421 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 422 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 423 OID_AUTO, "hdr_split", CTLTYPE_INT|CTLFLAG_RW, 424 &ixgbe_rx_hdr_split, 1, "RX Header Split"); 425
|
377 /* Set up the timer callout */ 378 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 379 380 /* Determine hardware revision */ 381 ixgbe_identify_hardware(adapter); 382
| 426 /* Set up the timer callout */ 427 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 428 429 /* Determine hardware revision */ 430 ixgbe_identify_hardware(adapter); 431
|
383 /* Indicate to RX setup to use Jumbo Clusters */ 384 adapter->bigbufs = TRUE; 385
| |
386 /* Do base PCI setup - map BAR0 */ 387 if (ixgbe_allocate_pci_resources(adapter)) { 388 device_printf(dev, "Allocation of PCI resources failed\n"); 389 error = ENXIO; 390 goto err_out; 391 } 392 393 /* Do descriptor calc and sanity checks */ 394 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 395 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 396 device_printf(dev, "TXD config issue, using default!\n"); 397 adapter->num_tx_desc = DEFAULT_TXD; 398 } else 399 adapter->num_tx_desc = ixgbe_txd; 400 401 /* 402 ** With many RX rings it is easy to exceed the 403 ** system mbuf allocation. Tuning nmbclusters 404 ** can alleviate this. 405 */ 406 if ((adapter->num_rx_queues > 1) && (nmbclusters > 0 )){ 407 int s; 408 /* Calculate the total RX mbuf needs */ 409 s = (ixgbe_rxd * adapter->num_rx_queues) * ixgbe_total_ports; 410 if (s > nmbclusters) { 411 device_printf(dev, "RX Descriptors exceed " 412 "system mbuf max, using default instead!\n"); 413 ixgbe_rxd = DEFAULT_RXD; 414 } 415 } 416 417 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 418 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 419 device_printf(dev, "RXD config issue, using default!\n"); 420 adapter->num_rx_desc = DEFAULT_RXD; 421 } else 422 adapter->num_rx_desc = ixgbe_rxd; 423 424 /* Allocate our TX/RX Queues */ 425 if (ixgbe_allocate_queues(adapter)) { 426 error = ENOMEM; 427 goto err_out; 428 } 429 430 /* Initialize the shared code */
| 432 /* Do base PCI setup - map BAR0 */ 433 if (ixgbe_allocate_pci_resources(adapter)) { 434 device_printf(dev, "Allocation of PCI resources failed\n"); 435 error = ENXIO; 436 goto err_out; 437 } 438 439 /* Do descriptor calc and sanity checks */ 440 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 441 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 442 device_printf(dev, "TXD config issue, using default!\n"); 443 adapter->num_tx_desc = DEFAULT_TXD; 444 } else 445 adapter->num_tx_desc = ixgbe_txd; 446 447 /* 448 ** With many RX rings it is easy to exceed the 449 ** system mbuf allocation. Tuning nmbclusters 450 ** can alleviate this. 451 */ 452 if ((adapter->num_rx_queues > 1) && (nmbclusters > 0 )){ 453 int s; 454 /* Calculate the total RX mbuf needs */ 455 s = (ixgbe_rxd * adapter->num_rx_queues) * ixgbe_total_ports; 456 if (s > nmbclusters) { 457 device_printf(dev, "RX Descriptors exceed " 458 "system mbuf max, using default instead!\n"); 459 ixgbe_rxd = DEFAULT_RXD; 460 } 461 } 462 463 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 464 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 465 device_printf(dev, "RXD config issue, using default!\n"); 466 adapter->num_rx_desc = DEFAULT_RXD; 467 } else 468 adapter->num_rx_desc = ixgbe_rxd; 469 470 /* Allocate our TX/RX Queues */ 471 if (ixgbe_allocate_queues(adapter)) { 472 error = ENOMEM; 473 goto err_out; 474 } 475 476 /* Initialize the shared code */
|
431 if (ixgbe_init_shared_code(&adapter->hw)) {
| 477 error = ixgbe_init_shared_code(&adapter->hw); 478 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 479 /* 480 ** No optics in this port, set up 481 ** so the timer routine will probe 482 ** for later insertion. 483 */ 484 adapter->sfp_probe = TRUE; 485 error = 0; 486 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 487 device_printf(dev,"Unsupported SFP+ module detected!\n"); 488 error = EIO; 489 goto err_late; 490 } else if (error) {
|
432 device_printf(dev,"Unable to initialize the shared code\n"); 433 error = EIO; 434 goto err_late; 435 } 436 437 /* Initialize the hardware */ 438 if (ixgbe_hardware_init(adapter)) { 439 device_printf(dev,"Unable to initialize the hardware\n"); 440 error = EIO; 441 goto err_late; 442 } 443 444 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 445 error = ixgbe_allocate_msix(adapter); 446 else 447 error = ixgbe_allocate_legacy(adapter); 448 if (error) 449 goto err_late; 450 451 /* Setup OS specific network interface */ 452 ixgbe_setup_interface(dev, adapter); 453 454 /* Sysctl for limiting the amount of work done in the taskqueue */ 455 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 456 "max number of rx packets to process", &adapter->rx_process_limit, 457 ixgbe_rx_process_limit); 458 459 /* Initialize statistics */ 460 ixgbe_update_stats_counters(adapter); 461
| 491 device_printf(dev,"Unable to initialize the shared code\n"); 492 error = EIO; 493 goto err_late; 494 } 495 496 /* Initialize the hardware */ 497 if (ixgbe_hardware_init(adapter)) { 498 device_printf(dev,"Unable to initialize the hardware\n"); 499 error = EIO; 500 goto err_late; 501 } 502 503 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 504 error = ixgbe_allocate_msix(adapter); 505 else 506 error = ixgbe_allocate_legacy(adapter); 507 if (error) 508 goto err_late; 509 510 /* Setup OS specific network interface */ 511 ixgbe_setup_interface(dev, adapter); 512 513 /* Sysctl for limiting the amount of work done in the taskqueue */ 514 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 515 "max number of rx packets to process", &adapter->rx_process_limit, 516 ixgbe_rx_process_limit); 517 518 /* Initialize statistics */ 519 ixgbe_update_stats_counters(adapter); 520
|
462#ifdef IXGBE_VLAN_EVENTS
| 521#ifdef IXGBE_HW_VLAN_SUPPORT
|
463 /* Register for VLAN events */ 464 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 465 ixgbe_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 466 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 467 ixgbe_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 468#endif
| 522 /* Register for VLAN events */ 523 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 524 ixgbe_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 525 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 526 ixgbe_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 527#endif
|
469
| 528
|
470 /* let hardware know driver is loaded */ 471 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 472 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 473 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 474 475 INIT_DEBUGOUT("ixgbe_attach: end"); 476 return (0); 477err_late: 478 ixgbe_free_transmit_structures(adapter); 479 ixgbe_free_receive_structures(adapter); 480err_out: 481 ixgbe_free_pci_resources(adapter); 482 return (error); 483 484} 485 486/********************************************************************* 487 * Device removal routine 488 * 489 * The detach entry point is called when the driver is being removed. 490 * This routine stops the adapter and deallocates all the resources 491 * that were allocated for driver operation. 492 * 493 * return 0 on success, positive on failure 494 *********************************************************************/ 495 496static int 497ixgbe_detach(device_t dev) 498{ 499 struct adapter *adapter = device_get_softc(dev); 500 struct tx_ring *txr = adapter->tx_rings; 501 struct rx_ring *rxr = adapter->rx_rings; 502 u32 ctrl_ext; 503 504 INIT_DEBUGOUT("ixgbe_detach: begin"); 505 506 /* Make sure VLANS are not using driver */ 507#if __FreeBSD_version >= 700000 508 if (adapter->ifp->if_vlantrunk != NULL) { 509#else 510 if (adapter->ifp->if_nvlans != 0) { 511#endif 512 device_printf(dev,"Vlan in use, detach first\n"); 513 return (EBUSY); 514 } 515 516 IXGBE_CORE_LOCK(adapter); 517 ixgbe_stop(adapter); 518 IXGBE_CORE_UNLOCK(adapter); 519 520 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 521 if (txr->tq) { 522 taskqueue_drain(txr->tq, &txr->tx_task); 523 taskqueue_free(txr->tq);
| 529 /* let hardware know driver is loaded */ 530 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 531 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 532 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 533 534 INIT_DEBUGOUT("ixgbe_attach: end"); 535 return (0); 536err_late: 537 ixgbe_free_transmit_structures(adapter); 538 ixgbe_free_receive_structures(adapter); 539err_out: 540 ixgbe_free_pci_resources(adapter); 541 return (error); 542 543} 544 545/********************************************************************* 546 * Device removal routine 547 * 548 * The detach entry point is called when the driver is being removed. 549 * This routine stops the adapter and deallocates all the resources 550 * that were allocated for driver operation. 551 * 552 * return 0 on success, positive on failure 553 *********************************************************************/ 554 555static int 556ixgbe_detach(device_t dev) 557{ 558 struct adapter *adapter = device_get_softc(dev); 559 struct tx_ring *txr = adapter->tx_rings; 560 struct rx_ring *rxr = adapter->rx_rings; 561 u32 ctrl_ext; 562 563 INIT_DEBUGOUT("ixgbe_detach: begin"); 564 565 /* Make sure VLANS are not using driver */ 566#if __FreeBSD_version >= 700000 567 if (adapter->ifp->if_vlantrunk != NULL) { 568#else 569 if (adapter->ifp->if_nvlans != 0) { 570#endif 571 device_printf(dev,"Vlan in use, detach first\n"); 572 return (EBUSY); 573 } 574 575 IXGBE_CORE_LOCK(adapter); 576 ixgbe_stop(adapter); 577 IXGBE_CORE_UNLOCK(adapter); 578 579 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 580 if (txr->tq) { 581 taskqueue_drain(txr->tq, &txr->tx_task); 582 taskqueue_free(txr->tq);
|
524 txr->tq = NULL;
| |
525 } 526 } 527 528 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 529 if (rxr->tq) { 530 taskqueue_drain(rxr->tq, &rxr->rx_task); 531 taskqueue_free(rxr->tq);
| 583 } 584 } 585 586 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 587 if (rxr->tq) { 588 taskqueue_drain(rxr->tq, &rxr->rx_task); 589 taskqueue_free(rxr->tq);
|
532 rxr->tq = NULL;
| |
533 } 534 } 535
| 590 } 591 } 592
|
536#ifdef IXGBE_VLAN_EVENTS
| 593 /* let hardware know driver is unloading */ 594 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 595 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 596 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 597 598#ifdef IXGBE_HW_VLAN_SUPPORT
|
537 /* Unregister VLAN events */ 538 if (adapter->vlan_attach != NULL) 539 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 540 if (adapter->vlan_detach != NULL) 541 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
| 599 /* Unregister VLAN events */ 600 if (adapter->vlan_attach != NULL) 601 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 602 if (adapter->vlan_detach != NULL) 603 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
|
542#endif
| 604#endif
|
543
| 605
|
544 /* let hardware know driver is unloading */ 545 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 546 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 547 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 548
| |
549 ether_ifdetach(adapter->ifp); 550 callout_drain(&adapter->timer); 551 ixgbe_free_pci_resources(adapter); 552 bus_generic_detach(dev); 553 if_free(adapter->ifp); 554 555 ixgbe_free_transmit_structures(adapter); 556 ixgbe_free_receive_structures(adapter); 557 558 IXGBE_CORE_LOCK_DESTROY(adapter); 559 return (0); 560} 561 562/********************************************************************* 563 * 564 * Shutdown entry point 565 * 566 **********************************************************************/ 567 568static int 569ixgbe_shutdown(device_t dev) 570{ 571 struct adapter *adapter = device_get_softc(dev); 572 IXGBE_CORE_LOCK(adapter); 573 ixgbe_stop(adapter); 574 IXGBE_CORE_UNLOCK(adapter); 575 return (0); 576} 577 578 579/********************************************************************* 580 * Transmit entry point 581 * 582 * ixgbe_start is called by the stack to initiate a transmit. 583 * The driver will remain in this routine as long as there are 584 * packets to transmit and transmit resources are available. 585 * In case resources are not available stack is notified and 586 * the packet is requeued. 587 **********************************************************************/ 588 589static void 590ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 591{ 592 struct mbuf *m_head; 593 struct adapter *adapter = txr->adapter; 594 595 IXGBE_TX_LOCK_ASSERT(txr); 596 597 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 598 IFF_DRV_RUNNING) 599 return; 600 if (!adapter->link_active) 601 return; 602 603 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 604 605 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 606 if (m_head == NULL) 607 break; 608 609 if (ixgbe_xmit(txr, &m_head)) { 610 if (m_head == NULL) 611 break; 612 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 613 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 614 break; 615 } 616 /* Send a copy of the frame to the BPF listener */ 617 ETHER_BPF_MTAP(ifp, m_head); 618 619 /* Set timeout in case hardware has problems transmitting */ 620 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 621 622 } 623 return; 624} 625 626 627static void 628ixgbe_start(struct ifnet *ifp) 629{ 630 struct adapter *adapter = ifp->if_softc; 631 struct tx_ring *txr = adapter->tx_rings; 632 u32 queue = 0; 633 634 /* 635 ** This is really just here for testing 636 ** TX multiqueue, ultimately what is 637 ** needed is the flow support in the stack 638 ** and appropriate logic here to deal with 639 ** it. -jfv 640 */ 641 if (adapter->num_tx_queues > 1) 642 queue = (curcpu % adapter->num_tx_queues); 643 644 txr = &adapter->tx_rings[queue]; 645 646 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 647 IXGBE_TX_LOCK(txr); 648 ixgbe_start_locked(txr, ifp); 649 IXGBE_TX_UNLOCK(txr); 650 } 651 return; 652} 653 654/********************************************************************* 655 * Ioctl entry point 656 * 657 * ixgbe_ioctl is called when the user wants to configure the 658 * interface. 659 * 660 * return 0 on success, positive on failure 661 **********************************************************************/ 662 663static int 664ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 665{ 666 int error = 0; 667 struct ifreq *ifr = (struct ifreq *) data; 668 struct ifaddr *ifa = (struct ifaddr *) data; 669 struct adapter *adapter = ifp->if_softc; 670 671 switch (command) { 672 case SIOCSIFADDR: 673 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 674 if (ifa->ifa_addr->sa_family == AF_INET) { 675 ifp->if_flags |= IFF_UP; 676 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 677 IXGBE_CORE_LOCK(adapter); 678 ixgbe_init_locked(adapter); 679 IXGBE_CORE_UNLOCK(adapter); 680 } 681 arp_ifinit(ifp, ifa); 682 } else 683 ether_ioctl(ifp, command, data); 684 break; 685 case SIOCSIFMTU: 686 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 687 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 688 error = EINVAL; 689 } else { 690 IXGBE_CORE_LOCK(adapter); 691 ifp->if_mtu = ifr->ifr_mtu; 692 adapter->max_frame_size = 693 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 694 ixgbe_init_locked(adapter); 695 IXGBE_CORE_UNLOCK(adapter); 696 } 697 break; 698 case SIOCSIFFLAGS: 699 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 700 IXGBE_CORE_LOCK(adapter); 701 if (ifp->if_flags & IFF_UP) { 702 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 703 if ((ifp->if_flags ^ adapter->if_flags) & 704 (IFF_PROMISC | IFF_ALLMULTI)) { 705 ixgbe_disable_promisc(adapter); 706 ixgbe_set_promisc(adapter); 707 } 708 } else 709 ixgbe_init_locked(adapter); 710 } else 711 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 712 ixgbe_stop(adapter); 713 adapter->if_flags = ifp->if_flags; 714 IXGBE_CORE_UNLOCK(adapter); 715 break; 716 case SIOCADDMULTI: 717 case SIOCDELMULTI: 718 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 719 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 720 IXGBE_CORE_LOCK(adapter); 721 ixgbe_disable_intr(adapter); 722 ixgbe_set_multi(adapter); 723 ixgbe_enable_intr(adapter); 724 IXGBE_CORE_UNLOCK(adapter); 725 } 726 break; 727 case SIOCSIFMEDIA: 728 case SIOCGIFMEDIA: 729 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 730 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 731 break; 732 case SIOCSIFCAP: 733 { 734 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 735 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 736 if (mask & IFCAP_HWCSUM) 737 ifp->if_capenable ^= IFCAP_HWCSUM; 738 if (mask & IFCAP_TSO4) 739 ifp->if_capenable ^= IFCAP_TSO4; 740 if (mask & IFCAP_VLAN_HWTAGGING) 741 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 742 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 743 ixgbe_init(adapter); 744#if __FreeBSD_version >= 700000 745 VLAN_CAPABILITIES(ifp); 746#endif 747 break; 748 } 749 default: 750 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 751 error = ether_ioctl(ifp, command, data); 752 break; 753 } 754 755 return (error); 756} 757 758/********************************************************************* 759 * Watchdog entry point 760 * 761 * This routine is called by the local timer 762 * to detect hardware hangs . 763 * 764 **********************************************************************/ 765 766static void 767ixgbe_watchdog(struct adapter *adapter) 768{ 769 device_t dev = adapter->dev; 770 struct tx_ring *txr = adapter->tx_rings; 771 struct ixgbe_hw *hw = &adapter->hw; 772 bool tx_hang = FALSE; 773 774 IXGBE_CORE_LOCK_ASSERT(adapter); 775 776 /* 777 * The timer is set to 5 every time ixgbe_start() queues a packet. 778 * Then ixgbe_txeof() keeps resetting to 5 as long as it cleans at 779 * least one descriptor. 780 * Finally, anytime all descriptors are clean the timer is 781 * set to 0. 782 */ 783 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 784 u32 head, tail; 785 786 IXGBE_TX_LOCK(txr); 787 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) { 788 IXGBE_TX_UNLOCK(txr); 789 continue; 790 } else { 791 head = IXGBE_READ_REG(hw, IXGBE_TDH(i)); 792 tail = IXGBE_READ_REG(hw, IXGBE_TDT(i)); 793 if (head == tail) { /* last minute check */ 794 IXGBE_TX_UNLOCK(txr); 795 continue; 796 } 797 /* Well, seems something is really hung */ 798 tx_hang = TRUE; 799 IXGBE_TX_UNLOCK(txr); 800 break; 801 } 802 } 803 if (tx_hang == FALSE) 804 return; 805 806 /* 807 * If we are in this routine because of pause frames, then don't 808 * reset the hardware. 809 */ 810 if (IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { 811 txr = adapter->tx_rings; /* reset pointer */ 812 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 813 IXGBE_TX_LOCK(txr); 814 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 815 IXGBE_TX_UNLOCK(txr); 816 } 817 return; 818 } 819 820 821 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 822 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 823 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i, 824 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 825 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 826 device_printf(dev,"TX(%d) desc avail = %d," 827 "Next TX to Clean = %d\n", 828 i, txr->tx_avail, txr->next_tx_to_clean); 829 } 830 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 831 adapter->watchdog_events++; 832 833 ixgbe_init_locked(adapter); 834} 835 836/********************************************************************* 837 * Init entry point 838 * 839 * This routine is used in two ways. It is used by the stack as 840 * init entry point in network interface structure. It is also used 841 * by the driver as a hw/sw initialization routine to get to a 842 * consistent state. 843 * 844 * return 0 on success, positive on failure 845 **********************************************************************/ 846#define IXGBE_MHADD_MFS_SHIFT 16 847 848static void 849ixgbe_init_locked(struct adapter *adapter) 850{
| 606 ether_ifdetach(adapter->ifp); 607 callout_drain(&adapter->timer); 608 ixgbe_free_pci_resources(adapter); 609 bus_generic_detach(dev); 610 if_free(adapter->ifp); 611 612 ixgbe_free_transmit_structures(adapter); 613 ixgbe_free_receive_structures(adapter); 614 615 IXGBE_CORE_LOCK_DESTROY(adapter); 616 return (0); 617} 618 619/********************************************************************* 620 * 621 * Shutdown entry point 622 * 623 **********************************************************************/ 624 625static int 626ixgbe_shutdown(device_t dev) 627{ 628 struct adapter *adapter = device_get_softc(dev); 629 IXGBE_CORE_LOCK(adapter); 630 ixgbe_stop(adapter); 631 IXGBE_CORE_UNLOCK(adapter); 632 return (0); 633} 634 635 636/********************************************************************* 637 * Transmit entry point 638 * 639 * ixgbe_start is called by the stack to initiate a transmit. 640 * The driver will remain in this routine as long as there are 641 * packets to transmit and transmit resources are available. 642 * In case resources are not available stack is notified and 643 * the packet is requeued. 644 **********************************************************************/ 645 646static void 647ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 648{ 649 struct mbuf *m_head; 650 struct adapter *adapter = txr->adapter; 651 652 IXGBE_TX_LOCK_ASSERT(txr); 653 654 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 655 IFF_DRV_RUNNING) 656 return; 657 if (!adapter->link_active) 658 return; 659 660 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 661 662 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 663 if (m_head == NULL) 664 break; 665 666 if (ixgbe_xmit(txr, &m_head)) { 667 if (m_head == NULL) 668 break; 669 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 670 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 671 break; 672 } 673 /* Send a copy of the frame to the BPF listener */ 674 ETHER_BPF_MTAP(ifp, m_head); 675 676 /* Set timeout in case hardware has problems transmitting */ 677 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 678 679 } 680 return; 681} 682 683 684static void 685ixgbe_start(struct ifnet *ifp) 686{ 687 struct adapter *adapter = ifp->if_softc; 688 struct tx_ring *txr = adapter->tx_rings; 689 u32 queue = 0; 690 691 /* 692 ** This is really just here for testing 693 ** TX multiqueue, ultimately what is 694 ** needed is the flow support in the stack 695 ** and appropriate logic here to deal with 696 ** it. -jfv 697 */ 698 if (adapter->num_tx_queues > 1) 699 queue = (curcpu % adapter->num_tx_queues); 700 701 txr = &adapter->tx_rings[queue]; 702 703 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 704 IXGBE_TX_LOCK(txr); 705 ixgbe_start_locked(txr, ifp); 706 IXGBE_TX_UNLOCK(txr); 707 } 708 return; 709} 710 711/********************************************************************* 712 * Ioctl entry point 713 * 714 * ixgbe_ioctl is called when the user wants to configure the 715 * interface. 716 * 717 * return 0 on success, positive on failure 718 **********************************************************************/ 719 720static int 721ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 722{ 723 int error = 0; 724 struct ifreq *ifr = (struct ifreq *) data; 725 struct ifaddr *ifa = (struct ifaddr *) data; 726 struct adapter *adapter = ifp->if_softc; 727 728 switch (command) { 729 case SIOCSIFADDR: 730 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 731 if (ifa->ifa_addr->sa_family == AF_INET) { 732 ifp->if_flags |= IFF_UP; 733 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 734 IXGBE_CORE_LOCK(adapter); 735 ixgbe_init_locked(adapter); 736 IXGBE_CORE_UNLOCK(adapter); 737 } 738 arp_ifinit(ifp, ifa); 739 } else 740 ether_ioctl(ifp, command, data); 741 break; 742 case SIOCSIFMTU: 743 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 744 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 745 error = EINVAL; 746 } else { 747 IXGBE_CORE_LOCK(adapter); 748 ifp->if_mtu = ifr->ifr_mtu; 749 adapter->max_frame_size = 750 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 751 ixgbe_init_locked(adapter); 752 IXGBE_CORE_UNLOCK(adapter); 753 } 754 break; 755 case SIOCSIFFLAGS: 756 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 757 IXGBE_CORE_LOCK(adapter); 758 if (ifp->if_flags & IFF_UP) { 759 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 760 if ((ifp->if_flags ^ adapter->if_flags) & 761 (IFF_PROMISC | IFF_ALLMULTI)) { 762 ixgbe_disable_promisc(adapter); 763 ixgbe_set_promisc(adapter); 764 } 765 } else 766 ixgbe_init_locked(adapter); 767 } else 768 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 769 ixgbe_stop(adapter); 770 adapter->if_flags = ifp->if_flags; 771 IXGBE_CORE_UNLOCK(adapter); 772 break; 773 case SIOCADDMULTI: 774 case SIOCDELMULTI: 775 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 776 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 777 IXGBE_CORE_LOCK(adapter); 778 ixgbe_disable_intr(adapter); 779 ixgbe_set_multi(adapter); 780 ixgbe_enable_intr(adapter); 781 IXGBE_CORE_UNLOCK(adapter); 782 } 783 break; 784 case SIOCSIFMEDIA: 785 case SIOCGIFMEDIA: 786 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 787 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 788 break; 789 case SIOCSIFCAP: 790 { 791 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 792 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 793 if (mask & IFCAP_HWCSUM) 794 ifp->if_capenable ^= IFCAP_HWCSUM; 795 if (mask & IFCAP_TSO4) 796 ifp->if_capenable ^= IFCAP_TSO4; 797 if (mask & IFCAP_VLAN_HWTAGGING) 798 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 799 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 800 ixgbe_init(adapter); 801#if __FreeBSD_version >= 700000 802 VLAN_CAPABILITIES(ifp); 803#endif 804 break; 805 } 806 default: 807 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 808 error = ether_ioctl(ifp, command, data); 809 break; 810 } 811 812 return (error); 813} 814 815/********************************************************************* 816 * Watchdog entry point 817 * 818 * This routine is called by the local timer 819 * to detect hardware hangs . 820 * 821 **********************************************************************/ 822 823static void 824ixgbe_watchdog(struct adapter *adapter) 825{ 826 device_t dev = adapter->dev; 827 struct tx_ring *txr = adapter->tx_rings; 828 struct ixgbe_hw *hw = &adapter->hw; 829 bool tx_hang = FALSE; 830 831 IXGBE_CORE_LOCK_ASSERT(adapter); 832 833 /* 834 * The timer is set to 5 every time ixgbe_start() queues a packet. 835 * Then ixgbe_txeof() keeps resetting to 5 as long as it cleans at 836 * least one descriptor. 837 * Finally, anytime all descriptors are clean the timer is 838 * set to 0. 839 */ 840 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 841 u32 head, tail; 842 843 IXGBE_TX_LOCK(txr); 844 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) { 845 IXGBE_TX_UNLOCK(txr); 846 continue; 847 } else { 848 head = IXGBE_READ_REG(hw, IXGBE_TDH(i)); 849 tail = IXGBE_READ_REG(hw, IXGBE_TDT(i)); 850 if (head == tail) { /* last minute check */ 851 IXGBE_TX_UNLOCK(txr); 852 continue; 853 } 854 /* Well, seems something is really hung */ 855 tx_hang = TRUE; 856 IXGBE_TX_UNLOCK(txr); 857 break; 858 } 859 } 860 if (tx_hang == FALSE) 861 return; 862 863 /* 864 * If we are in this routine because of pause frames, then don't 865 * reset the hardware. 866 */ 867 if (IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { 868 txr = adapter->tx_rings; /* reset pointer */ 869 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 870 IXGBE_TX_LOCK(txr); 871 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 872 IXGBE_TX_UNLOCK(txr); 873 } 874 return; 875 } 876 877 878 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 879 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 880 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i, 881 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 882 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 883 device_printf(dev,"TX(%d) desc avail = %d," 884 "Next TX to Clean = %d\n", 885 i, txr->tx_avail, txr->next_tx_to_clean); 886 } 887 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 888 adapter->watchdog_events++; 889 890 ixgbe_init_locked(adapter); 891} 892 893/********************************************************************* 894 * Init entry point 895 * 896 * This routine is used in two ways. It is used by the stack as 897 * init entry point in network interface structure. It is also used 898 * by the driver as a hw/sw initialization routine to get to a 899 * consistent state. 900 * 901 * return 0 on success, positive on failure 902 **********************************************************************/ 903#define IXGBE_MHADD_MFS_SHIFT 16 904 905static void 906ixgbe_init_locked(struct adapter *adapter) 907{
|
| 908 struct rx_ring *rxr = adapter->rx_rings; 909 struct tx_ring *txr = adapter->tx_rings;
|
851 struct ifnet *ifp = adapter->ifp; 852 device_t dev = adapter->dev; 853 struct ixgbe_hw *hw;
| 910 struct ifnet *ifp = adapter->ifp; 911 device_t dev = adapter->dev; 912 struct ixgbe_hw *hw;
|
854 u32 txdctl, rxdctl, mhadd, gpie;
| 913 u32 k, txdctl, mhadd, gpie; 914 u32 rxdctl, rxctrl;
|
855 856 INIT_DEBUGOUT("ixgbe_init: begin"); 857 858 hw = &adapter->hw; 859 mtx_assert(&adapter->core_mtx, MA_OWNED); 860 861 ixgbe_stop(adapter); 862 863 /* Get the latest mac address, User can use a LAA */ 864 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 865 IXGBE_ETH_LENGTH_OF_ADDRESS); 866 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, 1); 867 adapter->hw.addr_ctrl.rar_used_count = 1; 868 869 /* Initialize the hardware */ 870 if (ixgbe_hardware_init(adapter)) { 871 device_printf(dev, "Unable to initialize the hardware\n"); 872 return; 873 } 874
| 915 916 INIT_DEBUGOUT("ixgbe_init: begin"); 917 918 hw = &adapter->hw; 919 mtx_assert(&adapter->core_mtx, MA_OWNED); 920 921 ixgbe_stop(adapter); 922 923 /* Get the latest mac address, User can use a LAA */ 924 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 925 IXGBE_ETH_LENGTH_OF_ADDRESS); 926 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, 1); 927 adapter->hw.addr_ctrl.rar_used_count = 1; 928 929 /* Initialize the hardware */ 930 if (ixgbe_hardware_init(adapter)) { 931 device_printf(dev, "Unable to initialize the hardware\n"); 932 return; 933 } 934
|
875#ifndef IXGBE_VLAN_EVENTS 876 /* With events this is done when a vlan registers */
| 935#ifndef IXGBE_HW_VLAN_SUPPORT
|
877 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
| 936 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
|
878 u32 ctrl;
| 937 u32 ctrl; 938
|
879 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 880 ctrl |= IXGBE_VLNCTRL_VME; 881 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 882 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 883 } 884#endif
| 939 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 940 ctrl |= IXGBE_VLNCTRL_VME; 941 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 942 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 943 } 944#endif
|
885
| |
886 /* Prepare transmit descriptors and buffers */ 887 if (ixgbe_setup_transmit_structures(adapter)) { 888 device_printf(dev,"Could not setup transmit structures\n"); 889 ixgbe_stop(adapter); 890 return; 891 } 892 893 ixgbe_initialize_transmit_units(adapter); 894
| 945 /* Prepare transmit descriptors and buffers */ 946 if (ixgbe_setup_transmit_structures(adapter)) { 947 device_printf(dev,"Could not setup transmit structures\n"); 948 ixgbe_stop(adapter); 949 return; 950 } 951 952 ixgbe_initialize_transmit_units(adapter); 953
|
| 954 /* TX irq moderation rate is fixed */ 955 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 956 IXGBE_WRITE_REG(&adapter->hw, 957 IXGBE_EITR(txr->msix), ixgbe_ave_latency); 958 txr->watchdog_timer = FALSE; 959 } 960
|
895 /* Setup Multicast table */ 896 ixgbe_set_multi(adapter); 897 898 /*
| 961 /* Setup Multicast table */ 962 ixgbe_set_multi(adapter); 963 964 /*
|
899 ** If we are resetting MTU smaller than 2K 900 ** drop to small RX buffers
| 965 ** Determine the correct mbuf pool 966 ** for doing jumbo/headersplit
|
901 */
| 967 */
|
902 if (adapter->max_frame_size <= MCLBYTES) 903 adapter->bigbufs = FALSE;
| 968 if (ifp->if_mtu > ETHERMTU) 969 adapter->rx_mbuf_sz = MJUMPAGESIZE; 970 else 971 adapter->rx_mbuf_sz = MCLBYTES;
|
904 905 /* Prepare receive descriptors and buffers */ 906 if (ixgbe_setup_receive_structures(adapter)) { 907 device_printf(dev,"Could not setup receive structures\n"); 908 ixgbe_stop(adapter); 909 return; 910 } 911 912 /* Configure RX settings */ 913 ixgbe_initialize_receive_units(adapter); 914
| 972 973 /* Prepare receive descriptors and buffers */ 974 if (ixgbe_setup_receive_structures(adapter)) { 975 device_printf(dev,"Could not setup receive structures\n"); 976 ixgbe_stop(adapter); 977 return; 978 } 979 980 /* Configure RX settings */ 981 ixgbe_initialize_receive_units(adapter); 982
|
| 983 /* RX moderation will be adapted over time, set default */ 984 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 985 IXGBE_WRITE_REG(&adapter->hw, 986 IXGBE_EITR(rxr->msix), ixgbe_low_latency); 987 } 988 989 /* Set Link moderation */ 990 IXGBE_WRITE_REG(&adapter->hw, 991 IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 992
|
915 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
| 993 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
|
| 994
|
916 /* Enable Fan Failure Interrupt */ 917 if (adapter->hw.phy.media_type == ixgbe_media_type_copper) 918 gpie |= IXGBE_SDP1_GPIEN;
| 995 /* Enable Fan Failure Interrupt */ 996 if (adapter->hw.phy.media_type == ixgbe_media_type_copper) 997 gpie |= IXGBE_SDP1_GPIEN;
|
| 998
|
919 if (adapter->msix) { 920 /* Enable Enhanced MSIX mode */ 921 gpie |= IXGBE_GPIE_MSIX_MODE; 922 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 923 IXGBE_GPIE_OCD; 924 } 925 IXGBE_WRITE_REG(&adapter->hw, IXGBE_GPIE, gpie); 926 927 /* Set the various hardware offload abilities */ 928 ifp->if_hwassist = 0; 929 if (ifp->if_capenable & IFCAP_TSO4) 930 ifp->if_hwassist |= CSUM_TSO; 931 else if (ifp->if_capenable & IFCAP_TXCSUM) 932 ifp->if_hwassist = (CSUM_TCP | CSUM_UDP); 933 934 /* Set MTU size */ 935 if (ifp->if_mtu > ETHERMTU) { 936 mhadd = IXGBE_READ_REG(&adapter->hw, IXGBE_MHADD); 937 mhadd &= ~IXGBE_MHADD_MFS_MASK; 938 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 939 IXGBE_WRITE_REG(&adapter->hw, IXGBE_MHADD, mhadd); 940 } 941 942 /* Now enable all the queues */ 943 944 for (int i = 0; i < adapter->num_tx_queues; i++) { 945 txdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TXDCTL(i)); 946 txdctl |= IXGBE_TXDCTL_ENABLE; 947 /* Set WTHRESH to 8, burst writeback */ 948 txdctl |= (8 << 16); 949 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TXDCTL(i), txdctl); 950 } 951 952 for (int i = 0; i < adapter->num_rx_queues; i++) { 953 rxdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXDCTL(i)); 954 /* PTHRESH set to 32 */ 955 rxdctl |= 0x0020; 956 rxdctl |= IXGBE_RXDCTL_ENABLE; 957 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXDCTL(i), rxdctl);
| 999 if (adapter->msix) { 1000 /* Enable Enhanced MSIX mode */ 1001 gpie |= IXGBE_GPIE_MSIX_MODE; 1002 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1003 IXGBE_GPIE_OCD; 1004 } 1005 IXGBE_WRITE_REG(&adapter->hw, IXGBE_GPIE, gpie); 1006 1007 /* Set the various hardware offload abilities */ 1008 ifp->if_hwassist = 0; 1009 if (ifp->if_capenable & IFCAP_TSO4) 1010 ifp->if_hwassist |= CSUM_TSO; 1011 else if (ifp->if_capenable & IFCAP_TXCSUM) 1012 ifp->if_hwassist = (CSUM_TCP | CSUM_UDP); 1013 1014 /* Set MTU size */ 1015 if (ifp->if_mtu > ETHERMTU) { 1016 mhadd = IXGBE_READ_REG(&adapter->hw, IXGBE_MHADD); 1017 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1018 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1019 IXGBE_WRITE_REG(&adapter->hw, IXGBE_MHADD, mhadd); 1020 } 1021 1022 /* Now enable all the queues */ 1023 1024 for (int i = 0; i < adapter->num_tx_queues; i++) { 1025 txdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TXDCTL(i)); 1026 txdctl |= IXGBE_TXDCTL_ENABLE; 1027 /* Set WTHRESH to 8, burst writeback */ 1028 txdctl |= (8 << 16); 1029 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TXDCTL(i), txdctl); 1030 } 1031 1032 for (int i = 0; i < adapter->num_rx_queues; i++) { 1033 rxdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXDCTL(i)); 1034 /* PTHRESH set to 32 */ 1035 rxdctl |= 0x0020; 1036 rxdctl |= IXGBE_RXDCTL_ENABLE; 1037 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXDCTL(i), rxdctl);
|
| 1038 for (k = 0; k < 10; k++) { 1039 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1040 IXGBE_RXDCTL_ENABLE) 1041 break; 1042 else 1043 msec_delay(1); 1044 } 1045 wmb(); 1046 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
|
958 } 959
| 1047 } 1048
|
| 1049 /* Enable Receive engine */ 1050 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1051 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1052 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1053 rxctrl |= IXGBE_RXCTRL_RXEN; 1054 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl); 1055
|
960 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 961 962 /* Set up MSI/X routing */
| 1056 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1057 1058 /* Set up MSI/X routing */
|
963 ixgbe_configure_ivars(adapter);
| 1059 if (ixgbe_enable_msix) 1060 ixgbe_configure_ivars(adapter);
|
964 965 ixgbe_enable_intr(adapter); 966 967 /* Now inform the stack we're ready */ 968 ifp->if_drv_flags |= IFF_DRV_RUNNING; 969 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 970 971 return; 972} 973 974static void 975ixgbe_init(void *arg) 976{ 977 struct adapter *adapter = arg; 978 979 IXGBE_CORE_LOCK(adapter); 980 ixgbe_init_locked(adapter); 981 IXGBE_CORE_UNLOCK(adapter); 982 return; 983} 984 985 986/*
| 1061 1062 ixgbe_enable_intr(adapter); 1063 1064 /* Now inform the stack we're ready */ 1065 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1066 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1067 1068 return; 1069} 1070 1071static void 1072ixgbe_init(void *arg) 1073{ 1074 struct adapter *adapter = arg; 1075 1076 IXGBE_CORE_LOCK(adapter); 1077 ixgbe_init_locked(adapter); 1078 IXGBE_CORE_UNLOCK(adapter); 1079 return; 1080} 1081 1082 1083/*
|
987** Legacy Deferred Interrupt Handlers
| 1084** MSIX Interrupt Handlers
|
988*/ 989 990static void 991ixgbe_handle_rx(void *context, int pending) 992{ 993 struct rx_ring *rxr = context; 994 struct adapter *adapter = rxr->adapter;
| 1085*/ 1086 1087static void 1088ixgbe_handle_rx(void *context, int pending) 1089{ 1090 struct rx_ring *rxr = context; 1091 struct adapter *adapter = rxr->adapter;
|
995 u32 loop = 0;
| 1092 u32 loop = MAX_LOOP; 1093 bool more;
|
996
| 1094
|
997 while (loop++ < MAX_INTR) 998 if (ixgbe_rxeof(rxr, adapter->rx_process_limit) == 0) 999 break;
| 1095 do { 1096 more = ixgbe_rxeof(rxr, -1); 1097 } while (loop-- && more); 1098 /* Reenable this interrupt */ 1099 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims);
|
1000} 1001 1002static void 1003ixgbe_handle_tx(void *context, int pending) 1004{ 1005 struct tx_ring *txr = context; 1006 struct adapter *adapter = txr->adapter; 1007 struct ifnet *ifp = adapter->ifp;
| 1100} 1101 1102static void 1103ixgbe_handle_tx(void *context, int pending) 1104{ 1105 struct tx_ring *txr = context; 1106 struct adapter *adapter = txr->adapter; 1107 struct ifnet *ifp = adapter->ifp;
|
1008 u32 loop = 0;
| 1108 u32 loop = MAX_LOOP; 1109 bool more;
|
1009
| 1110
|
1010 IXGBE_TX_LOCK(txr); 1011 while (loop++ < MAX_INTR) 1012 if (ixgbe_txeof(txr) == 0) 1013 break; 1014 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1015 ixgbe_start_locked(txr, ifp); 1016 IXGBE_TX_UNLOCK(txr);
| 1111 IXGBE_TX_LOCK(txr); 1112 do { 1113 more = ixgbe_txeof(txr); 1114 } while (loop-- && more); 1115 1116 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1117 ixgbe_start_locked(txr, ifp); 1118 1119 IXGBE_TX_UNLOCK(txr); 1120 1121 /* Reenable this interrupt */ 1122 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims);
|
1017} 1018 1019 1020/********************************************************************* 1021 * 1022 * Legacy Interrupt Service routine 1023 * 1024 **********************************************************************/ 1025 1026static void 1027ixgbe_legacy_irq(void *arg) 1028{
| 1123} 1124 1125 1126/********************************************************************* 1127 * 1128 * Legacy Interrupt Service routine 1129 * 1130 **********************************************************************/ 1131 1132static void 1133ixgbe_legacy_irq(void *arg) 1134{
|
1029 u32 reg_eicr;
| |
1030 struct adapter *adapter = arg;
| 1135 struct adapter *adapter = arg;
|
| 1136 struct ixgbe_hw *hw = &adapter->hw;
|
1031 struct tx_ring *txr = adapter->tx_rings; 1032 struct rx_ring *rxr = adapter->rx_rings;
| 1137 struct tx_ring *txr = adapter->tx_rings; 1138 struct rx_ring *rxr = adapter->rx_rings;
|
1033 struct ixgbe_hw *hw;
| 1139 u32 reg_eicr;
|
1034
| 1140
|
1035 hw = &adapter->hw; 1036 reg_eicr = IXGBE_READ_REG(&adapter->hw, IXGBE_EICR); 1037 if (reg_eicr == 0)
| 1141 1142 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1143 1144 if (reg_eicr == 0) { 1145 ixgbe_enable_intr(adapter);
|
1038 return;
| 1146 return;
|
| 1147 }
|
1039
| 1148
|
1040 if (ixgbe_rxeof(rxr, adapter->rx_process_limit) != 0)
| 1149 if (ixgbe_rxeof(rxr, adapter->rx_process_limit))
|
1041 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
| 1150 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
|
1042 if (ixgbe_txeof(txr) != 0) 1043 taskqueue_enqueue(txr->tq, &txr->tx_task);
| 1151 if (ixgbe_txeof(txr)) 1152 taskqueue_enqueue(txr->tq, &txr->tx_task);
|
1044 1045 /* Check for fan failure */ 1046 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1047 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1048 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1049 "REPLACE IMMEDIATELY!!\n");
| 1153 1154 /* Check for fan failure */ 1155 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1156 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1157 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1158 "REPLACE IMMEDIATELY!!\n");
|
1050 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, 1051 IXGBE_EICR_GPI_SDP1);
| 1159 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
|
1052 }
| 1160 }
|
| 1161
|
1053 /* Link status change */ 1054 if (reg_eicr & IXGBE_EICR_LSC) 1055 ixgbe_update_link_status(adapter); 1056
| 1162 /* Link status change */ 1163 if (reg_eicr & IXGBE_EICR_LSC) 1164 ixgbe_update_link_status(adapter); 1165
|
| 1166 ixgbe_enable_intr(adapter);
|
1057 return; 1058} 1059 1060 1061/********************************************************************* 1062 * 1063 * MSI TX Interrupt Service routine 1064 * 1065 **********************************************************************/ 1066 1067void 1068ixgbe_msix_tx(void *arg) 1069{
| 1167 return; 1168} 1169 1170 1171/********************************************************************* 1172 * 1173 * MSI TX Interrupt Service routine 1174 * 1175 **********************************************************************/ 1176 1177void 1178ixgbe_msix_tx(void *arg) 1179{
|
1070 struct tx_ring *txr = arg; 1071 struct adapter *adapter = txr->adapter; 1072 u32 loop = 0;
| 1180 struct tx_ring *txr = arg; 1181 struct adapter *adapter = txr->adapter; 1182 bool more;
|
1073
| 1183
|
1074 ++txr->tx_irq;
| |
1075 IXGBE_TX_LOCK(txr);
| 1184 IXGBE_TX_LOCK(txr);
|
1076 while (loop++ < MAX_INTR) 1077 if (ixgbe_txeof(txr) == 0) 1078 break;
| 1185 ++txr->tx_irq; 1186 more = ixgbe_txeof(txr);
|
1079 IXGBE_TX_UNLOCK(txr);
| 1187 IXGBE_TX_UNLOCK(txr);
|
1080 /* Reenable this interrupt */ 1081 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); 1082
| 1188 if (more) 1189 taskqueue_enqueue(txr->tq, &txr->tx_task); 1190 else /* Reenable this interrupt */ 1191 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims);
|
1083 return; 1084} 1085
| 1192 return; 1193} 1194
|
| 1195
|
1086/********************************************************************* 1087 *
| 1196/********************************************************************* 1197 *
|
1088 * MSI RX Interrupt Service routine
| 1198 * MSIX RX Interrupt Service routine
|
1089 * 1090 **********************************************************************/ 1091 1092static void 1093ixgbe_msix_rx(void *arg) 1094{ 1095 struct rx_ring *rxr = arg;
| 1199 * 1200 **********************************************************************/ 1201 1202static void 1203ixgbe_msix_rx(void *arg) 1204{ 1205 struct rx_ring *rxr = arg;
|
1096 struct adapter *adapter = rxr->adapter; 1097 u32 loop = 0;
| 1206 struct adapter *adapter = rxr->adapter; 1207 bool more;
|
1098 1099 ++rxr->rx_irq;
| 1208 1209 ++rxr->rx_irq;
|
1100 while (loop++ < MAX_INTR) 1101 if (ixgbe_rxeof(rxr, adapter->rx_process_limit) == 0) 1102 break; 1103 /* Reenable this interrupt */ 1104 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims);
| 1210 more = ixgbe_rxeof(rxr, -1); 1211 if (more) 1212 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1213 else 1214 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); 1215 /* Update interrupt rate */ 1216 if (ixgbe_enable_aim == TRUE) 1217 ixgbe_update_aim(rxr);
|
1105 return; 1106} 1107
| 1218 return; 1219} 1220
|
| 1221/* 1222** Routine to do adjust the RX EITR value based on traffic, 1223** its a simple three state model, but seems to help. 1224** 1225** Note that the three EITR values are tuneable using 1226** sysctl in real time. The feature can be effectively 1227** nullified by setting them equal. 1228*/ 1229#define BULK_THRESHOLD 10000 1230#define AVE_THRESHOLD 1600 1231
|
1108static void
| 1232static void
|
| 1233ixgbe_update_aim(struct rx_ring *rxr) 1234{ 1235 struct adapter *adapter = rxr->adapter; 1236 u32 olditr, newitr; 1237 1238 /* Update interrupt moderation based on traffic */ 1239 olditr = rxr->eitr_setting; 1240 newitr = olditr; 1241 1242 /* Idle, don't change setting */ 1243 if (rxr->bytes == 0) 1244 return; 1245 1246 if (olditr == ixgbe_low_latency) { 1247 if (rxr->bytes > AVE_THRESHOLD) 1248 newitr = ixgbe_ave_latency; 1249 } else if (olditr == ixgbe_ave_latency) { 1250 if (rxr->bytes < AVE_THRESHOLD) 1251 newitr = ixgbe_low_latency; 1252 else if (rxr->bytes > BULK_THRESHOLD) 1253 newitr = ixgbe_bulk_latency; 1254 } else if (olditr == ixgbe_bulk_latency) { 1255 if (rxr->bytes < BULK_THRESHOLD) 1256 newitr = ixgbe_ave_latency; 1257 } 1258 1259 if (olditr != newitr) { 1260 /* Change interrupt rate */ 1261 rxr->eitr_setting = newitr; 1262 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(rxr->me), 1263 newitr | (newitr << 16)); 1264 } 1265 1266 rxr->bytes = 0; 1267 return; 1268} 1269 1270 1271static void
|
1109ixgbe_msix_link(void *arg) 1110{ 1111 struct adapter *adapter = arg; 1112 struct ixgbe_hw *hw = &adapter->hw; 1113 u32 reg_eicr; 1114 1115 ++adapter->link_irq; 1116 1117 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1118 1119 if (reg_eicr & IXGBE_EICR_LSC) 1120 ixgbe_update_link_status(adapter); 1121 1122 /* Check for fan failure */ 1123 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1124 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1125 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1126 "REPLACE IMMEDIATELY!!\n"); 1127 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1128 } 1129 1130 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1131 return; 1132} 1133 1134 1135/********************************************************************* 1136 * 1137 * Media Ioctl callback 1138 * 1139 * This routine is called whenever the user queries the status of 1140 * the interface using ifconfig. 1141 * 1142 **********************************************************************/ 1143static void 1144ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1145{ 1146 struct adapter *adapter = ifp->if_softc; 1147 1148 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1149 IXGBE_CORE_LOCK(adapter); 1150 ixgbe_update_link_status(adapter); 1151 1152 ifmr->ifm_status = IFM_AVALID; 1153 ifmr->ifm_active = IFM_ETHER; 1154 1155 if (!adapter->link_active) { 1156 IXGBE_CORE_UNLOCK(adapter); 1157 return; 1158 } 1159 1160 ifmr->ifm_status |= IFM_ACTIVE; 1161 1162 switch (adapter->link_speed) { 1163 case IXGBE_LINK_SPEED_1GB_FULL: 1164 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1165 break; 1166 case IXGBE_LINK_SPEED_10GB_FULL:
| 1272ixgbe_msix_link(void *arg) 1273{ 1274 struct adapter *adapter = arg; 1275 struct ixgbe_hw *hw = &adapter->hw; 1276 u32 reg_eicr; 1277 1278 ++adapter->link_irq; 1279 1280 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1281 1282 if (reg_eicr & IXGBE_EICR_LSC) 1283 ixgbe_update_link_status(adapter); 1284 1285 /* Check for fan failure */ 1286 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1287 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1288 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1289 "REPLACE IMMEDIATELY!!\n"); 1290 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1291 } 1292 1293 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1294 return; 1295} 1296 1297 1298/********************************************************************* 1299 * 1300 * Media Ioctl callback 1301 * 1302 * This routine is called whenever the user queries the status of 1303 * the interface using ifconfig. 1304 * 1305 **********************************************************************/ 1306static void 1307ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1308{ 1309 struct adapter *adapter = ifp->if_softc; 1310 1311 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1312 IXGBE_CORE_LOCK(adapter); 1313 ixgbe_update_link_status(adapter); 1314 1315 ifmr->ifm_status = IFM_AVALID; 1316 ifmr->ifm_active = IFM_ETHER; 1317 1318 if (!adapter->link_active) { 1319 IXGBE_CORE_UNLOCK(adapter); 1320 return; 1321 } 1322 1323 ifmr->ifm_status |= IFM_ACTIVE; 1324 1325 switch (adapter->link_speed) { 1326 case IXGBE_LINK_SPEED_1GB_FULL: 1327 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1328 break; 1329 case IXGBE_LINK_SPEED_10GB_FULL:
|
1167 ifmr->ifm_active |= ixgbe_optics | IFM_FDX;
| 1330 ifmr->ifm_active |= adapter->optics | IFM_FDX;
|
1168 break; 1169 } 1170 1171 IXGBE_CORE_UNLOCK(adapter); 1172 1173 return; 1174} 1175 1176/********************************************************************* 1177 * 1178 * Media Ioctl callback 1179 * 1180 * This routine is called when the user changes speed/duplex using 1181 * media/mediopt option with ifconfig. 1182 * 1183 **********************************************************************/ 1184static int 1185ixgbe_media_change(struct ifnet * ifp) 1186{ 1187 struct adapter *adapter = ifp->if_softc; 1188 struct ifmedia *ifm = &adapter->media; 1189 1190 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1191 1192 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1193 return (EINVAL); 1194 1195 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1196 case IFM_AUTO: 1197 adapter->hw.mac.autoneg = TRUE; 1198 adapter->hw.phy.autoneg_advertised = 1199 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 1200 break; 1201 default: 1202 device_printf(adapter->dev, "Only auto media type\n"); 1203 return (EINVAL); 1204 } 1205 1206 return (0); 1207} 1208 1209/********************************************************************* 1210 * 1211 * This routine maps the mbufs to tx descriptors. 1212 * WARNING: while this code is using an MQ style infrastructure, 1213 * it would NOT work as is with more than 1 queue. 1214 * 1215 * return 0 on success, positive on failure 1216 **********************************************************************/ 1217 1218static int 1219ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1220{ 1221 struct adapter *adapter = txr->adapter; 1222 u32 olinfo_status = 0, cmd_type_len = 0;
| 1331 break; 1332 } 1333 1334 IXGBE_CORE_UNLOCK(adapter); 1335 1336 return; 1337} 1338 1339/********************************************************************* 1340 * 1341 * Media Ioctl callback 1342 * 1343 * This routine is called when the user changes speed/duplex using 1344 * media/mediopt option with ifconfig. 1345 * 1346 **********************************************************************/ 1347static int 1348ixgbe_media_change(struct ifnet * ifp) 1349{ 1350 struct adapter *adapter = ifp->if_softc; 1351 struct ifmedia *ifm = &adapter->media; 1352 1353 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1354 1355 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1356 return (EINVAL); 1357 1358 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1359 case IFM_AUTO: 1360 adapter->hw.mac.autoneg = TRUE; 1361 adapter->hw.phy.autoneg_advertised = 1362 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 1363 break; 1364 default: 1365 device_printf(adapter->dev, "Only auto media type\n"); 1366 return (EINVAL); 1367 } 1368 1369 return (0); 1370} 1371 1372/********************************************************************* 1373 * 1374 * This routine maps the mbufs to tx descriptors. 1375 * WARNING: while this code is using an MQ style infrastructure, 1376 * it would NOT work as is with more than 1 queue. 1377 * 1378 * return 0 on success, positive on failure 1379 **********************************************************************/ 1380 1381static int 1382ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1383{ 1384 struct adapter *adapter = txr->adapter; 1385 u32 olinfo_status = 0, cmd_type_len = 0;
|
1223 u32 paylen;
| 1386 u32 paylen = 0;
|
1224 int i, j, error, nsegs; 1225 int first, last = 0; 1226 struct mbuf *m_head; 1227 bus_dma_segment_t segs[IXGBE_MAX_SCATTER]; 1228 bus_dmamap_t map; 1229 struct ixgbe_tx_buf *txbuf, *txbuf_mapped; 1230 union ixgbe_adv_tx_desc *txd = NULL; 1231 1232 m_head = *m_headp;
| 1387 int i, j, error, nsegs; 1388 int first, last = 0; 1389 struct mbuf *m_head; 1390 bus_dma_segment_t segs[IXGBE_MAX_SCATTER]; 1391 bus_dmamap_t map; 1392 struct ixgbe_tx_buf *txbuf, *txbuf_mapped; 1393 union ixgbe_adv_tx_desc *txd = NULL; 1394 1395 m_head = *m_headp;
|
1233 paylen = 0;
| |
1234 1235 /* Basic descriptor defines */ 1236 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 1237 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 1238 1239 if (m_head->m_flags & M_VLANTAG) 1240 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1241 1242 /* 1243 * Force a cleanup if number of TX descriptors 1244 * available is below the threshold. If it fails 1245 * to get above, then abort transmit. 1246 */ 1247 if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { 1248 ixgbe_txeof(txr); 1249 /* Make sure things have improved */ 1250 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) { 1251 txr->no_tx_desc_avail++; 1252 return (ENOBUFS); 1253 } 1254 } 1255 1256 /* 1257 * Important to capture the first descriptor 1258 * used because it will contain the index of 1259 * the one we tell the hardware to report back 1260 */ 1261 first = txr->next_avail_tx_desc; 1262 txbuf = &txr->tx_buffers[first]; 1263 txbuf_mapped = txbuf; 1264 map = txbuf->map; 1265 1266 /* 1267 * Map the packet for DMA. 1268 */ 1269 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1270 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1271 1272 if (error == EFBIG) { 1273 struct mbuf *m; 1274 1275 m = m_defrag(*m_headp, M_DONTWAIT); 1276 if (m == NULL) {
| 1396 1397 /* Basic descriptor defines */ 1398 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 1399 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 1400 1401 if (m_head->m_flags & M_VLANTAG) 1402 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1403 1404 /* 1405 * Force a cleanup if number of TX descriptors 1406 * available is below the threshold. If it fails 1407 * to get above, then abort transmit. 1408 */ 1409 if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { 1410 ixgbe_txeof(txr); 1411 /* Make sure things have improved */ 1412 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) { 1413 txr->no_tx_desc_avail++; 1414 return (ENOBUFS); 1415 } 1416 } 1417 1418 /* 1419 * Important to capture the first descriptor 1420 * used because it will contain the index of 1421 * the one we tell the hardware to report back 1422 */ 1423 first = txr->next_avail_tx_desc; 1424 txbuf = &txr->tx_buffers[first]; 1425 txbuf_mapped = txbuf; 1426 map = txbuf->map; 1427 1428 /* 1429 * Map the packet for DMA. 1430 */ 1431 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1432 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1433 1434 if (error == EFBIG) { 1435 struct mbuf *m; 1436 1437 m = m_defrag(*m_headp, M_DONTWAIT); 1438 if (m == NULL) {
|
1277 adapter->mbuf_alloc_failed++;
| 1439 adapter->mbuf_defrag_failed++;
|
1278 m_freem(*m_headp); 1279 *m_headp = NULL; 1280 return (ENOBUFS); 1281 } 1282 *m_headp = m; 1283 1284 /* Try it again */ 1285 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1286 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1287 1288 if (error == ENOMEM) { 1289 adapter->no_tx_dma_setup++; 1290 return (error); 1291 } else if (error != 0) { 1292 adapter->no_tx_dma_setup++; 1293 m_freem(*m_headp); 1294 *m_headp = NULL; 1295 return (error); 1296 } 1297 } else if (error == ENOMEM) { 1298 adapter->no_tx_dma_setup++; 1299 return (error); 1300 } else if (error != 0) { 1301 adapter->no_tx_dma_setup++; 1302 m_freem(*m_headp); 1303 *m_headp = NULL; 1304 return (error); 1305 } 1306 1307 /* Make certain there are enough descriptors */ 1308 if (nsegs > txr->tx_avail - 2) { 1309 txr->no_tx_desc_avail++; 1310 error = ENOBUFS; 1311 goto xmit_fail; 1312 } 1313 m_head = *m_headp; 1314 1315 /* 1316 ** Set the appropriate offload context 1317 ** this becomes the first descriptor of 1318 ** a packet. 1319 */ 1320 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 1321 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1322 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1323 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1324 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1325 ++adapter->tso_tx; 1326 } else if (ixgbe_tx_ctx_setup(txr, m_head)) 1327 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1328
| 1440 m_freem(*m_headp); 1441 *m_headp = NULL; 1442 return (ENOBUFS); 1443 } 1444 *m_headp = m; 1445 1446 /* Try it again */ 1447 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1448 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1449 1450 if (error == ENOMEM) { 1451 adapter->no_tx_dma_setup++; 1452 return (error); 1453 } else if (error != 0) { 1454 adapter->no_tx_dma_setup++; 1455 m_freem(*m_headp); 1456 *m_headp = NULL; 1457 return (error); 1458 } 1459 } else if (error == ENOMEM) { 1460 adapter->no_tx_dma_setup++; 1461 return (error); 1462 } else if (error != 0) { 1463 adapter->no_tx_dma_setup++; 1464 m_freem(*m_headp); 1465 *m_headp = NULL; 1466 return (error); 1467 } 1468 1469 /* Make certain there are enough descriptors */ 1470 if (nsegs > txr->tx_avail - 2) { 1471 txr->no_tx_desc_avail++; 1472 error = ENOBUFS; 1473 goto xmit_fail; 1474 } 1475 m_head = *m_headp; 1476 1477 /* 1478 ** Set the appropriate offload context 1479 ** this becomes the first descriptor of 1480 ** a packet. 1481 */ 1482 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 1483 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1484 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1485 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1486 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1487 ++adapter->tso_tx; 1488 } else if (ixgbe_tx_ctx_setup(txr, m_head)) 1489 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1490
|
| 1491 /* Record payload length */ 1492 if (paylen == 0) 1493 olinfo_status |= m_head->m_pkthdr.len << 1494 IXGBE_ADVTXD_PAYLEN_SHIFT; 1495
|
1329 i = txr->next_avail_tx_desc; 1330 for (j = 0; j < nsegs; j++) { 1331 bus_size_t seglen; 1332 bus_addr_t segaddr; 1333 1334 txbuf = &txr->tx_buffers[i]; 1335 txd = &txr->tx_base[i]; 1336 seglen = segs[j].ds_len; 1337 segaddr = htole64(segs[j].ds_addr); 1338 1339 txd->read.buffer_addr = segaddr; 1340 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1341 cmd_type_len |seglen); 1342 txd->read.olinfo_status = htole32(olinfo_status); 1343 last = i; /* Next descriptor that will get completed */ 1344 1345 if (++i == adapter->num_tx_desc) 1346 i = 0; 1347 1348 txbuf->m_head = NULL;
| 1496 i = txr->next_avail_tx_desc; 1497 for (j = 0; j < nsegs; j++) { 1498 bus_size_t seglen; 1499 bus_addr_t segaddr; 1500 1501 txbuf = &txr->tx_buffers[i]; 1502 txd = &txr->tx_base[i]; 1503 seglen = segs[j].ds_len; 1504 segaddr = htole64(segs[j].ds_addr); 1505 1506 txd->read.buffer_addr = segaddr; 1507 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1508 cmd_type_len |seglen); 1509 txd->read.olinfo_status = htole32(olinfo_status); 1510 last = i; /* Next descriptor that will get completed */ 1511 1512 if (++i == adapter->num_tx_desc) 1513 i = 0; 1514 1515 txbuf->m_head = NULL;
|
1349 /* 1350 ** we have to do this inside the loop right now 1351 ** because of the hardware workaround. 1352 */ 1353 if (j == (nsegs -1)) /* Last descriptor gets EOP and RS */ 1354 txd->read.cmd_type_len |= 1355 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1356#ifndef NO_82598_A0_SUPPORT 1357 if (adapter->hw.revision_id == 0) 1358 desc_flip(txd); 1359#endif
| |
1360 } 1361
| 1516 } 1517
|
| 1518 txd->read.cmd_type_len |= 1519 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
|
1362 txr->tx_avail -= nsegs; 1363 txr->next_avail_tx_desc = i; 1364 1365 txbuf->m_head = m_head; 1366 txbuf->map = map; 1367 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1368 1369 /* Set the index of the descriptor that will be marked done */ 1370 txbuf = &txr->tx_buffers[first]; 1371 1372 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1373 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1374 /* 1375 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1376 * hardware that this frame is available to transmit. 1377 */
| 1520 txr->tx_avail -= nsegs; 1521 txr->next_avail_tx_desc = i; 1522 1523 txbuf->m_head = m_head; 1524 txbuf->map = map; 1525 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1526 1527 /* Set the index of the descriptor that will be marked done */ 1528 txbuf = &txr->tx_buffers[first]; 1529 1530 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1531 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1532 /* 1533 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1534 * hardware that this frame is available to transmit. 1535 */
|
| 1536 ++txr->total_packets;
|
1378 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
| 1537 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
|
1379 ++txr->tx_packets;
| |
1380 return (0); 1381 1382xmit_fail: 1383 bus_dmamap_unload(txr->txtag, txbuf->map); 1384 return (error); 1385 1386} 1387 1388static void 1389ixgbe_set_promisc(struct adapter *adapter) 1390{ 1391 1392 u_int32_t reg_rctl; 1393 struct ifnet *ifp = adapter->ifp; 1394 1395 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1396 1397 if (ifp->if_flags & IFF_PROMISC) { 1398 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1399 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1400 } else if (ifp->if_flags & IFF_ALLMULTI) { 1401 reg_rctl |= IXGBE_FCTRL_MPE; 1402 reg_rctl &= ~IXGBE_FCTRL_UPE; 1403 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1404 } 1405 return; 1406} 1407 1408static void 1409ixgbe_disable_promisc(struct adapter * adapter) 1410{ 1411 u_int32_t reg_rctl; 1412 1413 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1414 1415 reg_rctl &= (~IXGBE_FCTRL_UPE); 1416 reg_rctl &= (~IXGBE_FCTRL_MPE); 1417 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1418 1419 return; 1420} 1421 1422 1423/********************************************************************* 1424 * Multicast Update 1425 * 1426 * This routine is called whenever multicast address list is updated. 1427 * 1428 **********************************************************************/ 1429#define IXGBE_RAR_ENTRIES 16 1430 1431static void 1432ixgbe_set_multi(struct adapter *adapter) 1433{ 1434 u32 fctrl; 1435 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1436 u8 *update_ptr; 1437 struct ifmultiaddr *ifma; 1438 int mcnt = 0; 1439 struct ifnet *ifp = adapter->ifp; 1440 1441 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1442 1443 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1444 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1445 if (ifp->if_flags & IFF_PROMISC) 1446 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1447 else if (ifp->if_flags & IFF_ALLMULTI) { 1448 fctrl |= IXGBE_FCTRL_MPE; 1449 fctrl &= ~IXGBE_FCTRL_UPE; 1450 } else 1451 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1452 1453 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1454 1455 IF_ADDR_LOCK(ifp); 1456 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1457 if (ifma->ifma_addr->sa_family != AF_LINK) 1458 continue; 1459 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1460 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1461 IXGBE_ETH_LENGTH_OF_ADDRESS); 1462 mcnt++; 1463 } 1464 IF_ADDR_UNLOCK(ifp); 1465 1466 update_ptr = mta; 1467 ixgbe_update_mc_addr_list(&adapter->hw, 1468 update_ptr, mcnt, ixgbe_mc_array_itr); 1469 1470 return; 1471} 1472 1473/* 1474 * This is an iterator function now needed by the multicast 1475 * shared code. It simply feeds the shared code routine the 1476 * addresses in the array of ixgbe_set_multi() one by one. 1477 */ 1478static u8 * 1479ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1480{ 1481 u8 *addr = *update_ptr; 1482 u8 *newptr; 1483 *vmdq = 0; 1484 1485 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1486 *update_ptr = newptr; 1487 return addr; 1488} 1489 1490 1491/********************************************************************* 1492 * Timer routine 1493 * 1494 * This routine checks for link status,updates statistics, 1495 * and runs the watchdog timer. 1496 * 1497 **********************************************************************/ 1498 1499static void 1500ixgbe_local_timer(void *arg) 1501{ 1502 struct adapter *adapter = arg; 1503 struct ifnet *ifp = adapter->ifp; 1504 1505 mtx_assert(&adapter->core_mtx, MA_OWNED); 1506
| 1538 return (0); 1539 1540xmit_fail: 1541 bus_dmamap_unload(txr->txtag, txbuf->map); 1542 return (error); 1543 1544} 1545 1546static void 1547ixgbe_set_promisc(struct adapter *adapter) 1548{ 1549 1550 u_int32_t reg_rctl; 1551 struct ifnet *ifp = adapter->ifp; 1552 1553 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1554 1555 if (ifp->if_flags & IFF_PROMISC) { 1556 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1557 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1558 } else if (ifp->if_flags & IFF_ALLMULTI) { 1559 reg_rctl |= IXGBE_FCTRL_MPE; 1560 reg_rctl &= ~IXGBE_FCTRL_UPE; 1561 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1562 } 1563 return; 1564} 1565 1566static void 1567ixgbe_disable_promisc(struct adapter * adapter) 1568{ 1569 u_int32_t reg_rctl; 1570 1571 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1572 1573 reg_rctl &= (~IXGBE_FCTRL_UPE); 1574 reg_rctl &= (~IXGBE_FCTRL_MPE); 1575 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1576 1577 return; 1578} 1579 1580 1581/********************************************************************* 1582 * Multicast Update 1583 * 1584 * This routine is called whenever multicast address list is updated. 1585 * 1586 **********************************************************************/ 1587#define IXGBE_RAR_ENTRIES 16 1588 1589static void 1590ixgbe_set_multi(struct adapter *adapter) 1591{ 1592 u32 fctrl; 1593 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1594 u8 *update_ptr; 1595 struct ifmultiaddr *ifma; 1596 int mcnt = 0; 1597 struct ifnet *ifp = adapter->ifp; 1598 1599 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1600 1601 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1602 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1603 if (ifp->if_flags & IFF_PROMISC) 1604 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1605 else if (ifp->if_flags & IFF_ALLMULTI) { 1606 fctrl |= IXGBE_FCTRL_MPE; 1607 fctrl &= ~IXGBE_FCTRL_UPE; 1608 } else 1609 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1610 1611 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1612 1613 IF_ADDR_LOCK(ifp); 1614 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1615 if (ifma->ifma_addr->sa_family != AF_LINK) 1616 continue; 1617 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1618 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1619 IXGBE_ETH_LENGTH_OF_ADDRESS); 1620 mcnt++; 1621 } 1622 IF_ADDR_UNLOCK(ifp); 1623 1624 update_ptr = mta; 1625 ixgbe_update_mc_addr_list(&adapter->hw, 1626 update_ptr, mcnt, ixgbe_mc_array_itr); 1627 1628 return; 1629} 1630 1631/* 1632 * This is an iterator function now needed by the multicast 1633 * shared code. It simply feeds the shared code routine the 1634 * addresses in the array of ixgbe_set_multi() one by one. 1635 */ 1636static u8 * 1637ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1638{ 1639 u8 *addr = *update_ptr; 1640 u8 *newptr; 1641 *vmdq = 0; 1642 1643 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1644 *update_ptr = newptr; 1645 return addr; 1646} 1647 1648 1649/********************************************************************* 1650 * Timer routine 1651 * 1652 * This routine checks for link status,updates statistics, 1653 * and runs the watchdog timer. 1654 * 1655 **********************************************************************/ 1656 1657static void 1658ixgbe_local_timer(void *arg) 1659{ 1660 struct adapter *adapter = arg; 1661 struct ifnet *ifp = adapter->ifp; 1662 1663 mtx_assert(&adapter->core_mtx, MA_OWNED); 1664
|
| 1665 /* Check for pluggable optics */ 1666 if (adapter->sfp_probe) 1667 if (!ixgbe_sfp_probe(adapter)) 1668 goto out; /* Nothing to do */ 1669
|
1507 ixgbe_update_link_status(adapter); 1508 ixgbe_update_stats_counters(adapter); 1509 if (ixgbe_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) { 1510 ixgbe_print_hw_stats(adapter); 1511 } 1512 /*
| 1670 ixgbe_update_link_status(adapter); 1671 ixgbe_update_stats_counters(adapter); 1672 if (ixgbe_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) { 1673 ixgbe_print_hw_stats(adapter); 1674 } 1675 /*
|
1513 * Each second we check the watchdog
| 1676 * Each tick we check the watchdog
|
1514 * to protect against hardware hangs. 1515 */ 1516 ixgbe_watchdog(adapter); 1517
| 1677 * to protect against hardware hangs. 1678 */ 1679 ixgbe_watchdog(adapter); 1680
|
| 1681out: 1682 /* Trigger an RX interrupt on all queues */ 1683 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, adapter->rx_mask); 1684
|
1518 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1519} 1520 1521static void 1522ixgbe_update_link_status(struct adapter *adapter) 1523{ 1524 boolean_t link_up = FALSE; 1525 struct ifnet *ifp = adapter->ifp; 1526 struct tx_ring *txr = adapter->tx_rings; 1527 device_t dev = adapter->dev; 1528 1529 ixgbe_check_link(&adapter->hw, &adapter->link_speed, &link_up, 0); 1530 1531 if (link_up){ 1532 if (adapter->link_active == FALSE) { 1533 if (bootverbose) 1534 device_printf(dev,"Link is up %d Gbps %s \n", 1535 ((adapter->link_speed == 128)? 10:1), 1536 "Full Duplex"); 1537 adapter->link_active = TRUE; 1538 if_link_state_change(ifp, LINK_STATE_UP); 1539 } 1540 } else { /* Link down */ 1541 if (adapter->link_active == TRUE) { 1542 if (bootverbose) 1543 device_printf(dev,"Link is Down\n"); 1544 if_link_state_change(ifp, LINK_STATE_DOWN); 1545 adapter->link_active = FALSE; 1546 for (int i = 0; i < adapter->num_tx_queues; 1547 i++, txr++) 1548 txr->watchdog_timer = FALSE; 1549 } 1550 } 1551 1552 return; 1553} 1554 1555 1556 1557/********************************************************************* 1558 * 1559 * This routine disables all traffic on the adapter by issuing a 1560 * global reset on the MAC and deallocates TX/RX buffers. 1561 * 1562 **********************************************************************/ 1563 1564static void 1565ixgbe_stop(void *arg) 1566{ 1567 struct ifnet *ifp; 1568 struct adapter *adapter = arg; 1569 ifp = adapter->ifp; 1570 1571 mtx_assert(&adapter->core_mtx, MA_OWNED); 1572 1573 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1574 ixgbe_disable_intr(adapter); 1575 1576 /* Tell the stack that the interface is no longer active */ 1577 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1578 1579 ixgbe_reset_hw(&adapter->hw); 1580 adapter->hw.adapter_stopped = FALSE; 1581 ixgbe_stop_adapter(&adapter->hw); 1582 callout_stop(&adapter->timer); 1583 1584 /* reprogram the RAR[0] in case user changed it. */ 1585 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1586 1587 return; 1588} 1589 1590 1591/********************************************************************* 1592 * 1593 * Determine hardware revision. 1594 * 1595 **********************************************************************/ 1596static void 1597ixgbe_identify_hardware(struct adapter *adapter) 1598{ 1599 device_t dev = adapter->dev; 1600 1601 /* Save off the information about this board */ 1602 adapter->hw.vendor_id = pci_get_vendor(dev); 1603 adapter->hw.device_id = pci_get_device(dev); 1604 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1605 adapter->hw.subsystem_vendor_id = 1606 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1607 adapter->hw.subsystem_device_id = 1608 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1609 1610 return; 1611} 1612 1613/********************************************************************* 1614 * 1615 * Setup the Legacy or MSI Interrupt handler 1616 * 1617 **********************************************************************/ 1618static int 1619ixgbe_allocate_legacy(struct adapter *adapter) 1620{ 1621 device_t dev = adapter->dev; 1622 struct tx_ring *txr = adapter->tx_rings; 1623 struct rx_ring *rxr = adapter->rx_rings; 1624 int error; 1625 1626 /* Legacy RID at 0 */ 1627 if (adapter->msix == 0) 1628 adapter->rid[0] = 0; 1629 1630 /* We allocate a single interrupt resource */ 1631 adapter->res[0] = bus_alloc_resource_any(dev, 1632 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 1633 if (adapter->res[0] == NULL) { 1634 device_printf(dev, "Unable to allocate bus resource: " 1635 "interrupt\n"); 1636 return (ENXIO); 1637 } 1638 1639 /* 1640 * Try allocating a fast interrupt and the associated deferred 1641 * processing contexts. 1642 */ 1643 TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr); 1644 TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr); 1645 txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT, 1646 taskqueue_thread_enqueue, &txr->tq); 1647 rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT, 1648 taskqueue_thread_enqueue, &rxr->tq); 1649 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 1650 device_get_nameunit(adapter->dev)); 1651 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 1652 device_get_nameunit(adapter->dev)); 1653 if ((error = bus_setup_intr(dev, adapter->res[0], 1654 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq, 1655 adapter, &adapter->tag[0])) != 0) { 1656 device_printf(dev, "Failed to register fast interrupt " 1657 "handler: %d\n", error); 1658 taskqueue_free(txr->tq); 1659 taskqueue_free(rxr->tq); 1660 txr->tq = NULL; 1661 rxr->tq = NULL; 1662 return (error); 1663 } 1664 1665 return (0); 1666} 1667 1668 1669/********************************************************************* 1670 * 1671 * Setup MSIX Interrupt resources and handlers 1672 * 1673 **********************************************************************/ 1674static int 1675ixgbe_allocate_msix(struct adapter *adapter) 1676{ 1677 device_t dev = adapter->dev; 1678 struct tx_ring *txr = adapter->tx_rings; 1679 struct rx_ring *rxr = adapter->rx_rings; 1680 int error, vector = 0; 1681 1682 /* TX setup: the code is here for multi tx, 1683 there are other parts of the driver not ready for it */ 1684 for (int i = 0; i < adapter->num_tx_queues; i++, vector++, txr++) { 1685 adapter->res[vector] = bus_alloc_resource_any(dev, 1686 SYS_RES_IRQ, &adapter->rid[vector], 1687 RF_SHAREABLE | RF_ACTIVE); 1688 if (!adapter->res[vector]) { 1689 device_printf(dev,"Unable to allocate" 1690 " bus resource: tx interrupt [%d]\n", vector); 1691 return (ENXIO); 1692 } 1693 /* Set the handler function */ 1694 error = bus_setup_intr(dev, adapter->res[vector], 1695 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1696 ixgbe_msix_tx, txr, &adapter->tag[vector]); 1697 if (error) { 1698 adapter->res[vector] = NULL; 1699 device_printf(dev, "Failed to register TX handler"); 1700 return (error); 1701 } 1702 txr->msix = vector; 1703 txr->eims = IXGBE_IVAR_TX_QUEUE(vector);
| 1685 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1686} 1687 1688static void 1689ixgbe_update_link_status(struct adapter *adapter) 1690{ 1691 boolean_t link_up = FALSE; 1692 struct ifnet *ifp = adapter->ifp; 1693 struct tx_ring *txr = adapter->tx_rings; 1694 device_t dev = adapter->dev; 1695 1696 ixgbe_check_link(&adapter->hw, &adapter->link_speed, &link_up, 0); 1697 1698 if (link_up){ 1699 if (adapter->link_active == FALSE) { 1700 if (bootverbose) 1701 device_printf(dev,"Link is up %d Gbps %s \n", 1702 ((adapter->link_speed == 128)? 10:1), 1703 "Full Duplex"); 1704 adapter->link_active = TRUE; 1705 if_link_state_change(ifp, LINK_STATE_UP); 1706 } 1707 } else { /* Link down */ 1708 if (adapter->link_active == TRUE) { 1709 if (bootverbose) 1710 device_printf(dev,"Link is Down\n"); 1711 if_link_state_change(ifp, LINK_STATE_DOWN); 1712 adapter->link_active = FALSE; 1713 for (int i = 0; i < adapter->num_tx_queues; 1714 i++, txr++) 1715 txr->watchdog_timer = FALSE; 1716 } 1717 } 1718 1719 return; 1720} 1721 1722 1723 1724/********************************************************************* 1725 * 1726 * This routine disables all traffic on the adapter by issuing a 1727 * global reset on the MAC and deallocates TX/RX buffers. 1728 * 1729 **********************************************************************/ 1730 1731static void 1732ixgbe_stop(void *arg) 1733{ 1734 struct ifnet *ifp; 1735 struct adapter *adapter = arg; 1736 ifp = adapter->ifp; 1737 1738 mtx_assert(&adapter->core_mtx, MA_OWNED); 1739 1740 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1741 ixgbe_disable_intr(adapter); 1742 1743 /* Tell the stack that the interface is no longer active */ 1744 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1745 1746 ixgbe_reset_hw(&adapter->hw); 1747 adapter->hw.adapter_stopped = FALSE; 1748 ixgbe_stop_adapter(&adapter->hw); 1749 callout_stop(&adapter->timer); 1750 1751 /* reprogram the RAR[0] in case user changed it. */ 1752 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1753 1754 return; 1755} 1756 1757 1758/********************************************************************* 1759 * 1760 * Determine hardware revision. 1761 * 1762 **********************************************************************/ 1763static void 1764ixgbe_identify_hardware(struct adapter *adapter) 1765{ 1766 device_t dev = adapter->dev; 1767 1768 /* Save off the information about this board */ 1769 adapter->hw.vendor_id = pci_get_vendor(dev); 1770 adapter->hw.device_id = pci_get_device(dev); 1771 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1772 adapter->hw.subsystem_vendor_id = 1773 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1774 adapter->hw.subsystem_device_id = 1775 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1776 1777 return; 1778} 1779 1780/********************************************************************* 1781 * 1782 * Setup the Legacy or MSI Interrupt handler 1783 * 1784 **********************************************************************/ 1785static int 1786ixgbe_allocate_legacy(struct adapter *adapter) 1787{ 1788 device_t dev = adapter->dev; 1789 struct tx_ring *txr = adapter->tx_rings; 1790 struct rx_ring *rxr = adapter->rx_rings; 1791 int error; 1792 1793 /* Legacy RID at 0 */ 1794 if (adapter->msix == 0) 1795 adapter->rid[0] = 0; 1796 1797 /* We allocate a single interrupt resource */ 1798 adapter->res[0] = bus_alloc_resource_any(dev, 1799 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 1800 if (adapter->res[0] == NULL) { 1801 device_printf(dev, "Unable to allocate bus resource: " 1802 "interrupt\n"); 1803 return (ENXIO); 1804 } 1805 1806 /* 1807 * Try allocating a fast interrupt and the associated deferred 1808 * processing contexts. 1809 */ 1810 TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr); 1811 TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr); 1812 txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT, 1813 taskqueue_thread_enqueue, &txr->tq); 1814 rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT, 1815 taskqueue_thread_enqueue, &rxr->tq); 1816 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 1817 device_get_nameunit(adapter->dev)); 1818 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 1819 device_get_nameunit(adapter->dev)); 1820 if ((error = bus_setup_intr(dev, adapter->res[0], 1821 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq, 1822 adapter, &adapter->tag[0])) != 0) { 1823 device_printf(dev, "Failed to register fast interrupt " 1824 "handler: %d\n", error); 1825 taskqueue_free(txr->tq); 1826 taskqueue_free(rxr->tq); 1827 txr->tq = NULL; 1828 rxr->tq = NULL; 1829 return (error); 1830 } 1831 1832 return (0); 1833} 1834 1835 1836/********************************************************************* 1837 * 1838 * Setup MSIX Interrupt resources and handlers 1839 * 1840 **********************************************************************/ 1841static int 1842ixgbe_allocate_msix(struct adapter *adapter) 1843{ 1844 device_t dev = adapter->dev; 1845 struct tx_ring *txr = adapter->tx_rings; 1846 struct rx_ring *rxr = adapter->rx_rings; 1847 int error, vector = 0; 1848 1849 /* TX setup: the code is here for multi tx, 1850 there are other parts of the driver not ready for it */ 1851 for (int i = 0; i < adapter->num_tx_queues; i++, vector++, txr++) { 1852 adapter->res[vector] = bus_alloc_resource_any(dev, 1853 SYS_RES_IRQ, &adapter->rid[vector], 1854 RF_SHAREABLE | RF_ACTIVE); 1855 if (!adapter->res[vector]) { 1856 device_printf(dev,"Unable to allocate" 1857 " bus resource: tx interrupt [%d]\n", vector); 1858 return (ENXIO); 1859 } 1860 /* Set the handler function */ 1861 error = bus_setup_intr(dev, adapter->res[vector], 1862 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1863 ixgbe_msix_tx, txr, &adapter->tag[vector]); 1864 if (error) { 1865 adapter->res[vector] = NULL; 1866 device_printf(dev, "Failed to register TX handler"); 1867 return (error); 1868 } 1869 txr->msix = vector; 1870 txr->eims = IXGBE_IVAR_TX_QUEUE(vector);
|
| 1871 TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr); 1872 txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT, 1873 taskqueue_thread_enqueue, &txr->tq); 1874 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 1875 device_get_nameunit(adapter->dev));
|
1704 } 1705 1706 /* RX setup */ 1707 for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rxr++) { 1708 adapter->res[vector] = bus_alloc_resource_any(dev, 1709 SYS_RES_IRQ, &adapter->rid[vector], 1710 RF_SHAREABLE | RF_ACTIVE); 1711 if (!adapter->res[vector]) { 1712 device_printf(dev,"Unable to allocate" 1713 " bus resource: rx interrupt [%d]," 1714 "rid = %d\n", i, adapter->rid[vector]); 1715 return (ENXIO); 1716 } 1717 /* Set the handler function */ 1718 error = bus_setup_intr(dev, adapter->res[vector], 1719 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_rx, 1720 rxr, &adapter->tag[vector]); 1721 if (error) { 1722 adapter->res[vector] = NULL; 1723 device_printf(dev, "Failed to register RX handler"); 1724 return (error); 1725 } 1726 rxr->msix = vector; 1727 rxr->eims = IXGBE_IVAR_RX_QUEUE(vector);
| 1876 } 1877 1878 /* RX setup */ 1879 for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rxr++) { 1880 adapter->res[vector] = bus_alloc_resource_any(dev, 1881 SYS_RES_IRQ, &adapter->rid[vector], 1882 RF_SHAREABLE | RF_ACTIVE); 1883 if (!adapter->res[vector]) { 1884 device_printf(dev,"Unable to allocate" 1885 " bus resource: rx interrupt [%d]," 1886 "rid = %d\n", i, adapter->rid[vector]); 1887 return (ENXIO); 1888 } 1889 /* Set the handler function */ 1890 error = bus_setup_intr(dev, adapter->res[vector], 1891 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_rx, 1892 rxr, &adapter->tag[vector]); 1893 if (error) { 1894 adapter->res[vector] = NULL; 1895 device_printf(dev, "Failed to register RX handler"); 1896 return (error); 1897 } 1898 rxr->msix = vector; 1899 rxr->eims = IXGBE_IVAR_RX_QUEUE(vector);
|
| 1900 /* used in local timer */ 1901 adapter->rx_mask |= rxr->eims; 1902 TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr); 1903 rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT, 1904 taskqueue_thread_enqueue, &rxr->tq); 1905 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 1906 device_get_nameunit(adapter->dev));
|
1728 } 1729 1730 /* Now for Link changes */ 1731 adapter->res[vector] = bus_alloc_resource_any(dev, 1732 SYS_RES_IRQ, &adapter->rid[vector], RF_SHAREABLE | RF_ACTIVE); 1733 if (!adapter->res[vector]) { 1734 device_printf(dev,"Unable to allocate" 1735 " bus resource: Link interrupt [%d]\n", adapter->rid[vector]); 1736 return (ENXIO); 1737 } 1738 /* Set the link handler function */ 1739 error = bus_setup_intr(dev, adapter->res[vector], 1740 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_link, 1741 adapter, &adapter->tag[vector]); 1742 if (error) { 1743 adapter->res[vector] = NULL; 1744 device_printf(dev, "Failed to register LINK handler"); 1745 return (error); 1746 } 1747 adapter->linkvec = vector; 1748 1749 return (0); 1750} 1751 1752 1753/* 1754 * Setup Either MSI/X or MSI 1755 */ 1756static int 1757ixgbe_setup_msix(struct adapter *adapter) 1758{ 1759 device_t dev = adapter->dev; 1760 int rid, want, queues, msgs; 1761
| 1907 } 1908 1909 /* Now for Link changes */ 1910 adapter->res[vector] = bus_alloc_resource_any(dev, 1911 SYS_RES_IRQ, &adapter->rid[vector], RF_SHAREABLE | RF_ACTIVE); 1912 if (!adapter->res[vector]) { 1913 device_printf(dev,"Unable to allocate" 1914 " bus resource: Link interrupt [%d]\n", adapter->rid[vector]); 1915 return (ENXIO); 1916 } 1917 /* Set the link handler function */ 1918 error = bus_setup_intr(dev, adapter->res[vector], 1919 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_link, 1920 adapter, &adapter->tag[vector]); 1921 if (error) { 1922 adapter->res[vector] = NULL; 1923 device_printf(dev, "Failed to register LINK handler"); 1924 return (error); 1925 } 1926 adapter->linkvec = vector; 1927 1928 return (0); 1929} 1930 1931 1932/* 1933 * Setup Either MSI/X or MSI 1934 */ 1935static int 1936ixgbe_setup_msix(struct adapter *adapter) 1937{ 1938 device_t dev = adapter->dev; 1939 int rid, want, queues, msgs; 1940
|
| 1941 /* Override by tuneable */ 1942 if (ixgbe_enable_msix == 0) 1943 goto msi; 1944
|
1762 /* First try MSI/X */
| 1945 /* First try MSI/X */
|
1763 rid = PCIR_BAR(IXGBE_MSIX_BAR);
| 1946 rid = PCIR_BAR(MSIX_82598_BAR);
|
1764 adapter->msix_mem = bus_alloc_resource_any(dev, 1765 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1766 if (!adapter->msix_mem) {
| 1947 adapter->msix_mem = bus_alloc_resource_any(dev, 1948 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1949 if (!adapter->msix_mem) {
|
| 1950 rid += 4; /* 82599 maps in higher BAR */ 1951 adapter->msix_mem = bus_alloc_resource_any(dev, 1952 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1953 } 1954 if (!adapter->msix_mem) {
|
1767 /* May not be enabled */ 1768 device_printf(adapter->dev, 1769 "Unable to map MSIX table \n"); 1770 goto msi; 1771 } 1772 1773 msgs = pci_msix_count(dev); 1774 if (msgs == 0) { /* system has msix disabled */ 1775 bus_release_resource(dev, SYS_RES_MEMORY,
| 1955 /* May not be enabled */ 1956 device_printf(adapter->dev, 1957 "Unable to map MSIX table \n"); 1958 goto msi; 1959 } 1960 1961 msgs = pci_msix_count(dev); 1962 if (msgs == 0) { /* system has msix disabled */ 1963 bus_release_resource(dev, SYS_RES_MEMORY,
|
1776 PCIR_BAR(IXGBE_MSIX_BAR), adapter->msix_mem);
| 1964 rid, adapter->msix_mem);
|
1777 adapter->msix_mem = NULL; 1778 goto msi; 1779 } 1780 1781 /* Figure out a reasonable auto config value */ 1782 queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus; 1783 1784 if (ixgbe_tx_queues == 0) 1785 ixgbe_tx_queues = queues; 1786 if (ixgbe_rx_queues == 0) 1787 ixgbe_rx_queues = queues; 1788 want = ixgbe_tx_queues + ixgbe_rx_queues + 1; 1789 if (msgs >= want) 1790 msgs = want; 1791 else { 1792 device_printf(adapter->dev, 1793 "MSIX Configuration Problem, " 1794 "%d vectors but %d queues wanted!\n", 1795 msgs, want); 1796 return (ENXIO); 1797 } 1798 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 1799 device_printf(adapter->dev, 1800 "Using MSIX interrupts with %d vectors\n", msgs); 1801 adapter->num_tx_queues = ixgbe_tx_queues; 1802 adapter->num_rx_queues = ixgbe_rx_queues; 1803 return (msgs); 1804 } 1805msi: 1806 msgs = pci_msi_count(dev); 1807 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 1808 device_printf(adapter->dev,"Using MSI interrupt\n"); 1809 return (msgs); 1810} 1811 1812static int 1813ixgbe_allocate_pci_resources(struct adapter *adapter) 1814{ 1815 int rid; 1816 device_t dev = adapter->dev; 1817 1818 rid = PCIR_BAR(0); 1819 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1820 &rid, RF_ACTIVE); 1821 1822 if (!(adapter->pci_mem)) { 1823 device_printf(dev,"Unable to allocate bus resource: memory\n"); 1824 return (ENXIO); 1825 } 1826 1827 adapter->osdep.mem_bus_space_tag = 1828 rman_get_bustag(adapter->pci_mem); 1829 adapter->osdep.mem_bus_space_handle = 1830 rman_get_bushandle(adapter->pci_mem); 1831 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 1832 1833 /* 1834 * Init the resource arrays 1835 */ 1836 for (int i = 0; i < IXGBE_MSGS; i++) { 1837 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 1838 adapter->tag[i] = NULL; 1839 adapter->res[i] = NULL; 1840 } 1841 1842 /* Legacy defaults */ 1843 adapter->num_tx_queues = 1; 1844 adapter->num_rx_queues = 1; 1845 1846 /* Now setup MSI or MSI/X */ 1847 adapter->msix = ixgbe_setup_msix(adapter); 1848 1849 adapter->hw.back = &adapter->osdep; 1850 return (0); 1851} 1852 1853static void 1854ixgbe_free_pci_resources(struct adapter * adapter) 1855{
| 1965 adapter->msix_mem = NULL; 1966 goto msi; 1967 } 1968 1969 /* Figure out a reasonable auto config value */ 1970 queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus; 1971 1972 if (ixgbe_tx_queues == 0) 1973 ixgbe_tx_queues = queues; 1974 if (ixgbe_rx_queues == 0) 1975 ixgbe_rx_queues = queues; 1976 want = ixgbe_tx_queues + ixgbe_rx_queues + 1; 1977 if (msgs >= want) 1978 msgs = want; 1979 else { 1980 device_printf(adapter->dev, 1981 "MSIX Configuration Problem, " 1982 "%d vectors but %d queues wanted!\n", 1983 msgs, want); 1984 return (ENXIO); 1985 } 1986 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 1987 device_printf(adapter->dev, 1988 "Using MSIX interrupts with %d vectors\n", msgs); 1989 adapter->num_tx_queues = ixgbe_tx_queues; 1990 adapter->num_rx_queues = ixgbe_rx_queues; 1991 return (msgs); 1992 } 1993msi: 1994 msgs = pci_msi_count(dev); 1995 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 1996 device_printf(adapter->dev,"Using MSI interrupt\n"); 1997 return (msgs); 1998} 1999 2000static int 2001ixgbe_allocate_pci_resources(struct adapter *adapter) 2002{ 2003 int rid; 2004 device_t dev = adapter->dev; 2005 2006 rid = PCIR_BAR(0); 2007 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2008 &rid, RF_ACTIVE); 2009 2010 if (!(adapter->pci_mem)) { 2011 device_printf(dev,"Unable to allocate bus resource: memory\n"); 2012 return (ENXIO); 2013 } 2014 2015 adapter->osdep.mem_bus_space_tag = 2016 rman_get_bustag(adapter->pci_mem); 2017 adapter->osdep.mem_bus_space_handle = 2018 rman_get_bushandle(adapter->pci_mem); 2019 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 2020 2021 /* 2022 * Init the resource arrays 2023 */ 2024 for (int i = 0; i < IXGBE_MSGS; i++) { 2025 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2026 adapter->tag[i] = NULL; 2027 adapter->res[i] = NULL; 2028 } 2029 2030 /* Legacy defaults */ 2031 adapter->num_tx_queues = 1; 2032 adapter->num_rx_queues = 1; 2033 2034 /* Now setup MSI or MSI/X */ 2035 adapter->msix = ixgbe_setup_msix(adapter); 2036 2037 adapter->hw.back = &adapter->osdep; 2038 return (0); 2039} 2040 2041static void 2042ixgbe_free_pci_resources(struct adapter * adapter) 2043{
|
1856 device_t dev = adapter->dev;
| 2044 device_t dev = adapter->dev; 2045 int rid;
|
1857 1858 /* 1859 * Legacy has this set to 0, but we need 1860 * to run this once, so reset it. 1861 */ 1862 if (adapter->msix == 0) 1863 adapter->msix = 1; 1864
| 2046 2047 /* 2048 * Legacy has this set to 0, but we need 2049 * to run this once, so reset it. 2050 */ 2051 if (adapter->msix == 0) 2052 adapter->msix = 1; 2053
|
| 2054 rid = PCIR_BAR(MSIX_82598_BAR); 2055
|
1865 /* 1866 * First release all the interrupt resources: 1867 * notice that since these are just kept 1868 * in an array we can do the same logic 1869 * whether its MSIX or just legacy. 1870 */ 1871 for (int i = 0; i < adapter->msix; i++) { 1872 if (adapter->tag[i] != NULL) { 1873 bus_teardown_intr(dev, adapter->res[i], 1874 adapter->tag[i]); 1875 adapter->tag[i] = NULL; 1876 } 1877 if (adapter->res[i] != NULL) { 1878 bus_release_resource(dev, SYS_RES_IRQ, 1879 adapter->rid[i], adapter->res[i]); 1880 } 1881 } 1882 1883 if (adapter->msix) 1884 pci_release_msi(dev); 1885 1886 if (adapter->msix_mem != NULL) 1887 bus_release_resource(dev, SYS_RES_MEMORY,
| 2056 /* 2057 * First release all the interrupt resources: 2058 * notice that since these are just kept 2059 * in an array we can do the same logic 2060 * whether its MSIX or just legacy. 2061 */ 2062 for (int i = 0; i < adapter->msix; i++) { 2063 if (adapter->tag[i] != NULL) { 2064 bus_teardown_intr(dev, adapter->res[i], 2065 adapter->tag[i]); 2066 adapter->tag[i] = NULL; 2067 } 2068 if (adapter->res[i] != NULL) { 2069 bus_release_resource(dev, SYS_RES_IRQ, 2070 adapter->rid[i], adapter->res[i]); 2071 } 2072 } 2073 2074 if (adapter->msix) 2075 pci_release_msi(dev); 2076 2077 if (adapter->msix_mem != NULL) 2078 bus_release_resource(dev, SYS_RES_MEMORY,
|
1888 PCIR_BAR(IXGBE_MSIX_BAR), adapter->msix_mem);
| 2079 rid, adapter->msix_mem);
|
1889 1890 if (adapter->pci_mem != NULL) 1891 bus_release_resource(dev, SYS_RES_MEMORY, 1892 PCIR_BAR(0), adapter->pci_mem); 1893 1894 return; 1895} 1896 1897/********************************************************************* 1898 * 1899 * Initialize the hardware to a configuration as specified by the 1900 * adapter structure. The controller is reset, the EEPROM is 1901 * verified, the MAC address is set, then the shared initialization 1902 * routines are called. 1903 * 1904 **********************************************************************/ 1905static int 1906ixgbe_hardware_init(struct adapter *adapter) 1907{ 1908 device_t dev = adapter->dev; 1909 u16 csum; 1910 1911 csum = 0; 1912 /* Issue a global reset */ 1913 adapter->hw.adapter_stopped = FALSE; 1914 ixgbe_stop_adapter(&adapter->hw); 1915 1916 /* Make sure we have a good EEPROM before we read from it */ 1917 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 1918 device_printf(dev,"The EEPROM Checksum Is Not Valid\n"); 1919 return (EIO); 1920 } 1921 1922 /* Get Hardware Flow Control setting */
| 2080 2081 if (adapter->pci_mem != NULL) 2082 bus_release_resource(dev, SYS_RES_MEMORY, 2083 PCIR_BAR(0), adapter->pci_mem); 2084 2085 return; 2086} 2087 2088/********************************************************************* 2089 * 2090 * Initialize the hardware to a configuration as specified by the 2091 * adapter structure. The controller is reset, the EEPROM is 2092 * verified, the MAC address is set, then the shared initialization 2093 * routines are called. 2094 * 2095 **********************************************************************/ 2096static int 2097ixgbe_hardware_init(struct adapter *adapter) 2098{ 2099 device_t dev = adapter->dev; 2100 u16 csum; 2101 2102 csum = 0; 2103 /* Issue a global reset */ 2104 adapter->hw.adapter_stopped = FALSE; 2105 ixgbe_stop_adapter(&adapter->hw); 2106 2107 /* Make sure we have a good EEPROM before we read from it */ 2108 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 2109 device_printf(dev,"The EEPROM Checksum Is Not Valid\n"); 2110 return (EIO); 2111 } 2112 2113 /* Get Hardware Flow Control setting */
|
1923 adapter->hw.fc.type = ixgbe_fc_full;
| 2114 adapter->hw.fc.requested_mode = ixgbe_fc_full;
|
1924 adapter->hw.fc.pause_time = IXGBE_FC_PAUSE; 1925 adapter->hw.fc.low_water = IXGBE_FC_LO; 1926 adapter->hw.fc.high_water = IXGBE_FC_HI; 1927 adapter->hw.fc.send_xon = TRUE; 1928 1929 if (ixgbe_init_hw(&adapter->hw)) { 1930 device_printf(dev,"Hardware Initialization Failed"); 1931 return (EIO); 1932 } 1933 1934 return (0); 1935} 1936 1937/********************************************************************* 1938 * 1939 * Setup networking device structure and register an interface. 1940 * 1941 **********************************************************************/ 1942static void 1943ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1944{ 1945 struct ifnet *ifp; 1946 struct ixgbe_hw *hw = &adapter->hw; 1947 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1948 1949 ifp = adapter->ifp = if_alloc(IFT_ETHER); 1950 if (ifp == NULL) 1951 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 1952 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1953 ifp->if_mtu = ETHERMTU; 1954 ifp->if_baudrate = 1000000000; 1955 ifp->if_init = ixgbe_init; 1956 ifp->if_softc = adapter; 1957 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1958 ifp->if_ioctl = ixgbe_ioctl; 1959 ifp->if_start = ixgbe_start; 1960 ifp->if_timer = 0; 1961 ifp->if_watchdog = NULL; 1962 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 1; 1963 1964 ether_ifattach(ifp, adapter->hw.mac.addr); 1965 1966 adapter->max_frame_size = 1967 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1968 1969 /* 1970 * Tell the upper layer(s) we support long frames. 1971 */ 1972 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1973 1974 ifp->if_capabilities |= (IFCAP_HWCSUM | IFCAP_TSO4); 1975 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 1976 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 1977 1978 ifp->if_capenable = ifp->if_capabilities; 1979
| 2115 adapter->hw.fc.pause_time = IXGBE_FC_PAUSE; 2116 adapter->hw.fc.low_water = IXGBE_FC_LO; 2117 adapter->hw.fc.high_water = IXGBE_FC_HI; 2118 adapter->hw.fc.send_xon = TRUE; 2119 2120 if (ixgbe_init_hw(&adapter->hw)) { 2121 device_printf(dev,"Hardware Initialization Failed"); 2122 return (EIO); 2123 } 2124 2125 return (0); 2126} 2127 2128/********************************************************************* 2129 * 2130 * Setup networking device structure and register an interface. 2131 * 2132 **********************************************************************/ 2133static void 2134ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2135{ 2136 struct ifnet *ifp; 2137 struct ixgbe_hw *hw = &adapter->hw; 2138 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2139 2140 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2141 if (ifp == NULL) 2142 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 2143 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2144 ifp->if_mtu = ETHERMTU; 2145 ifp->if_baudrate = 1000000000; 2146 ifp->if_init = ixgbe_init; 2147 ifp->if_softc = adapter; 2148 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2149 ifp->if_ioctl = ixgbe_ioctl; 2150 ifp->if_start = ixgbe_start; 2151 ifp->if_timer = 0; 2152 ifp->if_watchdog = NULL; 2153 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 1; 2154 2155 ether_ifattach(ifp, adapter->hw.mac.addr); 2156 2157 adapter->max_frame_size = 2158 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2159 2160 /* 2161 * Tell the upper layer(s) we support long frames. 2162 */ 2163 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2164 2165 ifp->if_capabilities |= (IFCAP_HWCSUM | IFCAP_TSO4); 2166 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2167 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2168 2169 ifp->if_capenable = ifp->if_capabilities; 2170
|
1980 if ((hw->device_id == IXGBE_DEV_ID_82598AT) || 1981 (hw->device_id == IXGBE_DEV_ID_82598AT_DUAL_PORT))
| 2171 if (hw->device_id == IXGBE_DEV_ID_82598AT)
|
1982 ixgbe_setup_link_speed(hw, (IXGBE_LINK_SPEED_10GB_FULL | 1983 IXGBE_LINK_SPEED_1GB_FULL), TRUE, TRUE); 1984 else 1985 ixgbe_setup_link_speed(hw, IXGBE_LINK_SPEED_10GB_FULL, 1986 TRUE, FALSE); 1987 1988 /* 1989 * Specify the media types supported by this adapter and register 1990 * callbacks to update media and link information 1991 */ 1992 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 1993 ixgbe_media_status);
| 2172 ixgbe_setup_link_speed(hw, (IXGBE_LINK_SPEED_10GB_FULL | 2173 IXGBE_LINK_SPEED_1GB_FULL), TRUE, TRUE); 2174 else 2175 ixgbe_setup_link_speed(hw, IXGBE_LINK_SPEED_10GB_FULL, 2176 TRUE, FALSE); 2177 2178 /* 2179 * Specify the media types supported by this adapter and register 2180 * callbacks to update media and link information 2181 */ 2182 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2183 ixgbe_media_status);
|
1994 ifmedia_add(&adapter->media, IFM_ETHER | ixgbe_optics |
| 2184 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics |
|
1995 IFM_FDX, 0, NULL);
| 2185 IFM_FDX, 0, NULL);
|
1996 if ((hw->device_id == IXGBE_DEV_ID_82598AT) || 1997 (hw->device_id == IXGBE_DEV_ID_82598AT_DUAL_PORT)) {
| 2186 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
|
1998 ifmedia_add(&adapter->media, 1999 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2000 ifmedia_add(&adapter->media, 2001 IFM_ETHER | IFM_1000_T, 0, NULL); 2002 } 2003 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2004 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2005 2006 return; 2007} 2008 2009/******************************************************************** 2010 * Manage DMA'able memory. 2011 *******************************************************************/ 2012static void 2013ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 2014{ 2015 if (error) 2016 return; 2017 *(bus_addr_t *) arg = segs->ds_addr; 2018 return; 2019} 2020 2021static int 2022ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size, 2023 struct ixgbe_dma_alloc *dma, int mapflags) 2024{ 2025 device_t dev = adapter->dev; 2026 int r; 2027 2028 r = bus_dma_tag_create(NULL, /* parent */ 2029 PAGE_SIZE, 0, /* alignment, bounds */ 2030 BUS_SPACE_MAXADDR, /* lowaddr */ 2031 BUS_SPACE_MAXADDR, /* highaddr */ 2032 NULL, NULL, /* filter, filterarg */ 2033 size, /* maxsize */ 2034 1, /* nsegments */ 2035 size, /* maxsegsize */ 2036 BUS_DMA_ALLOCNOW, /* flags */ 2037 NULL, /* lockfunc */ 2038 NULL, /* lockfuncarg */ 2039 &dma->dma_tag); 2040 if (r != 0) { 2041 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; " 2042 "error %u\n", r); 2043 goto fail_0; 2044 } 2045 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 2046 BUS_DMA_NOWAIT, &dma->dma_map); 2047 if (r != 0) { 2048 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2049 "error %u\n", r); 2050 goto fail_1; 2051 } 2052 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2053 size, 2054 ixgbe_dmamap_cb, 2055 &dma->dma_paddr, 2056 mapflags | BUS_DMA_NOWAIT); 2057 if (r != 0) { 2058 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; " 2059 "error %u\n", r); 2060 goto fail_2; 2061 } 2062 dma->dma_size = size; 2063 return (0); 2064fail_2: 2065 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2066fail_1: 2067 bus_dma_tag_destroy(dma->dma_tag); 2068fail_0: 2069 dma->dma_map = NULL; 2070 dma->dma_tag = NULL; 2071 return (r); 2072} 2073 2074static void 2075ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2076{ 2077 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2078 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2079 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2080 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2081 bus_dma_tag_destroy(dma->dma_tag); 2082} 2083 2084 2085/********************************************************************* 2086 * 2087 * Allocate memory for the transmit and receive rings, and then 2088 * the descriptors associated with each, called only once at attach. 2089 * 2090 **********************************************************************/ 2091static int 2092ixgbe_allocate_queues(struct adapter *adapter) 2093{ 2094 device_t dev = adapter->dev; 2095 struct tx_ring *txr; 2096 struct rx_ring *rxr; 2097 int rsize, tsize, error = IXGBE_SUCCESS;
| 2187 ifmedia_add(&adapter->media, 2188 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2189 ifmedia_add(&adapter->media, 2190 IFM_ETHER | IFM_1000_T, 0, NULL); 2191 } 2192 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2193 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2194 2195 return; 2196} 2197 2198/******************************************************************** 2199 * Manage DMA'able memory. 2200 *******************************************************************/ 2201static void 2202ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 2203{ 2204 if (error) 2205 return; 2206 *(bus_addr_t *) arg = segs->ds_addr; 2207 return; 2208} 2209 2210static int 2211ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size, 2212 struct ixgbe_dma_alloc *dma, int mapflags) 2213{ 2214 device_t dev = adapter->dev; 2215 int r; 2216 2217 r = bus_dma_tag_create(NULL, /* parent */ 2218 PAGE_SIZE, 0, /* alignment, bounds */ 2219 BUS_SPACE_MAXADDR, /* lowaddr */ 2220 BUS_SPACE_MAXADDR, /* highaddr */ 2221 NULL, NULL, /* filter, filterarg */ 2222 size, /* maxsize */ 2223 1, /* nsegments */ 2224 size, /* maxsegsize */ 2225 BUS_DMA_ALLOCNOW, /* flags */ 2226 NULL, /* lockfunc */ 2227 NULL, /* lockfuncarg */ 2228 &dma->dma_tag); 2229 if (r != 0) { 2230 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; " 2231 "error %u\n", r); 2232 goto fail_0; 2233 } 2234 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 2235 BUS_DMA_NOWAIT, &dma->dma_map); 2236 if (r != 0) { 2237 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2238 "error %u\n", r); 2239 goto fail_1; 2240 } 2241 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2242 size, 2243 ixgbe_dmamap_cb, 2244 &dma->dma_paddr, 2245 mapflags | BUS_DMA_NOWAIT); 2246 if (r != 0) { 2247 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; " 2248 "error %u\n", r); 2249 goto fail_2; 2250 } 2251 dma->dma_size = size; 2252 return (0); 2253fail_2: 2254 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2255fail_1: 2256 bus_dma_tag_destroy(dma->dma_tag); 2257fail_0: 2258 dma->dma_map = NULL; 2259 dma->dma_tag = NULL; 2260 return (r); 2261} 2262 2263static void 2264ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2265{ 2266 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2267 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2268 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2269 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2270 bus_dma_tag_destroy(dma->dma_tag); 2271} 2272 2273 2274/********************************************************************* 2275 * 2276 * Allocate memory for the transmit and receive rings, and then 2277 * the descriptors associated with each, called only once at attach. 2278 * 2279 **********************************************************************/ 2280static int 2281ixgbe_allocate_queues(struct adapter *adapter) 2282{ 2283 device_t dev = adapter->dev; 2284 struct tx_ring *txr; 2285 struct rx_ring *rxr; 2286 int rsize, tsize, error = IXGBE_SUCCESS;
|
2098 char name_string[16];
| |
2099 int txconf = 0, rxconf = 0; 2100 2101 /* First allocate the TX ring struct memory */ 2102 if (!(adapter->tx_rings = 2103 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2104 adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2105 device_printf(dev, "Unable to allocate TX ring memory\n"); 2106 error = ENOMEM; 2107 goto fail; 2108 } 2109 txr = adapter->tx_rings; 2110 2111 /* Next allocate the RX */ 2112 if (!(adapter->rx_rings = 2113 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2114 adapter->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2115 device_printf(dev, "Unable to allocate RX ring memory\n"); 2116 error = ENOMEM; 2117 goto rx_fail; 2118 } 2119 rxr = adapter->rx_rings; 2120 2121 /* For the ring itself */ 2122 tsize = roundup2(adapter->num_tx_desc * 2123 sizeof(union ixgbe_adv_tx_desc), 4096); 2124 2125 /* 2126 * Now set up the TX queues, txconf is needed to handle the 2127 * possibility that things fail midcourse and we need to 2128 * undo memory gracefully 2129 */ 2130 for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) { 2131 /* Set up some basics */ 2132 txr = &adapter->tx_rings[i]; 2133 txr->adapter = adapter; 2134 txr->me = i; 2135 2136 /* Initialize the TX side lock */
| 2287 int txconf = 0, rxconf = 0; 2288 2289 /* First allocate the TX ring struct memory */ 2290 if (!(adapter->tx_rings = 2291 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2292 adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2293 device_printf(dev, "Unable to allocate TX ring memory\n"); 2294 error = ENOMEM; 2295 goto fail; 2296 } 2297 txr = adapter->tx_rings; 2298 2299 /* Next allocate the RX */ 2300 if (!(adapter->rx_rings = 2301 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2302 adapter->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2303 device_printf(dev, "Unable to allocate RX ring memory\n"); 2304 error = ENOMEM; 2305 goto rx_fail; 2306 } 2307 rxr = adapter->rx_rings; 2308 2309 /* For the ring itself */ 2310 tsize = roundup2(adapter->num_tx_desc * 2311 sizeof(union ixgbe_adv_tx_desc), 4096); 2312 2313 /* 2314 * Now set up the TX queues, txconf is needed to handle the 2315 * possibility that things fail midcourse and we need to 2316 * undo memory gracefully 2317 */ 2318 for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) { 2319 /* Set up some basics */ 2320 txr = &adapter->tx_rings[i]; 2321 txr->adapter = adapter; 2322 txr->me = i; 2323 2324 /* Initialize the TX side lock */
|
2137 snprintf(name_string, sizeof(name_string), "%s:tx(%d)",
| 2325 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
|
2138 device_get_nameunit(dev), txr->me);
| 2326 device_get_nameunit(dev), txr->me);
|
2139 mtx_init(&txr->tx_mtx, name_string, NULL, MTX_DEF);
| 2327 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
|
2140 2141 if (ixgbe_dma_malloc(adapter, tsize, 2142 &txr->txdma, BUS_DMA_NOWAIT)) { 2143 device_printf(dev, 2144 "Unable to allocate TX Descriptor memory\n"); 2145 error = ENOMEM; 2146 goto err_tx_desc; 2147 } 2148 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2149 bzero((void *)txr->tx_base, tsize); 2150 2151 /* Now allocate transmit buffers for the ring */ 2152 if (ixgbe_allocate_transmit_buffers(txr)) { 2153 device_printf(dev, 2154 "Critical Failure setting up transmit buffers\n"); 2155 error = ENOMEM; 2156 goto err_tx_desc; 2157 } 2158 2159 } 2160 2161 /* 2162 * Next the RX queues... 2163 */ 2164 rsize = roundup2(adapter->num_rx_desc * 2165 sizeof(union ixgbe_adv_rx_desc), 4096); 2166 for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) { 2167 rxr = &adapter->rx_rings[i]; 2168 /* Set up some basics */ 2169 rxr->adapter = adapter; 2170 rxr->me = i; 2171
| 2328 2329 if (ixgbe_dma_malloc(adapter, tsize, 2330 &txr->txdma, BUS_DMA_NOWAIT)) { 2331 device_printf(dev, 2332 "Unable to allocate TX Descriptor memory\n"); 2333 error = ENOMEM; 2334 goto err_tx_desc; 2335 } 2336 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2337 bzero((void *)txr->tx_base, tsize); 2338 2339 /* Now allocate transmit buffers for the ring */ 2340 if (ixgbe_allocate_transmit_buffers(txr)) { 2341 device_printf(dev, 2342 "Critical Failure setting up transmit buffers\n"); 2343 error = ENOMEM; 2344 goto err_tx_desc; 2345 } 2346 2347 } 2348 2349 /* 2350 * Next the RX queues... 2351 */ 2352 rsize = roundup2(adapter->num_rx_desc * 2353 sizeof(union ixgbe_adv_rx_desc), 4096); 2354 for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) { 2355 rxr = &adapter->rx_rings[i]; 2356 /* Set up some basics */ 2357 rxr->adapter = adapter; 2358 rxr->me = i; 2359
|
2172 /* Initialize the TX side lock */ 2173 snprintf(name_string, sizeof(name_string), "%s:rx(%d)",
| 2360 /* Initialize the RX side lock */ 2361 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
|
2174 device_get_nameunit(dev), rxr->me);
| 2362 device_get_nameunit(dev), rxr->me);
|
2175 mtx_init(&rxr->rx_mtx, name_string, NULL, MTX_DEF);
| 2363 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
|
2176 2177 if (ixgbe_dma_malloc(adapter, rsize, 2178 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2179 device_printf(dev, 2180 "Unable to allocate RxDescriptor memory\n"); 2181 error = ENOMEM; 2182 goto err_rx_desc; 2183 } 2184 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2185 bzero((void *)rxr->rx_base, rsize); 2186 2187 /* Allocate receive buffers for the ring*/ 2188 if (ixgbe_allocate_receive_buffers(rxr)) { 2189 device_printf(dev, 2190 "Critical Failure setting up receive buffers\n"); 2191 error = ENOMEM; 2192 goto err_rx_desc; 2193 } 2194 } 2195 2196 return (0); 2197 2198err_rx_desc: 2199 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2200 ixgbe_dma_free(adapter, &rxr->rxdma); 2201err_tx_desc: 2202 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2203 ixgbe_dma_free(adapter, &txr->txdma); 2204 free(adapter->rx_rings, M_DEVBUF); 2205rx_fail: 2206 free(adapter->tx_rings, M_DEVBUF); 2207fail: 2208 return (error); 2209} 2210 2211/********************************************************************* 2212 * 2213 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2214 * the information needed to transmit a packet on the wire. This is 2215 * called only once at attach, setup is done every reset. 2216 * 2217 **********************************************************************/ 2218static int 2219ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2220{ 2221 struct adapter *adapter = txr->adapter; 2222 device_t dev = adapter->dev; 2223 struct ixgbe_tx_buf *txbuf; 2224 int error, i; 2225 2226 /* 2227 * Setup DMA descriptor areas. 2228 */ 2229 if ((error = bus_dma_tag_create(NULL, /* parent */ 2230 PAGE_SIZE, 0, /* alignment, bounds */ 2231 BUS_SPACE_MAXADDR, /* lowaddr */ 2232 BUS_SPACE_MAXADDR, /* highaddr */ 2233 NULL, NULL, /* filter, filterarg */ 2234 IXGBE_TSO_SIZE, /* maxsize */ 2235 IXGBE_MAX_SCATTER, /* nsegments */ 2236 PAGE_SIZE, /* maxsegsize */ 2237 0, /* flags */ 2238 NULL, /* lockfunc */ 2239 NULL, /* lockfuncarg */ 2240 &txr->txtag))) { 2241 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2242 goto fail; 2243 } 2244 2245 if (!(txr->tx_buffers = 2246 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 2247 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2248 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2249 error = ENOMEM; 2250 goto fail; 2251 } 2252 2253 /* Create the descriptor buffer dma maps */ 2254 txbuf = txr->tx_buffers; 2255 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2256 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2257 if (error != 0) { 2258 device_printf(dev, "Unable to create TX DMA map\n"); 2259 goto fail; 2260 } 2261 } 2262 2263 return 0; 2264fail: 2265 /* We free all, it handles case where we are in the middle */ 2266 ixgbe_free_transmit_structures(adapter); 2267 return (error); 2268} 2269 2270/********************************************************************* 2271 * 2272 * Initialize a transmit ring. 2273 * 2274 **********************************************************************/ 2275static void 2276ixgbe_setup_transmit_ring(struct tx_ring *txr) 2277{ 2278 struct adapter *adapter = txr->adapter; 2279 struct ixgbe_tx_buf *txbuf; 2280 int i; 2281 2282 /* Clear the old ring contents */ 2283 bzero((void *)txr->tx_base, 2284 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2285 /* Reset indices */ 2286 txr->next_avail_tx_desc = 0; 2287 txr->next_tx_to_clean = 0; 2288 2289 /* Free any existing tx buffers. */ 2290 txbuf = txr->tx_buffers; 2291 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2292 if (txbuf->m_head != NULL) { 2293 bus_dmamap_sync(txr->txtag, txbuf->map, 2294 BUS_DMASYNC_POSTWRITE); 2295 bus_dmamap_unload(txr->txtag, txbuf->map); 2296 m_freem(txbuf->m_head); 2297 txbuf->m_head = NULL; 2298 } 2299 } 2300 2301 /* Set number of descriptors available */ 2302 txr->tx_avail = adapter->num_tx_desc; 2303 2304 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2305 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2306 2307} 2308 2309/********************************************************************* 2310 * 2311 * Initialize all transmit rings. 2312 * 2313 **********************************************************************/ 2314static int 2315ixgbe_setup_transmit_structures(struct adapter *adapter) 2316{ 2317 struct tx_ring *txr = adapter->tx_rings; 2318 2319 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 2320 ixgbe_setup_transmit_ring(txr); 2321 2322 return (0); 2323} 2324 2325/********************************************************************* 2326 * 2327 * Enable transmit unit. 2328 * 2329 **********************************************************************/ 2330static void 2331ixgbe_initialize_transmit_units(struct adapter *adapter) 2332{ 2333 struct tx_ring *txr = adapter->tx_rings; 2334 struct ixgbe_hw *hw = &adapter->hw; 2335 2336 /* Setup the Base and Length of the Tx Descriptor Ring */ 2337 2338 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2339 u64 txhwb = 0, tdba = txr->txdma.dma_paddr; 2340 u32 txctrl; 2341 2342 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2343 (tdba & 0x00000000ffffffffULL)); 2344 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2345 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2346 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2347 2348 /* Setup for Head WriteBack */ 2349 txhwb = (u64)vtophys(&txr->tx_hwb); 2350 txhwb |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2351 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i), 2352 (txhwb & 0x00000000ffffffffULL)); 2353 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i), 2354 (txhwb >> 32)); 2355 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2356 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 2357 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2358 2359 /* Setup the HW Tx Head and Tail descriptor pointers */ 2360 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2361 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2362 2363 /* Setup Transmit Descriptor Cmd Settings */ 2364 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2365 2366 txr->watchdog_timer = 0; 2367 } 2368 2369 return; 2370} 2371 2372/********************************************************************* 2373 * 2374 * Free all transmit rings. 2375 * 2376 **********************************************************************/ 2377static void 2378ixgbe_free_transmit_structures(struct adapter *adapter) 2379{ 2380 struct tx_ring *txr = adapter->tx_rings; 2381 2382 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2383 IXGBE_TX_LOCK(txr); 2384 ixgbe_free_transmit_buffers(txr); 2385 ixgbe_dma_free(adapter, &txr->txdma); 2386 IXGBE_TX_UNLOCK(txr); 2387 IXGBE_TX_LOCK_DESTROY(txr); 2388 } 2389 free(adapter->tx_rings, M_DEVBUF); 2390} 2391 2392/********************************************************************* 2393 * 2394 * Free transmit ring related data structures. 2395 * 2396 **********************************************************************/ 2397static void 2398ixgbe_free_transmit_buffers(struct tx_ring *txr) 2399{ 2400 struct adapter *adapter = txr->adapter; 2401 struct ixgbe_tx_buf *tx_buffer; 2402 int i; 2403 2404 INIT_DEBUGOUT("free_transmit_ring: begin"); 2405 2406 if (txr->tx_buffers == NULL) 2407 return; 2408 2409 tx_buffer = txr->tx_buffers; 2410 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2411 if (tx_buffer->m_head != NULL) { 2412 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2413 BUS_DMASYNC_POSTWRITE); 2414 bus_dmamap_unload(txr->txtag, 2415 tx_buffer->map); 2416 m_freem(tx_buffer->m_head); 2417 tx_buffer->m_head = NULL; 2418 if (tx_buffer->map != NULL) { 2419 bus_dmamap_destroy(txr->txtag, 2420 tx_buffer->map); 2421 tx_buffer->map = NULL; 2422 } 2423 } else if (tx_buffer->map != NULL) { 2424 bus_dmamap_unload(txr->txtag, 2425 tx_buffer->map); 2426 bus_dmamap_destroy(txr->txtag, 2427 tx_buffer->map); 2428 tx_buffer->map = NULL; 2429 } 2430 } 2431 2432 if (txr->tx_buffers != NULL) { 2433 free(txr->tx_buffers, M_DEVBUF); 2434 txr->tx_buffers = NULL; 2435 } 2436 if (txr->txtag != NULL) { 2437 bus_dma_tag_destroy(txr->txtag); 2438 txr->txtag = NULL; 2439 } 2440 return; 2441} 2442 2443/********************************************************************* 2444 * 2445 * Advanced Context Descriptor setup for VLAN or CSUM 2446 * 2447 **********************************************************************/ 2448 2449static boolean_t 2450ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2451{ 2452 struct adapter *adapter = txr->adapter; 2453 struct ixgbe_adv_tx_context_desc *TXD; 2454 struct ixgbe_tx_buf *tx_buffer; 2455 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2456 struct ether_vlan_header *eh; 2457 struct ip *ip; 2458 struct ip6_hdr *ip6; 2459 int ehdrlen, ip_hlen = 0; 2460 u16 etype; 2461 u8 ipproto = 0; 2462 bool offload = TRUE; 2463 int ctxd = txr->next_avail_tx_desc; 2464#if __FreeBSD_version < 700000 2465 struct m_tag *mtag; 2466#else 2467 u16 vtag = 0; 2468#endif 2469 2470 2471 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2472 offload = FALSE; 2473 2474 tx_buffer = &txr->tx_buffers[ctxd]; 2475 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2476 2477 /* 2478 ** In advanced descriptors the vlan tag must 2479 ** be placed into the descriptor itself. 2480 */ 2481#if __FreeBSD_version < 700000 2482 mtag = VLAN_OUTPUT_TAG(ifp, mp); 2483 if (mtag != NULL) { 2484 vlan_macip_lens |= 2485 htole16(VLAN_TAG_VALUE(mtag)) << IXGBE_ADVTXD_VLAN_SHIFT; 2486 } else if (offload == FALSE) 2487 return FALSE; /* No need for CTX */ 2488#else 2489 if (mp->m_flags & M_VLANTAG) { 2490 vtag = htole16(mp->m_pkthdr.ether_vtag); 2491 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2492 } else if (offload == FALSE) 2493 return FALSE; 2494#endif 2495 /* 2496 * Determine where frame payload starts. 2497 * Jump over vlan headers if already present, 2498 * helpful for QinQ too. 2499 */ 2500 eh = mtod(mp, struct ether_vlan_header *); 2501 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2502 etype = ntohs(eh->evl_proto); 2503 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2504 } else { 2505 etype = ntohs(eh->evl_encap_proto); 2506 ehdrlen = ETHER_HDR_LEN; 2507 } 2508 2509 /* Set the ether header length */ 2510 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2511 2512 switch (etype) { 2513 case ETHERTYPE_IP: 2514 ip = (struct ip *)(mp->m_data + ehdrlen); 2515 ip_hlen = ip->ip_hl << 2; 2516 if (mp->m_len < ehdrlen + ip_hlen) 2517 return FALSE; /* failure */ 2518 ipproto = ip->ip_p; 2519 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2520 break; 2521 case ETHERTYPE_IPV6: 2522 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2523 ip_hlen = sizeof(struct ip6_hdr); 2524 if (mp->m_len < ehdrlen + ip_hlen) 2525 return FALSE; /* failure */ 2526 ipproto = ip6->ip6_nxt; 2527 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2528 break; 2529 default: 2530 offload = FALSE; 2531 break; 2532 } 2533 2534 vlan_macip_lens |= ip_hlen; 2535 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2536 2537 switch (ipproto) { 2538 case IPPROTO_TCP: 2539 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2540 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2541 break; 2542 case IPPROTO_UDP: 2543 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2544 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2545 break; 2546 default: 2547 offload = FALSE; 2548 break; 2549 } 2550 2551 /* Now copy bits into descriptor */ 2552 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2553 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2554 TXD->seqnum_seed = htole32(0); 2555 TXD->mss_l4len_idx = htole32(0); 2556
| 2364 2365 if (ixgbe_dma_malloc(adapter, rsize, 2366 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2367 device_printf(dev, 2368 "Unable to allocate RxDescriptor memory\n"); 2369 error = ENOMEM; 2370 goto err_rx_desc; 2371 } 2372 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2373 bzero((void *)rxr->rx_base, rsize); 2374 2375 /* Allocate receive buffers for the ring*/ 2376 if (ixgbe_allocate_receive_buffers(rxr)) { 2377 device_printf(dev, 2378 "Critical Failure setting up receive buffers\n"); 2379 error = ENOMEM; 2380 goto err_rx_desc; 2381 } 2382 } 2383 2384 return (0); 2385 2386err_rx_desc: 2387 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2388 ixgbe_dma_free(adapter, &rxr->rxdma); 2389err_tx_desc: 2390 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2391 ixgbe_dma_free(adapter, &txr->txdma); 2392 free(adapter->rx_rings, M_DEVBUF); 2393rx_fail: 2394 free(adapter->tx_rings, M_DEVBUF); 2395fail: 2396 return (error); 2397} 2398 2399/********************************************************************* 2400 * 2401 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2402 * the information needed to transmit a packet on the wire. This is 2403 * called only once at attach, setup is done every reset. 2404 * 2405 **********************************************************************/ 2406static int 2407ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2408{ 2409 struct adapter *adapter = txr->adapter; 2410 device_t dev = adapter->dev; 2411 struct ixgbe_tx_buf *txbuf; 2412 int error, i; 2413 2414 /* 2415 * Setup DMA descriptor areas. 2416 */ 2417 if ((error = bus_dma_tag_create(NULL, /* parent */ 2418 PAGE_SIZE, 0, /* alignment, bounds */ 2419 BUS_SPACE_MAXADDR, /* lowaddr */ 2420 BUS_SPACE_MAXADDR, /* highaddr */ 2421 NULL, NULL, /* filter, filterarg */ 2422 IXGBE_TSO_SIZE, /* maxsize */ 2423 IXGBE_MAX_SCATTER, /* nsegments */ 2424 PAGE_SIZE, /* maxsegsize */ 2425 0, /* flags */ 2426 NULL, /* lockfunc */ 2427 NULL, /* lockfuncarg */ 2428 &txr->txtag))) { 2429 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2430 goto fail; 2431 } 2432 2433 if (!(txr->tx_buffers = 2434 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 2435 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2436 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2437 error = ENOMEM; 2438 goto fail; 2439 } 2440 2441 /* Create the descriptor buffer dma maps */ 2442 txbuf = txr->tx_buffers; 2443 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2444 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2445 if (error != 0) { 2446 device_printf(dev, "Unable to create TX DMA map\n"); 2447 goto fail; 2448 } 2449 } 2450 2451 return 0; 2452fail: 2453 /* We free all, it handles case where we are in the middle */ 2454 ixgbe_free_transmit_structures(adapter); 2455 return (error); 2456} 2457 2458/********************************************************************* 2459 * 2460 * Initialize a transmit ring. 2461 * 2462 **********************************************************************/ 2463static void 2464ixgbe_setup_transmit_ring(struct tx_ring *txr) 2465{ 2466 struct adapter *adapter = txr->adapter; 2467 struct ixgbe_tx_buf *txbuf; 2468 int i; 2469 2470 /* Clear the old ring contents */ 2471 bzero((void *)txr->tx_base, 2472 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2473 /* Reset indices */ 2474 txr->next_avail_tx_desc = 0; 2475 txr->next_tx_to_clean = 0; 2476 2477 /* Free any existing tx buffers. */ 2478 txbuf = txr->tx_buffers; 2479 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2480 if (txbuf->m_head != NULL) { 2481 bus_dmamap_sync(txr->txtag, txbuf->map, 2482 BUS_DMASYNC_POSTWRITE); 2483 bus_dmamap_unload(txr->txtag, txbuf->map); 2484 m_freem(txbuf->m_head); 2485 txbuf->m_head = NULL; 2486 } 2487 } 2488 2489 /* Set number of descriptors available */ 2490 txr->tx_avail = adapter->num_tx_desc; 2491 2492 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2493 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2494 2495} 2496 2497/********************************************************************* 2498 * 2499 * Initialize all transmit rings. 2500 * 2501 **********************************************************************/ 2502static int 2503ixgbe_setup_transmit_structures(struct adapter *adapter) 2504{ 2505 struct tx_ring *txr = adapter->tx_rings; 2506 2507 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 2508 ixgbe_setup_transmit_ring(txr); 2509 2510 return (0); 2511} 2512 2513/********************************************************************* 2514 * 2515 * Enable transmit unit. 2516 * 2517 **********************************************************************/ 2518static void 2519ixgbe_initialize_transmit_units(struct adapter *adapter) 2520{ 2521 struct tx_ring *txr = adapter->tx_rings; 2522 struct ixgbe_hw *hw = &adapter->hw; 2523 2524 /* Setup the Base and Length of the Tx Descriptor Ring */ 2525 2526 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2527 u64 txhwb = 0, tdba = txr->txdma.dma_paddr; 2528 u32 txctrl; 2529 2530 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2531 (tdba & 0x00000000ffffffffULL)); 2532 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2533 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2534 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2535 2536 /* Setup for Head WriteBack */ 2537 txhwb = (u64)vtophys(&txr->tx_hwb); 2538 txhwb |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2539 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i), 2540 (txhwb & 0x00000000ffffffffULL)); 2541 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i), 2542 (txhwb >> 32)); 2543 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2544 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 2545 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2546 2547 /* Setup the HW Tx Head and Tail descriptor pointers */ 2548 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2549 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2550 2551 /* Setup Transmit Descriptor Cmd Settings */ 2552 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2553 2554 txr->watchdog_timer = 0; 2555 } 2556 2557 return; 2558} 2559 2560/********************************************************************* 2561 * 2562 * Free all transmit rings. 2563 * 2564 **********************************************************************/ 2565static void 2566ixgbe_free_transmit_structures(struct adapter *adapter) 2567{ 2568 struct tx_ring *txr = adapter->tx_rings; 2569 2570 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2571 IXGBE_TX_LOCK(txr); 2572 ixgbe_free_transmit_buffers(txr); 2573 ixgbe_dma_free(adapter, &txr->txdma); 2574 IXGBE_TX_UNLOCK(txr); 2575 IXGBE_TX_LOCK_DESTROY(txr); 2576 } 2577 free(adapter->tx_rings, M_DEVBUF); 2578} 2579 2580/********************************************************************* 2581 * 2582 * Free transmit ring related data structures. 2583 * 2584 **********************************************************************/ 2585static void 2586ixgbe_free_transmit_buffers(struct tx_ring *txr) 2587{ 2588 struct adapter *adapter = txr->adapter; 2589 struct ixgbe_tx_buf *tx_buffer; 2590 int i; 2591 2592 INIT_DEBUGOUT("free_transmit_ring: begin"); 2593 2594 if (txr->tx_buffers == NULL) 2595 return; 2596 2597 tx_buffer = txr->tx_buffers; 2598 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2599 if (tx_buffer->m_head != NULL) { 2600 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2601 BUS_DMASYNC_POSTWRITE); 2602 bus_dmamap_unload(txr->txtag, 2603 tx_buffer->map); 2604 m_freem(tx_buffer->m_head); 2605 tx_buffer->m_head = NULL; 2606 if (tx_buffer->map != NULL) { 2607 bus_dmamap_destroy(txr->txtag, 2608 tx_buffer->map); 2609 tx_buffer->map = NULL; 2610 } 2611 } else if (tx_buffer->map != NULL) { 2612 bus_dmamap_unload(txr->txtag, 2613 tx_buffer->map); 2614 bus_dmamap_destroy(txr->txtag, 2615 tx_buffer->map); 2616 tx_buffer->map = NULL; 2617 } 2618 } 2619 2620 if (txr->tx_buffers != NULL) { 2621 free(txr->tx_buffers, M_DEVBUF); 2622 txr->tx_buffers = NULL; 2623 } 2624 if (txr->txtag != NULL) { 2625 bus_dma_tag_destroy(txr->txtag); 2626 txr->txtag = NULL; 2627 } 2628 return; 2629} 2630 2631/********************************************************************* 2632 * 2633 * Advanced Context Descriptor setup for VLAN or CSUM 2634 * 2635 **********************************************************************/ 2636 2637static boolean_t 2638ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2639{ 2640 struct adapter *adapter = txr->adapter; 2641 struct ixgbe_adv_tx_context_desc *TXD; 2642 struct ixgbe_tx_buf *tx_buffer; 2643 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2644 struct ether_vlan_header *eh; 2645 struct ip *ip; 2646 struct ip6_hdr *ip6; 2647 int ehdrlen, ip_hlen = 0; 2648 u16 etype; 2649 u8 ipproto = 0; 2650 bool offload = TRUE; 2651 int ctxd = txr->next_avail_tx_desc; 2652#if __FreeBSD_version < 700000 2653 struct m_tag *mtag; 2654#else 2655 u16 vtag = 0; 2656#endif 2657 2658 2659 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2660 offload = FALSE; 2661 2662 tx_buffer = &txr->tx_buffers[ctxd]; 2663 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2664 2665 /* 2666 ** In advanced descriptors the vlan tag must 2667 ** be placed into the descriptor itself. 2668 */ 2669#if __FreeBSD_version < 700000 2670 mtag = VLAN_OUTPUT_TAG(ifp, mp); 2671 if (mtag != NULL) { 2672 vlan_macip_lens |= 2673 htole16(VLAN_TAG_VALUE(mtag)) << IXGBE_ADVTXD_VLAN_SHIFT; 2674 } else if (offload == FALSE) 2675 return FALSE; /* No need for CTX */ 2676#else 2677 if (mp->m_flags & M_VLANTAG) { 2678 vtag = htole16(mp->m_pkthdr.ether_vtag); 2679 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2680 } else if (offload == FALSE) 2681 return FALSE; 2682#endif 2683 /* 2684 * Determine where frame payload starts. 2685 * Jump over vlan headers if already present, 2686 * helpful for QinQ too. 2687 */ 2688 eh = mtod(mp, struct ether_vlan_header *); 2689 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2690 etype = ntohs(eh->evl_proto); 2691 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2692 } else { 2693 etype = ntohs(eh->evl_encap_proto); 2694 ehdrlen = ETHER_HDR_LEN; 2695 } 2696 2697 /* Set the ether header length */ 2698 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2699 2700 switch (etype) { 2701 case ETHERTYPE_IP: 2702 ip = (struct ip *)(mp->m_data + ehdrlen); 2703 ip_hlen = ip->ip_hl << 2; 2704 if (mp->m_len < ehdrlen + ip_hlen) 2705 return FALSE; /* failure */ 2706 ipproto = ip->ip_p; 2707 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2708 break; 2709 case ETHERTYPE_IPV6: 2710 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2711 ip_hlen = sizeof(struct ip6_hdr); 2712 if (mp->m_len < ehdrlen + ip_hlen) 2713 return FALSE; /* failure */ 2714 ipproto = ip6->ip6_nxt; 2715 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2716 break; 2717 default: 2718 offload = FALSE; 2719 break; 2720 } 2721 2722 vlan_macip_lens |= ip_hlen; 2723 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2724 2725 switch (ipproto) { 2726 case IPPROTO_TCP: 2727 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2728 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2729 break; 2730 case IPPROTO_UDP: 2731 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2732 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2733 break; 2734 default: 2735 offload = FALSE; 2736 break; 2737 } 2738 2739 /* Now copy bits into descriptor */ 2740 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2741 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2742 TXD->seqnum_seed = htole32(0); 2743 TXD->mss_l4len_idx = htole32(0); 2744
|
2557#ifndef NO_82598_A0_SUPPORT 2558 if (adapter->hw.revision_id == 0) 2559 desc_flip(TXD); 2560#endif 2561
| |
2562 tx_buffer->m_head = NULL; 2563 2564 /* We've consumed the first desc, adjust counters */ 2565 if (++ctxd == adapter->num_tx_desc) 2566 ctxd = 0; 2567 txr->next_avail_tx_desc = ctxd; 2568 --txr->tx_avail; 2569 2570 return (offload); 2571} 2572 2573#if __FreeBSD_version >= 700000 2574/********************************************************************** 2575 * 2576 * Setup work for hardware segmentation offload (TSO) on 2577 * adapters using advanced tx descriptors 2578 * 2579 **********************************************************************/ 2580static boolean_t 2581ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2582{ 2583 struct adapter *adapter = txr->adapter; 2584 struct ixgbe_adv_tx_context_desc *TXD; 2585 struct ixgbe_tx_buf *tx_buffer; 2586 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2587 u32 mss_l4len_idx = 0; 2588 u16 vtag = 0; 2589 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2590 struct ether_vlan_header *eh; 2591 struct ip *ip; 2592 struct tcphdr *th; 2593 2594 if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) || 2595 (mp->m_pkthdr.len <= IXGBE_TX_BUFFER_SIZE)) 2596 return FALSE; 2597 2598 /* 2599 * Determine where frame payload starts. 2600 * Jump over vlan headers if already present 2601 */ 2602 eh = mtod(mp, struct ether_vlan_header *); 2603 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2604 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2605 else 2606 ehdrlen = ETHER_HDR_LEN; 2607 2608 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2609 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2610 return FALSE; 2611 2612 ctxd = txr->next_avail_tx_desc; 2613 tx_buffer = &txr->tx_buffers[ctxd]; 2614 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2615 2616 ip = (struct ip *)(mp->m_data + ehdrlen); 2617 if (ip->ip_p != IPPROTO_TCP) 2618 return FALSE; /* 0 */ 2619 ip->ip_len = 0; 2620 ip->ip_sum = 0; 2621 ip_hlen = ip->ip_hl << 2; 2622 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2623 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2624 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2625 tcp_hlen = th->th_off << 2; 2626 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2627 /* This is used in the transmit desc in encap */ 2628 *paylen = mp->m_pkthdr.len - hdrlen; 2629 2630 /* VLAN MACLEN IPLEN */ 2631 if (mp->m_flags & M_VLANTAG) { 2632 vtag = htole16(mp->m_pkthdr.ether_vtag); 2633 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2634 } 2635 2636 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2637 vlan_macip_lens |= ip_hlen; 2638 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2639 2640 /* ADV DTYPE TUCMD */ 2641 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2642 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2643 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2644 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2645 2646 2647 /* MSS L4LEN IDX */ 2648 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2649 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2650 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2651 2652 TXD->seqnum_seed = htole32(0); 2653 tx_buffer->m_head = NULL; 2654
| 2745 tx_buffer->m_head = NULL; 2746 2747 /* We've consumed the first desc, adjust counters */ 2748 if (++ctxd == adapter->num_tx_desc) 2749 ctxd = 0; 2750 txr->next_avail_tx_desc = ctxd; 2751 --txr->tx_avail; 2752 2753 return (offload); 2754} 2755 2756#if __FreeBSD_version >= 700000 2757/********************************************************************** 2758 * 2759 * Setup work for hardware segmentation offload (TSO) on 2760 * adapters using advanced tx descriptors 2761 * 2762 **********************************************************************/ 2763static boolean_t 2764ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2765{ 2766 struct adapter *adapter = txr->adapter; 2767 struct ixgbe_adv_tx_context_desc *TXD; 2768 struct ixgbe_tx_buf *tx_buffer; 2769 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2770 u32 mss_l4len_idx = 0; 2771 u16 vtag = 0; 2772 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2773 struct ether_vlan_header *eh; 2774 struct ip *ip; 2775 struct tcphdr *th; 2776 2777 if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) || 2778 (mp->m_pkthdr.len <= IXGBE_TX_BUFFER_SIZE)) 2779 return FALSE; 2780 2781 /* 2782 * Determine where frame payload starts. 2783 * Jump over vlan headers if already present 2784 */ 2785 eh = mtod(mp, struct ether_vlan_header *); 2786 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2787 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2788 else 2789 ehdrlen = ETHER_HDR_LEN; 2790 2791 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2792 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2793 return FALSE; 2794 2795 ctxd = txr->next_avail_tx_desc; 2796 tx_buffer = &txr->tx_buffers[ctxd]; 2797 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2798 2799 ip = (struct ip *)(mp->m_data + ehdrlen); 2800 if (ip->ip_p != IPPROTO_TCP) 2801 return FALSE; /* 0 */ 2802 ip->ip_len = 0; 2803 ip->ip_sum = 0; 2804 ip_hlen = ip->ip_hl << 2; 2805 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2806 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2807 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2808 tcp_hlen = th->th_off << 2; 2809 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2810 /* This is used in the transmit desc in encap */ 2811 *paylen = mp->m_pkthdr.len - hdrlen; 2812 2813 /* VLAN MACLEN IPLEN */ 2814 if (mp->m_flags & M_VLANTAG) { 2815 vtag = htole16(mp->m_pkthdr.ether_vtag); 2816 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2817 } 2818 2819 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2820 vlan_macip_lens |= ip_hlen; 2821 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2822 2823 /* ADV DTYPE TUCMD */ 2824 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2825 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2826 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2827 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2828 2829 2830 /* MSS L4LEN IDX */ 2831 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2832 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2833 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2834 2835 TXD->seqnum_seed = htole32(0); 2836 tx_buffer->m_head = NULL; 2837
|
2655#ifndef NO_82598_A0_SUPPORT 2656 if (adapter->hw.revision_id == 0) 2657 desc_flip(TXD); 2658#endif 2659
| |
2660 if (++ctxd == adapter->num_tx_desc) 2661 ctxd = 0; 2662 2663 txr->tx_avail--; 2664 txr->next_avail_tx_desc = ctxd; 2665 return TRUE; 2666} 2667 2668#else /* For 6.2 RELEASE */ 2669/* This makes it easy to keep the code common */ 2670static boolean_t 2671ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2672{ 2673 return (FALSE); 2674} 2675#endif 2676 2677/********************************************************************** 2678 * 2679 * Examine each tx_buffer in the used queue. If the hardware is done 2680 * processing the packet then free associated resources. The 2681 * tx_buffer is put back on the free queue. 2682 * 2683 **********************************************************************/ 2684static boolean_t 2685ixgbe_txeof(struct tx_ring *txr) 2686{ 2687 struct adapter * adapter = txr->adapter; 2688 struct ifnet *ifp = adapter->ifp; 2689 u32 first, last, done, num_avail; 2690 u32 cleaned = 0; 2691 struct ixgbe_tx_buf *tx_buffer; 2692 struct ixgbe_legacy_tx_desc *tx_desc; 2693 2694 mtx_assert(&txr->mtx, MA_OWNED); 2695 2696 if (txr->tx_avail == adapter->num_tx_desc) 2697 return FALSE; 2698 2699 num_avail = txr->tx_avail; 2700 first = txr->next_tx_to_clean; 2701 2702 tx_buffer = &txr->tx_buffers[first]; 2703 /* For cleanup we just use legacy struct */ 2704 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2705 2706 /* Get the HWB */ 2707 rmb(); 2708 done = txr->tx_hwb; 2709 2710 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2711 BUS_DMASYNC_POSTREAD); 2712 2713 while (TRUE) { 2714 /* We clean the range til last head write back */ 2715 while (first != done) { 2716 tx_desc->upper.data = 0; 2717 tx_desc->lower.data = 0; 2718 tx_desc->buffer_addr = 0; 2719 num_avail++; cleaned++; 2720 2721 if (tx_buffer->m_head) { 2722 ifp->if_opackets++; 2723 bus_dmamap_sync(txr->txtag, 2724 tx_buffer->map, 2725 BUS_DMASYNC_POSTWRITE); 2726 bus_dmamap_unload(txr->txtag, 2727 tx_buffer->map); 2728 m_freem(tx_buffer->m_head); 2729 tx_buffer->m_head = NULL; 2730 tx_buffer->map = NULL; 2731 } 2732 2733 if (++first == adapter->num_tx_desc) 2734 first = 0; 2735 2736 tx_buffer = &txr->tx_buffers[first]; 2737 tx_desc = 2738 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2739 } 2740 /* See if there is more work now */ 2741 last = done; 2742 rmb(); 2743 done = txr->tx_hwb; 2744 if (last == done) 2745 break; 2746 } 2747 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2748 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2749 2750 txr->next_tx_to_clean = first; 2751 2752 /* 2753 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2754 * it is OK to send packets. If there are no pending descriptors, 2755 * clear the timeout. Otherwise, if some descriptors have been freed, 2756 * restart the timeout. 2757 */ 2758 if (num_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 2759 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2760 /* If all are clean turn off the timer */ 2761 if (num_avail == adapter->num_tx_desc) { 2762 txr->watchdog_timer = 0; 2763 txr->tx_avail = num_avail; 2764 return FALSE; 2765 } 2766 } 2767 2768 /* Some were cleaned, so reset timer */ 2769 if (cleaned) 2770 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 2771 txr->tx_avail = num_avail; 2772 return TRUE; 2773} 2774 2775/********************************************************************* 2776 * 2777 * Get a buffer from system mbuf buffer pool. 2778 * 2779 **********************************************************************/ 2780static int
| 2838 if (++ctxd == adapter->num_tx_desc) 2839 ctxd = 0; 2840 2841 txr->tx_avail--; 2842 txr->next_avail_tx_desc = ctxd; 2843 return TRUE; 2844} 2845 2846#else /* For 6.2 RELEASE */ 2847/* This makes it easy to keep the code common */ 2848static boolean_t 2849ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2850{ 2851 return (FALSE); 2852} 2853#endif 2854 2855/********************************************************************** 2856 * 2857 * Examine each tx_buffer in the used queue. If the hardware is done 2858 * processing the packet then free associated resources. The 2859 * tx_buffer is put back on the free queue. 2860 * 2861 **********************************************************************/ 2862static boolean_t 2863ixgbe_txeof(struct tx_ring *txr) 2864{ 2865 struct adapter * adapter = txr->adapter; 2866 struct ifnet *ifp = adapter->ifp; 2867 u32 first, last, done, num_avail; 2868 u32 cleaned = 0; 2869 struct ixgbe_tx_buf *tx_buffer; 2870 struct ixgbe_legacy_tx_desc *tx_desc; 2871 2872 mtx_assert(&txr->mtx, MA_OWNED); 2873 2874 if (txr->tx_avail == adapter->num_tx_desc) 2875 return FALSE; 2876 2877 num_avail = txr->tx_avail; 2878 first = txr->next_tx_to_clean; 2879 2880 tx_buffer = &txr->tx_buffers[first]; 2881 /* For cleanup we just use legacy struct */ 2882 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2883 2884 /* Get the HWB */ 2885 rmb(); 2886 done = txr->tx_hwb; 2887 2888 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2889 BUS_DMASYNC_POSTREAD); 2890 2891 while (TRUE) { 2892 /* We clean the range til last head write back */ 2893 while (first != done) { 2894 tx_desc->upper.data = 0; 2895 tx_desc->lower.data = 0; 2896 tx_desc->buffer_addr = 0; 2897 num_avail++; cleaned++; 2898 2899 if (tx_buffer->m_head) { 2900 ifp->if_opackets++; 2901 bus_dmamap_sync(txr->txtag, 2902 tx_buffer->map, 2903 BUS_DMASYNC_POSTWRITE); 2904 bus_dmamap_unload(txr->txtag, 2905 tx_buffer->map); 2906 m_freem(tx_buffer->m_head); 2907 tx_buffer->m_head = NULL; 2908 tx_buffer->map = NULL; 2909 } 2910 2911 if (++first == adapter->num_tx_desc) 2912 first = 0; 2913 2914 tx_buffer = &txr->tx_buffers[first]; 2915 tx_desc = 2916 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2917 } 2918 /* See if there is more work now */ 2919 last = done; 2920 rmb(); 2921 done = txr->tx_hwb; 2922 if (last == done) 2923 break; 2924 } 2925 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2926 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2927 2928 txr->next_tx_to_clean = first; 2929 2930 /* 2931 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2932 * it is OK to send packets. If there are no pending descriptors, 2933 * clear the timeout. Otherwise, if some descriptors have been freed, 2934 * restart the timeout. 2935 */ 2936 if (num_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 2937 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2938 /* If all are clean turn off the timer */ 2939 if (num_avail == adapter->num_tx_desc) { 2940 txr->watchdog_timer = 0; 2941 txr->tx_avail = num_avail; 2942 return FALSE; 2943 } 2944 } 2945 2946 /* Some were cleaned, so reset timer */ 2947 if (cleaned) 2948 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 2949 txr->tx_avail = num_avail; 2950 return TRUE; 2951} 2952 2953/********************************************************************* 2954 * 2955 * Get a buffer from system mbuf buffer pool. 2956 * 2957 **********************************************************************/ 2958static int
|
2781ixgbe_get_buf(struct rx_ring *rxr, int i)
| 2959ixgbe_get_buf(struct rx_ring *rxr, int i, u8 clean)
|
2782{
| 2960{
|
2783 struct adapter *adapter = rxr->adapter; 2784 struct mbuf *mp; 2785 bus_dmamap_t map; 2786 int nsegs, error, old, s = 0; 2787 int size = MCLBYTES; 2788 2789 2790 bus_dma_segment_t segs[1];
| 2961 struct adapter *adapter = rxr->adapter; 2962 bus_dma_segment_t seg[2];
|
2791 struct ixgbe_rx_buf *rxbuf;
| 2963 struct ixgbe_rx_buf *rxbuf;
|
| 2964 struct mbuf *mh, *mp; 2965 bus_dmamap_t map; 2966 int nsegs, error; 2967 int merr = 0;
|
2792
| 2968
|
2793 /* Are we going to Jumbo clusters? */ 2794 if (adapter->bigbufs) { 2795 size = MJUMPAGESIZE; 2796 s = 1; 2797 }; 2798 2799 mp = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, size); 2800 if (mp == NULL) { 2801 adapter->mbuf_alloc_failed++; 2802 return (ENOBUFS); 2803 }
| |
2804
| 2969
|
2805 mp->m_len = mp->m_pkthdr.len = size;
| 2970 rxbuf = &rxr->rx_buffers[i];
|
2806
| 2971
|
2807 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 2808 m_adj(mp, ETHER_ALIGN);
| 2972 /* First get our header and payload mbuf */ 2973 if (clean & IXGBE_CLEAN_HDR) { 2974 mh = m_gethdr(M_DONTWAIT, MT_DATA); 2975 if (mh == NULL) 2976 goto remap; 2977 } else /* reuse */ 2978 mh = rxr->rx_buffers[i].m_head;
|
2809
| 2979
|
| 2980 mh->m_len = MHLEN; 2981 mh->m_flags |= M_PKTHDR; 2982 2983 if (clean & IXGBE_CLEAN_PKT) { 2984 mp = m_getjcl(M_DONTWAIT, MT_DATA, 2985 M_PKTHDR, adapter->rx_mbuf_sz); 2986 if (mp == NULL) 2987 goto remap; 2988 mp->m_len = adapter->rx_mbuf_sz; 2989 mp->m_flags &= ~M_PKTHDR; 2990 } else { /* reusing */ 2991 mp = rxr->rx_buffers[i].m_pack; 2992 mp->m_len = adapter->rx_mbuf_sz; 2993 mp->m_flags &= ~M_PKTHDR; 2994 }
|
2810 /*
| 2995 /*
|
2811 * Using memory from the mbuf cluster pool, invoke the bus_dma 2812 * machinery to arrange the memory mapping. 2813 */ 2814 error = bus_dmamap_load_mbuf_sg(rxr->rxtag[s], rxr->spare_map[s], 2815 mp, segs, &nsegs, BUS_DMA_NOWAIT); 2816 if (error) { 2817 m_free(mp);
| 2996 ** Need to create a chain for the following 2997 ** dmamap call at this point. 2998 */ 2999 mh->m_next = mp; 3000 mh->m_pkthdr.len = mh->m_len + mp->m_len; 3001 3002 /* Get the memory mapping */ 3003 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3004 rxr->spare_map, mh, seg, &nsegs, BUS_DMA_NOWAIT); 3005 if (error != 0) { 3006 printf("GET BUF: dmamap load failure - %d\n", error); 3007 m_free(mh);
|
2818 return (error); 2819 } 2820
| 3008 return (error); 3009 } 3010
|
2821 /* Now check our target buffer for existing mapping */ 2822 rxbuf = &rxr->rx_buffers[i]; 2823 old = rxbuf->bigbuf;
| 3011 /* Unload old mapping and update buffer struct */
|
2824 if (rxbuf->m_head != NULL)
| 3012 if (rxbuf->m_head != NULL)
|
2825 bus_dmamap_unload(rxr->rxtag[old], rxbuf->map[old]);
| 3013 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3014 map = rxbuf->map; 3015 rxbuf->map = rxr->spare_map; 3016 rxr->spare_map = map; 3017 rxbuf->m_head = mh; 3018 rxbuf->m_pack = mp; 3019 bus_dmamap_sync(rxr->rxtag, 3020 rxbuf->map, BUS_DMASYNC_PREREAD);
|
2826
| 3021
|
2827 map = rxbuf->map[old]; 2828 rxbuf->map[s] = rxr->spare_map[s]; 2829 rxr->spare_map[old] = map; 2830 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s], BUS_DMASYNC_PREREAD); 2831 rxbuf->m_head = mp; 2832 rxbuf->bigbuf = s;
| 3022 /* Update descriptor */ 3023 rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); 3024 rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr);
|
2833
| 3025
|
2834 rxr->rx_base[i].read.pkt_addr = htole64(segs[0].ds_addr);
| 3026 return (0);
|
2835
| 3027
|
2836#ifndef NO_82598_A0_SUPPORT 2837 /* A0 needs to One's Compliment descriptors */ 2838 if (adapter->hw.revision_id == 0) { 2839 struct dhack {u32 a1; u32 a2; u32 b1; u32 b2;}; 2840 struct dhack *d; 2841 2842 d = (struct dhack *)&rxr->rx_base[i]; 2843 d->a1 = ~(d->a1); 2844 d->a2 = ~(d->a2);
| 3028 /* 3029 ** If we get here, we have an mbuf resource 3030 ** issue, so we discard the incoming packet 3031 ** and attempt to reuse existing mbufs next 3032 ** pass thru the ring, but to do so we must 3033 ** fix up the descriptor which had the address 3034 ** clobbered with writeback info. 3035 */ 3036remap: 3037 adapter->mbuf_header_failed++; 3038 merr = ENOBUFS; 3039 /* Is there a reusable buffer? */ 3040 mh = rxr->rx_buffers[i].m_head; 3041 if (mh == NULL) /* Nope, init error */ 3042 return (merr); 3043 mp = rxr->rx_buffers[i].m_pack; 3044 if (mp == NULL) /* Nope, init error */ 3045 return (merr); 3046 /* Get our old mapping */ 3047 rxbuf = &rxr->rx_buffers[i]; 3048 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3049 rxbuf->map, mh, seg, &nsegs, BUS_DMA_NOWAIT); 3050 if (error != 0) { 3051 /* We really have a problem */ 3052 m_free(mh); 3053 return (error);
|
2845 }
| 3054 }
|
2846#endif
| 3055 /* Now fix the descriptor as needed */ 3056 rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); 3057 rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr);
|
2847
| 3058
|
2848 return (0);
| 3059 return (merr);
|
2849} 2850
| 3060} 3061
|
| 3062
|
2851/********************************************************************* 2852 * 2853 * Allocate memory for rx_buffer structures. Since we use one 2854 * rx_buffer per received packet, the maximum number of rx_buffer's 2855 * that we'll need is equal to the number of receive descriptors 2856 * that we've allocated. 2857 * 2858 **********************************************************************/ 2859static int 2860ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 2861{ 2862 struct adapter *adapter = rxr->adapter; 2863 device_t dev = adapter->dev; 2864 struct ixgbe_rx_buf *rxbuf; 2865 int i, bsize, error; 2866 2867 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 2868 if (!(rxr->rx_buffers = 2869 (struct ixgbe_rx_buf *) malloc(bsize, 2870 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2871 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 2872 error = ENOMEM; 2873 goto fail; 2874 } 2875
| 3063/********************************************************************* 3064 * 3065 * Allocate memory for rx_buffer structures. Since we use one 3066 * rx_buffer per received packet, the maximum number of rx_buffer's 3067 * that we'll need is equal to the number of receive descriptors 3068 * that we've allocated. 3069 * 3070 **********************************************************************/ 3071static int 3072ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 3073{ 3074 struct adapter *adapter = rxr->adapter; 3075 device_t dev = adapter->dev; 3076 struct ixgbe_rx_buf *rxbuf; 3077 int i, bsize, error; 3078 3079 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 3080 if (!(rxr->rx_buffers = 3081 (struct ixgbe_rx_buf *) malloc(bsize, 3082 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3083 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3084 error = ENOMEM; 3085 goto fail; 3086 } 3087
|
2876 /* First make the small (2K) tag/map */
| 3088 /* 3089 ** The tag is made to accomodate the largest buffer size 3090 ** with packet split (hence the two segments, even though 3091 ** it may not always use this. 3092 */
|
2877 if ((error = bus_dma_tag_create(NULL, /* parent */ 2878 PAGE_SIZE, 0, /* alignment, bounds */ 2879 BUS_SPACE_MAXADDR, /* lowaddr */ 2880 BUS_SPACE_MAXADDR, /* highaddr */ 2881 NULL, NULL, /* filter, filterarg */
| 3093 if ((error = bus_dma_tag_create(NULL, /* parent */ 3094 PAGE_SIZE, 0, /* alignment, bounds */ 3095 BUS_SPACE_MAXADDR, /* lowaddr */ 3096 BUS_SPACE_MAXADDR, /* highaddr */ 3097 NULL, NULL, /* filter, filterarg */
|
2882 MCLBYTES, /* maxsize */ 2883 1, /* nsegments */ 2884 MCLBYTES, /* maxsegsize */ 2885 0, /* flags */ 2886 NULL, /* lockfunc */ 2887 NULL, /* lockfuncarg */ 2888 &rxr->rxtag[0]))) { 2889 device_printf(dev, "Unable to create RX Small DMA tag\n"); 2890 goto fail; 2891 } 2892 2893 /* Next make the large (4K) tag/map */ 2894 if ((error = bus_dma_tag_create(NULL, /* parent */ 2895 PAGE_SIZE, 0, /* alignment, bounds */ 2896 BUS_SPACE_MAXADDR, /* lowaddr */ 2897 BUS_SPACE_MAXADDR, /* highaddr */ 2898 NULL, NULL, /* filter, filterarg */ 2899 MJUMPAGESIZE, /* maxsize */ 2900 1, /* nsegments */
| 3098 MJUM16BYTES, /* maxsize */ 3099 2, /* nsegments */
|
2901 MJUMPAGESIZE, /* maxsegsize */ 2902 0, /* flags */ 2903 NULL, /* lockfunc */ 2904 NULL, /* lockfuncarg */
| 3100 MJUMPAGESIZE, /* maxsegsize */ 3101 0, /* flags */ 3102 NULL, /* lockfunc */ 3103 NULL, /* lockfuncarg */
|
2905 &rxr->rxtag[1]))) { 2906 device_printf(dev, "Unable to create RX Large DMA tag\n");
| 3104 &rxr->rxtag))) { 3105 device_printf(dev, "Unable to create RX DMA tag\n");
|
2907 goto fail; 2908 } 2909
| 3106 goto fail; 3107 } 3108
|
2910 /* Create the spare maps (used by getbuf) */ 2911 error = bus_dmamap_create(rxr->rxtag[0], BUS_DMA_NOWAIT, 2912 &rxr->spare_map[0]); 2913 error = bus_dmamap_create(rxr->rxtag[1], BUS_DMA_NOWAIT, 2914 &rxr->spare_map[1]);
| 3109 /* Create the spare map (used by getbuf) */ 3110 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3111 &rxr->spare_map);
|
2915 if (error) { 2916 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 2917 __func__, error); 2918 goto fail; 2919 } 2920 2921 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2922 rxbuf = &rxr->rx_buffers[i];
| 3112 if (error) { 3113 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3114 __func__, error); 3115 goto fail; 3116 } 3117 3118 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3119 rxbuf = &rxr->rx_buffers[i];
|
2923 error = bus_dmamap_create(rxr->rxtag[0], 2924 BUS_DMA_NOWAIT, &rxbuf->map[0]);
| 3120 error = bus_dmamap_create(rxr->rxtag, 3121 BUS_DMA_NOWAIT, &rxbuf->map);
|
2925 if (error) {
| 3122 if (error) {
|
2926 device_printf(dev, "Unable to create Small RX DMA map\n");
| 3123 device_printf(dev, "Unable to create RX DMA map\n");
|
2927 goto fail; 2928 }
| 3124 goto fail; 3125 }
|
2929 error = bus_dmamap_create(rxr->rxtag[1], 2930 BUS_DMA_NOWAIT, &rxbuf->map[1]); 2931 if (error) { 2932 device_printf(dev, "Unable to create Large RX DMA map\n"); 2933 goto fail; 2934 }
| |
2935 } 2936 2937 return (0); 2938 2939fail: 2940 /* Frees all, but can handle partial completion */ 2941 ixgbe_free_receive_structures(adapter); 2942 return (error); 2943} 2944 2945/********************************************************************* 2946 * 2947 * Initialize a receive ring and its buffers. 2948 * 2949 **********************************************************************/ 2950static int 2951ixgbe_setup_receive_ring(struct rx_ring *rxr) 2952{ 2953 struct adapter *adapter; 2954 device_t dev; 2955 struct ixgbe_rx_buf *rxbuf; 2956 struct lro_ctrl *lro = &rxr->lro;
| 3126 } 3127 3128 return (0); 3129 3130fail: 3131 /* Frees all, but can handle partial completion */ 3132 ixgbe_free_receive_structures(adapter); 3133 return (error); 3134} 3135 3136/********************************************************************* 3137 * 3138 * Initialize a receive ring and its buffers. 3139 * 3140 **********************************************************************/ 3141static int 3142ixgbe_setup_receive_ring(struct rx_ring *rxr) 3143{ 3144 struct adapter *adapter; 3145 device_t dev; 3146 struct ixgbe_rx_buf *rxbuf; 3147 struct lro_ctrl *lro = &rxr->lro;
|
2957 int j, rsize, s = 0;
| 3148 int j, rsize;
|
2958 2959 adapter = rxr->adapter; 2960 dev = adapter->dev;
| 3149 3150 adapter = rxr->adapter; 3151 dev = adapter->dev;
|
2961 rsize = roundup2(adapter->num_rx_desc * 2962 sizeof(union ixgbe_adv_rx_desc), 4096);
| 3152
|
2963 /* Clear the ring contents */
| 3153 /* Clear the ring contents */
|
| 3154 rsize = roundup2(adapter->num_rx_desc * 3155 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
|
2964 bzero((void *)rxr->rx_base, rsize); 2965 2966 /*
| 3156 bzero((void *)rxr->rx_base, rsize); 3157 3158 /*
|
2967 ** Free current RX buffers: the size buffer 2968 ** that is loaded is indicated by the buffer 2969 ** bigbuf value.
| 3159 ** Free current RX buffer structs and their mbufs
|
2970 */ 2971 for (int i = 0; i < adapter->num_rx_desc; i++) { 2972 rxbuf = &rxr->rx_buffers[i];
| 3160 */ 3161 for (int i = 0; i < adapter->num_rx_desc; i++) { 3162 rxbuf = &rxr->rx_buffers[i];
|
2973 s = rxbuf->bigbuf;
| |
2974 if (rxbuf->m_head != NULL) {
| 3163 if (rxbuf->m_head != NULL) {
|
2975 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s],
| 3164 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
|
2976 BUS_DMASYNC_POSTREAD);
| 3165 BUS_DMASYNC_POSTREAD);
|
2977 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]); 2978 m_freem(rxbuf->m_head);
| 3166 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3167 if (rxbuf->m_head) { 3168 rxbuf->m_head->m_next = rxbuf->m_pack; 3169 m_freem(rxbuf->m_head); 3170 }
|
2979 rxbuf->m_head = NULL;
| 3171 rxbuf->m_head = NULL;
|
| 3172 rxbuf->m_pack = NULL;
|
2980 } 2981 } 2982
| 3173 } 3174 } 3175
|
| 3176 /* Now refresh the mbufs */
|
2983 for (j = 0; j < adapter->num_rx_desc; j++) {
| 3177 for (j = 0; j < adapter->num_rx_desc; j++) {
|
2984 if (ixgbe_get_buf(rxr, j) == ENOBUFS) {
| 3178 if (ixgbe_get_buf(rxr, j, IXGBE_CLEAN_ALL) == ENOBUFS) {
|
2985 rxr->rx_buffers[j].m_head = NULL;
| 3179 rxr->rx_buffers[j].m_head = NULL;
|
| 3180 rxr->rx_buffers[j].m_pack = NULL; 3181 rxr->rx_base[j].read.hdr_addr = 0;
|
2986 rxr->rx_base[j].read.pkt_addr = 0;
| 3182 rxr->rx_base[j].read.pkt_addr = 0;
|
2987 /* If we fail some may have change size */ 2988 s = adapter->bigbufs;
| |
2989 goto fail; 2990 } 2991 } 2992 2993 /* Setup our descriptor indices */ 2994 rxr->next_to_check = 0; 2995 rxr->last_cleaned = 0; 2996 2997 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2998 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2999 3000 /* Now set up the LRO interface */ 3001 if (ixgbe_enable_lro) { 3002 int err = tcp_lro_init(lro); 3003 if (err) {
| 3183 goto fail; 3184 } 3185 } 3186 3187 /* Setup our descriptor indices */ 3188 rxr->next_to_check = 0; 3189 rxr->last_cleaned = 0; 3190 3191 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3192 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3193 3194 /* Now set up the LRO interface */ 3195 if (ixgbe_enable_lro) { 3196 int err = tcp_lro_init(lro); 3197 if (err) {
|
3004 device_printf(dev,"LRO Initialization failed!\n");
| 3198 INIT_DEBUGOUT("LRO Initialization failed!\n");
|
3005 goto fail; 3006 }
| 3199 goto fail; 3200 }
|
3007 device_printf(dev,"RX LRO Initialized\n");
| 3201 INIT_DEBUGOUT("RX LRO Initialized\n");
|
3008 lro->ifp = adapter->ifp; 3009 } 3010
| 3202 lro->ifp = adapter->ifp; 3203 } 3204
|
3011
| |
3012 return (0);
| 3205 return (0);
|
| 3206
|
3013fail: 3014 /*
| 3207fail: 3208 /*
|
3015 * We need to clean up any buffers allocated so far 3016 * 'j' is the failing index, decrement it to get the 3017 * last success.
| 3209 * We need to clean up any buffers allocated 3210 * so far, 'j' is the failing index.
|
3018 */
| 3211 */
|
3019 for (--j; j < 0; j--) { 3020 rxbuf = &rxr->rx_buffers[j];
| 3212 for (int i = 0; i < j; i++) { 3213 rxbuf = &rxr->rx_buffers[i];
|
3021 if (rxbuf->m_head != NULL) {
| 3214 if (rxbuf->m_head != NULL) {
|
3022 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s],
| 3215 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
|
3023 BUS_DMASYNC_POSTREAD);
| 3216 BUS_DMASYNC_POSTREAD);
|
3024 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]);
| 3217 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
|
3025 m_freem(rxbuf->m_head); 3026 rxbuf->m_head = NULL; 3027 } 3028 } 3029 return (ENOBUFS); 3030} 3031 3032/********************************************************************* 3033 * 3034 * Initialize all receive rings. 3035 * 3036 **********************************************************************/ 3037static int 3038ixgbe_setup_receive_structures(struct adapter *adapter) 3039{ 3040 struct rx_ring *rxr = adapter->rx_rings;
| 3218 m_freem(rxbuf->m_head); 3219 rxbuf->m_head = NULL; 3220 } 3221 } 3222 return (ENOBUFS); 3223} 3224 3225/********************************************************************* 3226 * 3227 * Initialize all receive rings. 3228 * 3229 **********************************************************************/ 3230static int 3231ixgbe_setup_receive_structures(struct adapter *adapter) 3232{ 3233 struct rx_ring *rxr = adapter->rx_rings;
|
3041 int i, j, s;
| 3234 int j;
|
3042
| 3235
|
3043 for (i = 0; i < adapter->num_rx_queues; i++, rxr++)
| 3236 for (j = 0; j < adapter->num_rx_queues; j++, rxr++)
|
3044 if (ixgbe_setup_receive_ring(rxr)) 3045 goto fail; 3046 3047 return (0); 3048fail: 3049 /* 3050 * Free RX buffers allocated so far, we will only handle 3051 * the rings that completed, the failing case will have
| 3237 if (ixgbe_setup_receive_ring(rxr)) 3238 goto fail; 3239 3240 return (0); 3241fail: 3242 /* 3243 * Free RX buffers allocated so far, we will only handle 3244 * the rings that completed, the failing case will have
|
3052 * cleaned up for itself. The value of 'i' will be the 3053 * failed ring so we must pre-decrement it.
| 3245 * cleaned up for itself. 'j' failed, so its the terminus.
|
3054 */
| 3246 */
|
3055 rxr = adapter->rx_rings; 3056 for (--i; i > 0; i--, rxr++) { 3057 for (j = 0; j < adapter->num_rx_desc; j++) {
| 3247 for (int i = 0; i < j; ++i) { 3248 rxr = &adapter->rx_rings[i]; 3249 for (int n = 0; n < adapter->num_rx_desc; n++) {
|
3058 struct ixgbe_rx_buf *rxbuf;
| 3250 struct ixgbe_rx_buf *rxbuf;
|
3059 rxbuf = &rxr->rx_buffers[j]; 3060 s = rxbuf->bigbuf;
| 3251 rxbuf = &rxr->rx_buffers[n];
|
3061 if (rxbuf->m_head != NULL) {
| 3252 if (rxbuf->m_head != NULL) {
|
3062 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s],
| 3253 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
|
3063 BUS_DMASYNC_POSTREAD);
| 3254 BUS_DMASYNC_POSTREAD);
|
3064 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]);
| 3255 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
|
3065 m_freem(rxbuf->m_head); 3066 rxbuf->m_head = NULL; 3067 } 3068 } 3069 } 3070 3071 return (ENOBUFS); 3072} 3073 3074/********************************************************************* 3075 *
| 3256 m_freem(rxbuf->m_head); 3257 rxbuf->m_head = NULL; 3258 } 3259 } 3260 } 3261 3262 return (ENOBUFS); 3263} 3264 3265/********************************************************************* 3266 *
|
3076 * Enable receive unit.
| 3267 * Setup receive registers and features.
|
3077 * 3078 **********************************************************************/
| 3268 * 3269 **********************************************************************/
|
| 3270#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3271
|
3079static void 3080ixgbe_initialize_receive_units(struct adapter *adapter) 3081{ 3082 struct rx_ring *rxr = adapter->rx_rings;
| 3272static void 3273ixgbe_initialize_receive_units(struct adapter *adapter) 3274{ 3275 struct rx_ring *rxr = adapter->rx_rings;
|
| 3276 struct ixgbe_hw *hw = &adapter->hw;
|
3083 struct ifnet *ifp = adapter->ifp; 3084 u32 rxctrl, fctrl, srrctl, rxcsum;
| 3277 struct ifnet *ifp = adapter->ifp; 3278 u32 rxctrl, fctrl, srrctl, rxcsum;
|
3085 u32 mrqc, hlreg, linkvec; 3086 u32 random[10]; 3087 int i,j; 3088 union { 3089 u8 c[128]; 3090 u32 i[32]; 3091 } reta;
| 3279 u32 reta, mrqc = 0, hlreg, random[10];
|
3092 3093 3094 /* 3095 * Make sure receives are disabled while 3096 * setting up the descriptor ring 3097 */
| 3280 3281 3282 /* 3283 * Make sure receives are disabled while 3284 * setting up the descriptor ring 3285 */
|
3098 rxctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCTRL); 3099 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXCTRL,
| 3286 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 3287 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
|
3100 rxctrl & ~IXGBE_RXCTRL_RXEN); 3101 3102 /* Enable broadcasts */
| 3288 rxctrl & ~IXGBE_RXCTRL_RXEN); 3289 3290 /* Enable broadcasts */
|
3103 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
| 3291 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
|
3104 fctrl |= IXGBE_FCTRL_BAM;
| 3292 fctrl |= IXGBE_FCTRL_BAM;
|
3105 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
| 3293 fctrl |= IXGBE_FCTRL_DPF; 3294 fctrl |= IXGBE_FCTRL_PMCF; 3295 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
|
3106
| 3296
|
3107 hlreg = IXGBE_READ_REG(&adapter->hw, IXGBE_HLREG0); 3108 if (ifp->if_mtu > ETHERMTU) 3109 hlreg |= IXGBE_HLREG0_JUMBOEN; 3110 else 3111 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3112 IXGBE_WRITE_REG(&adapter->hw, IXGBE_HLREG0, hlreg); 3113 3114 srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(0));
| 3297 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(0));
|
3115 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3116 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
| 3298 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3299 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
|
3117 if (adapter->bigbufs)
| 3300 3301 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3302 /* Set for Jumbo Frames? */ 3303 if (ifp->if_mtu > ETHERMTU) { 3304 hlreg |= IXGBE_HLREG0_JUMBOEN;
|
3118 srrctl |= 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
| 3305 srrctl |= 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
|
3119 else
| 3306 } else { 3307 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
|
3120 srrctl |= 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
| 3308 srrctl |= 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
|
3121 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3122 IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(0), srrctl);
| 3309 } 3310 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
|
3123
| 3311
|
3124 /* Set Queue moderation rate */ 3125 for (i = 0; i < IXGBE_MSGS; i++) 3126 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(i), DEFAULT_ITR);
| 3312 if (ixgbe_rx_hdr_split) { 3313 /* Use a standard mbuf for the header */ 3314 srrctl |= ((IXGBE_RX_HDR << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3315 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3316 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3317 } else 3318 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
|
3127
| 3319
|
3128 /* Set Link moderation lower */ 3129 linkvec = adapter->num_tx_queues + adapter->num_rx_queues; 3130 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(linkvec), LINK_ITR);
| 3320 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(0), srrctl);
|
3131 3132 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3133 u64 rdba = rxr->rxdma.dma_paddr; 3134 /* Setup the Base and Length of the Rx Descriptor Ring */
| 3321 3322 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3323 u64 rdba = rxr->rxdma.dma_paddr; 3324 /* Setup the Base and Length of the Rx Descriptor Ring */
|
3135 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDBAL(i),
| 3325 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
|
3136 (rdba & 0x00000000ffffffffULL));
| 3326 (rdba & 0x00000000ffffffffULL));
|
3137 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDBAH(i), (rdba >> 32)); 3138 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDLEN(i),
| 3327 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 3328 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
|
3139 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3140 3141 /* Setup the HW Rx Head and Tail Descriptor Pointers */
| 3329 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3330 3331 /* Setup the HW Rx Head and Tail Descriptor Pointers */
|
3142 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDH(i), 0); 3143 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(i), 3144 adapter->num_rx_desc - 1);
| 3332 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 3333 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
|
3145 } 3146
| 3334 } 3335
|
3147 rxcsum = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCSUM);
| 3336 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
|
3148
| 3337
|
| 3338 /* Setup RSS */
|
3149 if (adapter->num_rx_queues > 1) {
| 3339 if (adapter->num_rx_queues > 1) {
|
| 3340 int i, j; 3341 reta = 0; 3342
|
3150 /* set up random bits */ 3151 arc4rand(&random, sizeof(random), 0); 3152
| 3343 /* set up random bits */ 3344 arc4rand(&random, sizeof(random), 0); 3345
|
3153 /* Create reta data */ 3154 for (i = 0; i < 128; ) 3155 for (j = 0; j < adapter->num_rx_queues && 3156 i < 128; j++, i++) 3157 reta.c[i] = j; 3158
| |
3159 /* Set up the redirection table */
| 3346 /* Set up the redirection table */
|
3160 for (i = 0; i < 32; i++) 3161 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RETA(i), reta.i[i]);
| 3347 for (i = 0, j = 0; i < 128; i++, j++) { 3348 if (j == adapter->num_rx_queues) j = 0; 3349 reta = (reta << 8) | (j * 0x11); 3350 if ((i & 3) == 3) 3351 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3352 }
|
3162 3163 /* Now fill our hash function seeds */ 3164 for (int i = 0; i < 10; i++)
| 3353 3354 /* Now fill our hash function seeds */ 3355 for (int i = 0; i < 10; i++)
|
3165 IXGBE_WRITE_REG_ARRAY(&adapter->hw, 3166 IXGBE_RSSRK(0), i, random[i]);
| 3356 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
|
3167
| 3357
|
3168 mrqc = IXGBE_MRQC_RSSEN 3169 /* Perform hash on these packet types */ 3170 | IXGBE_MRQC_RSS_FIELD_IPV4 3171 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3172 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 3173 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3174 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3175 | IXGBE_MRQC_RSS_FIELD_IPV6 3176 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3177 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 3178 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 3179 IXGBE_WRITE_REG(&adapter->hw, IXGBE_MRQC, mrqc);
| 3358 /* Perform hash on these packet types */ 3359 mrqc |= IXGBE_MRQC_RSSEN 3360 | IXGBE_MRQC_RSS_FIELD_IPV4 3361 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3362 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 3363 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3364 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3365 | IXGBE_MRQC_RSS_FIELD_IPV6 3366 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3367 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 3368 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 3369 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
|
3180 3181 /* RSS and RX IPP Checksum are mutually exclusive */ 3182 rxcsum |= IXGBE_RXCSUM_PCSD; 3183 } 3184 3185 if (ifp->if_capenable & IFCAP_RXCSUM) 3186 rxcsum |= IXGBE_RXCSUM_PCSD; 3187 3188 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3189 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3190
| 3370 3371 /* RSS and RX IPP Checksum are mutually exclusive */ 3372 rxcsum |= IXGBE_RXCSUM_PCSD; 3373 } 3374 3375 if (ifp->if_capenable & IFCAP_RXCSUM) 3376 rxcsum |= IXGBE_RXCSUM_PCSD; 3377 3378 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3379 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3380
|
3191 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXCSUM, rxcsum);
| 3381 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
|
3192
| 3382
|
3193 /* Enable Receive engine */ 3194 rxctrl |= (IXGBE_RXCTRL_RXEN | IXGBE_RXCTRL_DMBYPS); 3195 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXCTRL, rxctrl); 3196
| |
3197 return; 3198} 3199 3200/********************************************************************* 3201 * 3202 * Free all receive rings. 3203 * 3204 **********************************************************************/ 3205static void 3206ixgbe_free_receive_structures(struct adapter *adapter) 3207{ 3208 struct rx_ring *rxr = adapter->rx_rings; 3209 3210 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3211 struct lro_ctrl *lro = &rxr->lro; 3212 ixgbe_free_receive_buffers(rxr); 3213 /* Free LRO memory */ 3214 tcp_lro_free(lro); 3215 /* Free the ring memory as well */ 3216 ixgbe_dma_free(adapter, &rxr->rxdma); 3217 } 3218 3219 free(adapter->rx_rings, M_DEVBUF); 3220} 3221 3222/********************************************************************* 3223 * 3224 * Free receive ring data structures 3225 * 3226 **********************************************************************/ 3227void 3228ixgbe_free_receive_buffers(struct rx_ring *rxr) 3229{ 3230 struct adapter *adapter = NULL; 3231 struct ixgbe_rx_buf *rxbuf = NULL; 3232 3233 INIT_DEBUGOUT("free_receive_buffers: begin"); 3234 adapter = rxr->adapter; 3235 if (rxr->rx_buffers != NULL) { 3236 rxbuf = &rxr->rx_buffers[0]; 3237 for (int i = 0; i < adapter->num_rx_desc; i++) {
| 3383 return; 3384} 3385 3386/********************************************************************* 3387 * 3388 * Free all receive rings. 3389 * 3390 **********************************************************************/ 3391static void 3392ixgbe_free_receive_structures(struct adapter *adapter) 3393{ 3394 struct rx_ring *rxr = adapter->rx_rings; 3395 3396 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3397 struct lro_ctrl *lro = &rxr->lro; 3398 ixgbe_free_receive_buffers(rxr); 3399 /* Free LRO memory */ 3400 tcp_lro_free(lro); 3401 /* Free the ring memory as well */ 3402 ixgbe_dma_free(adapter, &rxr->rxdma); 3403 } 3404 3405 free(adapter->rx_rings, M_DEVBUF); 3406} 3407 3408/********************************************************************* 3409 * 3410 * Free receive ring data structures 3411 * 3412 **********************************************************************/ 3413void 3414ixgbe_free_receive_buffers(struct rx_ring *rxr) 3415{ 3416 struct adapter *adapter = NULL; 3417 struct ixgbe_rx_buf *rxbuf = NULL; 3418 3419 INIT_DEBUGOUT("free_receive_buffers: begin"); 3420 adapter = rxr->adapter; 3421 if (rxr->rx_buffers != NULL) { 3422 rxbuf = &rxr->rx_buffers[0]; 3423 for (int i = 0; i < adapter->num_rx_desc; i++) {
|
3238 int s = rxbuf->bigbuf;
| |
3239 if (rxbuf->map != NULL) {
| 3424 if (rxbuf->map != NULL) {
|
3240 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]); 3241 bus_dmamap_destroy(rxr->rxtag[s], rxbuf->map[s]);
| 3425 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3426 BUS_DMASYNC_POSTREAD); 3427 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3428 bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
|
3242 } 3243 if (rxbuf->m_head != NULL) { 3244 m_freem(rxbuf->m_head); 3245 } 3246 rxbuf->m_head = NULL; 3247 ++rxbuf; 3248 } 3249 } 3250 if (rxr->rx_buffers != NULL) { 3251 free(rxr->rx_buffers, M_DEVBUF); 3252 rxr->rx_buffers = NULL; 3253 }
| 3429 } 3430 if (rxbuf->m_head != NULL) { 3431 m_freem(rxbuf->m_head); 3432 } 3433 rxbuf->m_head = NULL; 3434 ++rxbuf; 3435 } 3436 } 3437 if (rxr->rx_buffers != NULL) { 3438 free(rxr->rx_buffers, M_DEVBUF); 3439 rxr->rx_buffers = NULL; 3440 }
|
3254 for (int s = 0; s < 2; s++) { 3255 if (rxr->rxtag[s] != NULL) { 3256 bus_dma_tag_destroy(rxr->rxtag[s]); 3257 rxr->rxtag[s] = NULL; 3258 }
| 3441 if (rxr->rxtag != NULL) { 3442 bus_dma_tag_destroy(rxr->rxtag); 3443 rxr->rxtag = NULL;
|
3259 } 3260 return; 3261} 3262 3263/********************************************************************* 3264 * 3265 * This routine executes in interrupt context. It replenishes 3266 * the mbufs in the descriptor and sends data which has been 3267 * dma'ed into host memory to upper layer. 3268 * 3269 * We loop at most count times if count is > 0, or until done if 3270 * count < 0. 3271 *
| 3444 } 3445 return; 3446} 3447 3448/********************************************************************* 3449 * 3450 * This routine executes in interrupt context. It replenishes 3451 * the mbufs in the descriptor and sends data which has been 3452 * dma'ed into host memory to upper layer. 3453 * 3454 * We loop at most count times if count is > 0, or until done if 3455 * count < 0. 3456 *
|
| 3457 * Return TRUE for more work, FALSE for all clean.
|
3272 *********************************************************************/ 3273static bool 3274ixgbe_rxeof(struct rx_ring *rxr, int count) 3275{ 3276 struct adapter *adapter = rxr->adapter; 3277 struct ifnet *ifp = adapter->ifp; 3278 struct lro_ctrl *lro = &rxr->lro; 3279 struct lro_entry *queued;
| 3458 *********************************************************************/ 3459static bool 3460ixgbe_rxeof(struct rx_ring *rxr, int count) 3461{ 3462 struct adapter *adapter = rxr->adapter; 3463 struct ifnet *ifp = adapter->ifp; 3464 struct lro_ctrl *lro = &rxr->lro; 3465 struct lro_entry *queued;
|
3280 struct mbuf *mp; 3281 int len, i, eop = 0; 3282 u8 accept_frame = 0; 3283 u32 staterr;
| 3466 int i; 3467 u32 staterr;
|
3284 union ixgbe_adv_rx_desc *cur; 3285 3286 3287 IXGBE_RX_LOCK(rxr); 3288 i = rxr->next_to_check; 3289 cur = &rxr->rx_base[i]; 3290 staterr = cur->wb.upper.status_error; 3291 3292 if (!(staterr & IXGBE_RXD_STAT_DD)) { 3293 IXGBE_RX_UNLOCK(rxr); 3294 return FALSE; 3295 } 3296
| 3468 union ixgbe_adv_rx_desc *cur; 3469 3470 3471 IXGBE_RX_LOCK(rxr); 3472 i = rxr->next_to_check; 3473 cur = &rxr->rx_base[i]; 3474 staterr = cur->wb.upper.status_error; 3475 3476 if (!(staterr & IXGBE_RXD_STAT_DD)) { 3477 IXGBE_RX_UNLOCK(rxr); 3478 return FALSE; 3479 } 3480
|
| 3481 /* Sync the ring */ 3482 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3483 BUS_DMASYNC_POSTREAD); 3484
|
3297 while ((staterr & IXGBE_RXD_STAT_DD) && (count != 0) && 3298 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
| 3485 while ((staterr & IXGBE_RXD_STAT_DD) && (count != 0) && 3486 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
|
3299 struct mbuf *m = NULL; 3300 int s;
| 3487 struct mbuf *sendmp, *mh, *mp; 3488 u16 hlen, plen, hdr; 3489 u8 dopayload, accept_frame, eop;
|
3301
| 3490
|
3302 mp = rxr->rx_buffers[i].m_head; 3303 s = rxr->rx_buffers[i].bigbuf; 3304 bus_dmamap_sync(rxr->rxtag[s], rxr->rx_buffers[i].map[s], 3305 BUS_DMASYNC_POSTREAD);
| 3491
|
3306 accept_frame = 1;
| 3492 accept_frame = 1;
|
| 3493 hlen = plen = 0; 3494 sendmp = mh = mp = NULL; 3495 3496 /* Sync the buffers */ 3497 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map, 3498 BUS_DMASYNC_POSTREAD); 3499 3500 /* 3501 ** The way the hardware is configured to 3502 ** split, it will ONLY use the header buffer 3503 ** when header split is enabled, otherwise we 3504 ** get normal behavior, ie, both header and 3505 ** payload are DMA'd into the payload buffer. 3506 ** 3507 ** The fmp test is to catch the case where a 3508 ** packet spans multiple descriptors, in that 3509 ** case only the first header is valid. 3510 */ 3511 if ((ixgbe_rx_hdr_split) && (rxr->fmp == NULL)){ 3512 hdr = le16toh(cur-> 3513 wb.lower.lo_dword.hs_rss.hdr_info); 3514 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3515 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3516 if (hlen > IXGBE_RX_HDR) 3517 hlen = IXGBE_RX_HDR; 3518 plen = le16toh(cur->wb.upper.length); 3519 /* Handle the header mbuf */ 3520 mh = rxr->rx_buffers[i].m_head; 3521 mh->m_len = hlen; 3522 dopayload = IXGBE_CLEAN_HDR; 3523 /* 3524 ** Get the payload length, this 3525 ** could be zero if its a small 3526 ** packet. 3527 */ 3528 if (plen) { 3529 mp = rxr->rx_buffers[i].m_pack; 3530 mp->m_len = plen; 3531 mp->m_next = NULL; 3532 mp->m_flags &= ~M_PKTHDR; 3533 mh->m_next = mp; 3534 mh->m_flags |= M_PKTHDR; 3535 dopayload = IXGBE_CLEAN_ALL; 3536 rxr->rx_split_packets++; 3537 } else { /* small packets */ 3538 mh->m_flags &= ~M_PKTHDR; 3539 mh->m_next = NULL; 3540 } 3541 } else { 3542 /* 3543 ** Either no header split, or a 3544 ** secondary piece of a fragmented 3545 ** split packet. 3546 */ 3547 mh = rxr->rx_buffers[i].m_pack; 3548 mh->m_flags |= M_PKTHDR; 3549 mh->m_len = le16toh(cur->wb.upper.length); 3550 dopayload = IXGBE_CLEAN_PKT; 3551 } 3552
|
3307 if (staterr & IXGBE_RXD_STAT_EOP) { 3308 count--; 3309 eop = 1;
| 3553 if (staterr & IXGBE_RXD_STAT_EOP) { 3554 count--; 3555 eop = 1;
|
3310 } else {
| 3556 } else
|
3311 eop = 0;
| 3557 eop = 0;
|
3312 } 3313 len = cur->wb.upper.length;
| |
3314 3315 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) 3316 accept_frame = 0; 3317 3318 if (accept_frame) {
| 3558 3559 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) 3560 accept_frame = 0; 3561 3562 if (accept_frame) {
|
3319 /* Get a fresh buffer first */ 3320 if (ixgbe_get_buf(rxr, i) != 0) {
| 3563 if (ixgbe_get_buf(rxr, i, dopayload) != 0) {
|
3321 ifp->if_iqdrops++; 3322 goto discard; 3323 }
| 3564 ifp->if_iqdrops++; 3565 goto discard; 3566 }
|
3324 3325 /* Assign correct length to the current fragment */ 3326 mp->m_len = len; 3327
| 3567 /* Initial frame - setup */
|
3328 if (rxr->fmp == NULL) {
| 3568 if (rxr->fmp == NULL) {
|
3329 mp->m_pkthdr.len = len; 3330 rxr->fmp = mp; /* Store the first mbuf */ 3331 rxr->lmp = mp;
| 3569 mh->m_flags |= M_PKTHDR; 3570 mh->m_pkthdr.len = mh->m_len; 3571 rxr->fmp = mh; /* Store the first mbuf */ 3572 rxr->lmp = mh; 3573 if (mp) { /* Add payload if split */ 3574 mh->m_pkthdr.len += mp->m_len; 3575 rxr->lmp = mh->m_next; 3576 }
|
3332 } else { 3333 /* Chain mbuf's together */
| 3577 } else { 3578 /* Chain mbuf's together */
|
3334 mp->m_flags &= ~M_PKTHDR; 3335 rxr->lmp->m_next = mp;
| 3579 mh->m_flags &= ~M_PKTHDR; 3580 rxr->lmp->m_next = mh;
|
3336 rxr->lmp = rxr->lmp->m_next;
| 3581 rxr->lmp = rxr->lmp->m_next;
|
3337 rxr->fmp->m_pkthdr.len += len;
| 3582 rxr->fmp->m_pkthdr.len += mh->m_len;
|
3338 } 3339 3340 if (eop) { 3341 rxr->fmp->m_pkthdr.rcvif = ifp; 3342 ifp->if_ipackets++;
| 3583 } 3584 3585 if (eop) { 3586 rxr->fmp->m_pkthdr.rcvif = ifp; 3587 ifp->if_ipackets++;
|
3343 rxr->packet_count++; 3344 rxr->byte_count += rxr->fmp->m_pkthdr.len; 3345 3346 ixgbe_rx_checksum(adapter, 3347 staterr, rxr->fmp); 3348
| 3588 rxr->rx_packets++; 3589 /* capture data for AIM */ 3590 rxr->bytes += rxr->fmp->m_pkthdr.len; 3591 rxr->rx_bytes += rxr->bytes; 3592 if (ifp->if_capenable & IFCAP_RXCSUM) 3593 ixgbe_rx_checksum(staterr, rxr->fmp); 3594 else 3595 rxr->fmp->m_pkthdr.csum_flags = 0;
|
3349 if (staterr & IXGBE_RXD_STAT_VP) {
| 3596 if (staterr & IXGBE_RXD_STAT_VP) {
|
3350#if __FreeBSD_version < 700000 3351 VLAN_INPUT_TAG_NEW(ifp, rxr->fmp, 3352 (le16toh(cur->wb.upper.vlan) & 3353 IXGBE_RX_DESC_SPECIAL_VLAN_MASK)); 3354#else
| |
3355 rxr->fmp->m_pkthdr.ether_vtag =
| 3597 rxr->fmp->m_pkthdr.ether_vtag =
|
3356 le16toh(cur->wb.upper.vlan); 3357 rxr->fmp->m_flags |= M_VLANTAG; 3358#endif
| 3598 le16toh(cur->wb.upper.vlan); 3599 rxr->fmp->m_flags |= M_VLANTAG;
|
3359 }
| 3600 }
|
3360 m = rxr->fmp;
| 3601 sendmp = rxr->fmp;
|
3361 rxr->fmp = NULL; 3362 rxr->lmp = NULL; 3363 } 3364 } else { 3365 ifp->if_ierrors++; 3366discard: 3367 /* Reuse loaded DMA map and just update mbuf chain */
| 3602 rxr->fmp = NULL; 3603 rxr->lmp = NULL; 3604 } 3605 } else { 3606 ifp->if_ierrors++; 3607discard: 3608 /* Reuse loaded DMA map and just update mbuf chain */
|
3368 mp = rxr->rx_buffers[i].m_head; 3369 mp->m_len = mp->m_pkthdr.len = 3370 (rxr->rx_buffers[i].bigbuf ? MJUMPAGESIZE:MCLBYTES);
| 3609 if (hlen) { 3610 mh = rxr->rx_buffers[i].m_head; 3611 mh->m_len = MHLEN; 3612 mh->m_next = NULL; 3613 } 3614 mp = rxr->rx_buffers[i].m_pack; 3615 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz;
|
3371 mp->m_data = mp->m_ext.ext_buf; 3372 mp->m_next = NULL;
| 3616 mp->m_data = mp->m_ext.ext_buf; 3617 mp->m_next = NULL;
|
3373 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
| 3618 if (adapter->max_frame_size <= 3619 (MCLBYTES - ETHER_ALIGN))
|
3374 m_adj(mp, ETHER_ALIGN); 3375 if (rxr->fmp != NULL) {
| 3620 m_adj(mp, ETHER_ALIGN); 3621 if (rxr->fmp != NULL) {
|
| 3622 /* handles the whole chain */
|
3376 m_freem(rxr->fmp); 3377 rxr->fmp = NULL; 3378 rxr->lmp = NULL; 3379 }
| 3623 m_freem(rxr->fmp); 3624 rxr->fmp = NULL; 3625 rxr->lmp = NULL; 3626 }
|
3380 m = NULL;
| 3627 sendmp = NULL;
|
3381 }
| 3628 }
|
3382 3383 /* Zero out the receive descriptors status */ 3384 cur->wb.upper.status_error = 0;
| |
3385 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3386 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3387 3388 rxr->last_cleaned = i; /* for updating tail */ 3389 3390 if (++i == adapter->num_rx_desc) 3391 i = 0; 3392
| 3629 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3630 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3631 3632 rxr->last_cleaned = i; /* for updating tail */ 3633 3634 if (++i == adapter->num_rx_desc) 3635 i = 0; 3636
|
3393 /* Now send up to the stack */ 3394 if (m != NULL) { 3395 rxr->next_to_check = i;
| 3637 /* 3638 ** Now send up to the stack, 3639 ** note the the value of next_to_check 3640 ** is safe because we keep the RX lock 3641 ** thru this call. 3642 */ 3643 if (sendmp != NULL) {
|
3396 /* Use LRO if possible */
| 3644 /* Use LRO if possible */
|
3397 if ((!lro->lro_cnt) || (tcp_lro_rx(lro, m, 0))) { 3398 IXGBE_RX_UNLOCK(rxr); 3399 (*ifp->if_input)(ifp, m); 3400 IXGBE_RX_LOCK(rxr); 3401 i = rxr->next_to_check; 3402 }
| 3645 if ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0))) 3646 (*ifp->if_input)(ifp, sendmp);
|
3403 }
| 3647 }
|
| 3648
|
3404 /* Get next descriptor */ 3405 cur = &rxr->rx_base[i]; 3406 staterr = cur->wb.upper.status_error; 3407 } 3408 rxr->next_to_check = i; 3409 3410 /* Advance the IXGB's Receive Queue "Tail Pointer" */ 3411 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), rxr->last_cleaned);
| 3649 /* Get next descriptor */ 3650 cur = &rxr->rx_base[i]; 3651 staterr = cur->wb.upper.status_error; 3652 } 3653 rxr->next_to_check = i; 3654 3655 /* Advance the IXGB's Receive Queue "Tail Pointer" */ 3656 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), rxr->last_cleaned);
|
3412 IXGBE_RX_UNLOCK(rxr);
| |
3413 3414 /*
| 3657 3658 /*
|
3415 ** Flush any outstanding LRO work 3416 ** this may call into the stack and 3417 ** must not hold a driver lock. 3418 */ 3419 while(!SLIST_EMPTY(&lro->lro_active)) {
| 3659 * Flush any outstanding LRO work 3660 */ 3661 while (!SLIST_EMPTY(&lro->lro_active)) {
|
3420 queued = SLIST_FIRST(&lro->lro_active); 3421 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3422 tcp_lro_flush(lro, queued); 3423 } 3424
| 3662 queued = SLIST_FIRST(&lro->lro_active); 3663 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3664 tcp_lro_flush(lro, queued); 3665 } 3666
|
3425 if (!(staterr & IXGBE_RXD_STAT_DD)) 3426 return FALSE;
| 3667 IXGBE_RX_UNLOCK(rxr);
|
3427
| 3668
|
3428 return TRUE;
| 3669 /* 3670 ** Leaving with more to clean? 3671 ** then schedule another interrupt. 3672 */ 3673 if (staterr & IXGBE_RXD_STAT_DD) { 3674 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, rxr->eims); 3675 return TRUE; 3676 } 3677 3678 return FALSE;
|
3429} 3430 3431/********************************************************************* 3432 * 3433 * Verify that the hardware indicated that the checksum is valid. 3434 * Inform the stack about the status of checksum so that stack 3435 * doesn't spend time verifying the checksum. 3436 * 3437 *********************************************************************/ 3438static void
| 3679} 3680 3681/********************************************************************* 3682 * 3683 * Verify that the hardware indicated that the checksum is valid. 3684 * Inform the stack about the status of checksum so that stack 3685 * doesn't spend time verifying the checksum. 3686 * 3687 *********************************************************************/ 3688static void
|
3439ixgbe_rx_checksum(struct adapter *adapter, 3440 u32 staterr, struct mbuf * mp)
| 3689ixgbe_rx_checksum(u32 staterr, struct mbuf * mp)
|
3441{
| 3690{
|
3442 struct ifnet *ifp = adapter->ifp;
| |
3443 u16 status = (u16) staterr; 3444 u8 errors = (u8) (staterr >> 24); 3445
| 3691 u16 status = (u16) staterr; 3692 u8 errors = (u8) (staterr >> 24); 3693
|
3446 /* Not offloading */ 3447 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0) { 3448 mp->m_pkthdr.csum_flags = 0; 3449 return; 3450 } 3451
| |
3452 if (status & IXGBE_RXD_STAT_IPCS) { 3453 /* Did it pass? */ 3454 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3455 /* IP Checksum Good */ 3456 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3457 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3458 3459 } else 3460 mp->m_pkthdr.csum_flags = 0; 3461 } 3462 if (status & IXGBE_RXD_STAT_L4CS) { 3463 /* Did it pass? */ 3464 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3465 mp->m_pkthdr.csum_flags |= 3466 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3467 mp->m_pkthdr.csum_data = htons(0xffff); 3468 } 3469 } 3470 return; 3471} 3472
| 3694 if (status & IXGBE_RXD_STAT_IPCS) { 3695 /* Did it pass? */ 3696 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3697 /* IP Checksum Good */ 3698 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3699 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3700 3701 } else 3702 mp->m_pkthdr.csum_flags = 0; 3703 } 3704 if (status & IXGBE_RXD_STAT_L4CS) { 3705 /* Did it pass? */ 3706 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3707 mp->m_pkthdr.csum_flags |= 3708 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3709 mp->m_pkthdr.csum_data = htons(0xffff); 3710 } 3711 } 3712 return; 3713} 3714
|
3473#ifdef IXGBE_VLAN_EVENTS
| 3715 3716#ifdef IXGBE_HW_VLAN_SUPPORT
|
3474/* 3475 * This routine is run via an vlan 3476 * config EVENT 3477 */ 3478static void 3479ixgbe_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 3480{ 3481 struct adapter *adapter = ifp->if_softc;
| 3717/* 3718 * This routine is run via an vlan 3719 * config EVENT 3720 */ 3721static void 3722ixgbe_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 3723{ 3724 struct adapter *adapter = ifp->if_softc;
|
3482 u32 ctrl;
| 3725 u32 ctrl, rctl, index, vfta;
|
3483 3484 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 3485 ctrl |= IXGBE_VLNCTRL_VME | IXGBE_VLNCTRL_VFE; 3486 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3487 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 3488 3489 /* Make entry in the hardware filter table */ 3490 ixgbe_set_vfta(&adapter->hw, vtag, 0, TRUE); 3491} 3492 3493/* 3494 * This routine is run via an vlan 3495 * unconfig EVENT 3496 */ 3497static void 3498ixgbe_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 3499{ 3500 struct adapter *adapter = ifp->if_softc;
| 3726 3727 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 3728 ctrl |= IXGBE_VLNCTRL_VME | IXGBE_VLNCTRL_VFE; 3729 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3730 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 3731 3732 /* Make entry in the hardware filter table */ 3733 ixgbe_set_vfta(&adapter->hw, vtag, 0, TRUE); 3734} 3735 3736/* 3737 * This routine is run via an vlan 3738 * unconfig EVENT 3739 */ 3740static void 3741ixgbe_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 3742{ 3743 struct adapter *adapter = ifp->if_softc;
|
| 3744 u32 index, vfta;
|
3501 3502 /* Remove entry in the hardware filter table */ 3503 ixgbe_set_vfta(&adapter->hw, vtag, 0, FALSE); 3504 3505 /* Have all vlans unregistered? */ 3506 if (adapter->ifp->if_vlantrunk == NULL) { 3507 u32 ctrl; 3508 /* Turn off the filter table */ 3509 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 3510 ctrl &= ~IXGBE_VLNCTRL_VME; 3511 ctrl &= ~IXGBE_VLNCTRL_VFE; 3512 ctrl |= IXGBE_VLNCTRL_CFIEN; 3513 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 3514 } 3515}
| 3745 3746 /* Remove entry in the hardware filter table */ 3747 ixgbe_set_vfta(&adapter->hw, vtag, 0, FALSE); 3748 3749 /* Have all vlans unregistered? */ 3750 if (adapter->ifp->if_vlantrunk == NULL) { 3751 u32 ctrl; 3752 /* Turn off the filter table */ 3753 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 3754 ctrl &= ~IXGBE_VLNCTRL_VME; 3755 ctrl &= ~IXGBE_VLNCTRL_VFE; 3756 ctrl |= IXGBE_VLNCTRL_CFIEN; 3757 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 3758 } 3759}
|
3516#endif /* IXGBE_VLAN_EVENTS */
| 3760#endif
|
3517 3518static void 3519ixgbe_enable_intr(struct adapter *adapter) 3520{ 3521 struct ixgbe_hw *hw = &adapter->hw; 3522 u32 mask = IXGBE_EIMS_ENABLE_MASK; 3523 3524 /* Enable Fan Failure detection */ 3525 if (hw->phy.media_type == ixgbe_media_type_copper) 3526 mask |= IXGBE_EIMS_GPI_SDP1;
| 3761 3762static void 3763ixgbe_enable_intr(struct adapter *adapter) 3764{ 3765 struct ixgbe_hw *hw = &adapter->hw; 3766 u32 mask = IXGBE_EIMS_ENABLE_MASK; 3767 3768 /* Enable Fan Failure detection */ 3769 if (hw->phy.media_type == ixgbe_media_type_copper) 3770 mask |= IXGBE_EIMS_GPI_SDP1;
|
| 3771 3772 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3773
|
3527 /* With RSS we use auto clear */ 3528 if (adapter->msix_mem) { 3529 /* Dont autoclear Link */ 3530 mask &= ~IXGBE_EIMS_OTHER; 3531 mask &= ~IXGBE_EIMS_LSC;
| 3774 /* With RSS we use auto clear */ 3775 if (adapter->msix_mem) { 3776 /* Dont autoclear Link */ 3777 mask &= ~IXGBE_EIMS_OTHER; 3778 mask &= ~IXGBE_EIMS_LSC;
|
3532 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 3533 adapter->eims_mask | mask);
| 3779 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
|
3534 } 3535
| 3780 } 3781
|
3536 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
| |
3537 IXGBE_WRITE_FLUSH(hw); 3538 3539 return; 3540} 3541 3542static void 3543ixgbe_disable_intr(struct adapter *adapter) 3544{ 3545 if (adapter->msix_mem) 3546 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 3547 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 3548 IXGBE_WRITE_FLUSH(&adapter->hw); 3549 return; 3550} 3551 3552u16 3553ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 3554{ 3555 u16 value; 3556 3557 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, 3558 reg, 2); 3559 3560 return (value); 3561} 3562
| 3782 IXGBE_WRITE_FLUSH(hw); 3783 3784 return; 3785} 3786 3787static void 3788ixgbe_disable_intr(struct adapter *adapter) 3789{ 3790 if (adapter->msix_mem) 3791 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 3792 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 3793 IXGBE_WRITE_FLUSH(&adapter->hw); 3794 return; 3795} 3796 3797u16 3798ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 3799{ 3800 u16 value; 3801 3802 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, 3803 reg, 2); 3804 3805 return (value); 3806} 3807
|
| 3808/* 3809** Setup the correct IVAR register for a particular MSIX interrupt 3810** (yes this is all very magic and confusing :) 3811** - entry is the register array entry 3812** - vector is the MSIX vector for this queue 3813** - type is RX/TX/MISC 3814*/
|
3563static void
| 3815static void
|
3564ixgbe_set_ivar(struct adapter *adapter, u16 entry, u8 vector)
| 3816ixgbe_set_ivar(struct adapter *adapter, u16 entry, u8 vector, s8 type)
|
3565{
| 3817{
|
| 3818 struct ixgbe_hw *hw = &adapter->hw;
|
3566 u32 ivar, index; 3567 3568 vector |= IXGBE_IVAR_ALLOC_VAL;
| 3819 u32 ivar, index; 3820 3821 vector |= IXGBE_IVAR_ALLOC_VAL;
|
3569 index = (entry >> 2) & 0x1F; 3570 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR(index)); 3571 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3572 ivar |= (vector << (8 * (entry & 0x3))); 3573 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
| 3822 3823 switch (hw->mac.type) { 3824 3825 case ixgbe_mac_82598EB: 3826 if (type == -1) 3827 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3828 else 3829 entry += (type * 64); 3830 index = (entry >> 2) & 0x1F; 3831 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3832 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3833 ivar |= (vector << (8 * (entry & 0x3))); 3834 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 3835 break; 3836 3837 default: 3838 break; 3839 }
|
3574} 3575 3576static void 3577ixgbe_configure_ivars(struct adapter *adapter) 3578{ 3579 struct tx_ring *txr = adapter->tx_rings; 3580 struct rx_ring *rxr = adapter->rx_rings; 3581
| 3840} 3841 3842static void 3843ixgbe_configure_ivars(struct adapter *adapter) 3844{ 3845 struct tx_ring *txr = adapter->tx_rings; 3846 struct rx_ring *rxr = adapter->rx_rings; 3847
|
3582 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3583 ixgbe_set_ivar(adapter, IXGBE_IVAR_RX_QUEUE(i), rxr->msix); 3584 adapter->eims_mask |= rxr->eims; 3585 }
| 3848 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) 3849 ixgbe_set_ivar(adapter, i, rxr->msix, 0);
|
3586
| 3850
|
3587 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 3588 ixgbe_set_ivar(adapter, IXGBE_IVAR_TX_QUEUE(i), txr->msix); 3589 adapter->eims_mask |= txr->eims; 3590 }
| 3851 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 3852 ixgbe_set_ivar(adapter, i, txr->msix, 1);
|
3591 3592 /* For the Link interrupt */
| 3853 3854 /* For the Link interrupt */
|
3593 ixgbe_set_ivar(adapter, IXGBE_IVAR_OTHER_CAUSES_INDEX, 3594 adapter->linkvec); 3595 adapter->eims_mask |= IXGBE_IVAR_OTHER_CAUSES_INDEX;
| 3855 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
|
3596} 3597
| 3856} 3857
|
| 3858/* 3859** ixgbe_sfp_probe - called in the local timer to 3860** determine if a port had optics inserted. 3861*/ 3862static bool ixgbe_sfp_probe(struct adapter *adapter) 3863{ 3864 struct ixgbe_hw *hw = &adapter->hw; 3865 device_t dev = adapter->dev; 3866 bool result = FALSE; 3867 3868 if ((hw->phy.type == ixgbe_phy_nl) && 3869 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 3870 s32 ret = hw->phy.ops.identify_sfp(hw); 3871 if (ret) 3872 goto out; 3873 ret = hw->phy.ops.reset(hw); 3874 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3875 device_printf(dev,"Unsupported SFP+ module detected!"); 3876 printf(" Reload driver with supported module.\n"); 3877 adapter->sfp_probe = FALSE; 3878 goto out; 3879 } else 3880 device_printf(dev,"SFP+ module detected!\n"); 3881 /* We now have supported optics */ 3882 adapter->sfp_probe = FALSE; 3883 result = TRUE; 3884 } 3885out: 3886 return (result); 3887} 3888 3889
|
3598/********************************************************************** 3599 * 3600 * Update the board statistics counters. 3601 * 3602 **********************************************************************/ 3603static void 3604ixgbe_update_stats_counters(struct adapter *adapter) 3605{ 3606 struct ifnet *ifp = adapter->ifp;; 3607 struct ixgbe_hw *hw = &adapter->hw; 3608 u32 missed_rx = 0, bprc, lxon, lxoff, total; 3609 3610 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3611 3612 for (int i = 0; i < 8; i++) { 3613 int mp; 3614 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3615 missed_rx += mp; 3616 adapter->stats.mpc[i] += mp; 3617 adapter->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3618 } 3619 3620 /* Hardware workaround, gprc counts missed packets */ 3621 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3622 adapter->stats.gprc -= missed_rx; 3623 3624 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3625 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3626 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3627 3628 /* 3629 * Workaround: mprc hardware is incorrectly counting 3630 * broadcasts, so for now we subtract those. 3631 */ 3632 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3633 adapter->stats.bprc += bprc; 3634 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3635 adapter->stats.mprc -= bprc; 3636 3637 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3638 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3639 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3640 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3641 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3642 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3643 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3644 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3645
| 3890/********************************************************************** 3891 * 3892 * Update the board statistics counters. 3893 * 3894 **********************************************************************/ 3895static void 3896ixgbe_update_stats_counters(struct adapter *adapter) 3897{ 3898 struct ifnet *ifp = adapter->ifp;; 3899 struct ixgbe_hw *hw = &adapter->hw; 3900 u32 missed_rx = 0, bprc, lxon, lxoff, total; 3901 3902 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 3903 3904 for (int i = 0; i < 8; i++) { 3905 int mp; 3906 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 3907 missed_rx += mp; 3908 adapter->stats.mpc[i] += mp; 3909 adapter->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3910 } 3911 3912 /* Hardware workaround, gprc counts missed packets */ 3913 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 3914 adapter->stats.gprc -= missed_rx; 3915 3916 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 3917 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 3918 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); 3919 3920 /* 3921 * Workaround: mprc hardware is incorrectly counting 3922 * broadcasts, so for now we subtract those. 3923 */ 3924 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 3925 adapter->stats.bprc += bprc; 3926 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 3927 adapter->stats.mprc -= bprc; 3928 3929 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); 3930 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3931 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3932 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3933 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3934 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3935 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3936 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3937
|
3646 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3647 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3648
| |
3649 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3650 adapter->stats.lxontxc += lxon; 3651 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3652 adapter->stats.lxofftxc += lxoff; 3653 total = lxon + lxoff; 3654 3655 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3656 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3657 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3658 adapter->stats.gptc -= total; 3659 adapter->stats.mptc -= total; 3660 adapter->stats.ptc64 -= total; 3661 adapter->stats.gotc -= total * ETHER_MIN_LEN; 3662 3663 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3664 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3665 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3666 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3667 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3668 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3669 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3670 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3671 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3672 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3673 3674 3675 /* Fill out the OS statistics structure */ 3676 ifp->if_ipackets = adapter->stats.gprc; 3677 ifp->if_opackets = adapter->stats.gptc; 3678 ifp->if_ibytes = adapter->stats.gorc; 3679 ifp->if_obytes = adapter->stats.gotc; 3680 ifp->if_imcasts = adapter->stats.mprc; 3681 ifp->if_collisions = 0; 3682 3683 /* Rx Errors */ 3684 ifp->if_ierrors = missed_rx + adapter->stats.crcerrs + 3685 adapter->stats.rlec; 3686} 3687 3688 3689/********************************************************************** 3690 * 3691 * This routine is called only when ixgbe_display_debug_stats is enabled. 3692 * This routine provides a way to take a look at important statistics 3693 * maintained by the driver and hardware. 3694 * 3695 **********************************************************************/ 3696static void 3697ixgbe_print_hw_stats(struct adapter * adapter) 3698{ 3699 device_t dev = adapter->dev; 3700 3701 3702 device_printf(dev,"Std Mbuf Failed = %lu\n",
| 3938 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3939 adapter->stats.lxontxc += lxon; 3940 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3941 adapter->stats.lxofftxc += lxoff; 3942 total = lxon + lxoff; 3943 3944 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3945 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 3946 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 3947 adapter->stats.gptc -= total; 3948 adapter->stats.mptc -= total; 3949 adapter->stats.ptc64 -= total; 3950 adapter->stats.gotc -= total * ETHER_MIN_LEN; 3951 3952 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 3953 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 3954 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 3955 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 3956 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 3957 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 3958 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 3959 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 3960 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 3961 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 3962 3963 3964 /* Fill out the OS statistics structure */ 3965 ifp->if_ipackets = adapter->stats.gprc; 3966 ifp->if_opackets = adapter->stats.gptc; 3967 ifp->if_ibytes = adapter->stats.gorc; 3968 ifp->if_obytes = adapter->stats.gotc; 3969 ifp->if_imcasts = adapter->stats.mprc; 3970 ifp->if_collisions = 0; 3971 3972 /* Rx Errors */ 3973 ifp->if_ierrors = missed_rx + adapter->stats.crcerrs + 3974 adapter->stats.rlec; 3975} 3976 3977 3978/********************************************************************** 3979 * 3980 * This routine is called only when ixgbe_display_debug_stats is enabled. 3981 * This routine provides a way to take a look at important statistics 3982 * maintained by the driver and hardware. 3983 * 3984 **********************************************************************/ 3985static void 3986ixgbe_print_hw_stats(struct adapter * adapter) 3987{ 3988 device_t dev = adapter->dev; 3989 3990 3991 device_printf(dev,"Std Mbuf Failed = %lu\n",
|
3703 adapter->mbuf_alloc_failed); 3704 device_printf(dev,"Std Cluster Failed = %lu\n", 3705 adapter->mbuf_cluster_failed); 3706
| 3992 adapter->mbuf_defrag_failed);
|
3707 device_printf(dev,"Missed Packets = %llu\n", 3708 (long long)adapter->stats.mpc[0]); 3709 device_printf(dev,"Receive length errors = %llu\n", 3710 ((long long)adapter->stats.roc + 3711 (long long)adapter->stats.ruc)); 3712 device_printf(dev,"Crc errors = %llu\n", 3713 (long long)adapter->stats.crcerrs); 3714 device_printf(dev,"Driver dropped packets = %lu\n", 3715 adapter->dropped_pkts); 3716 device_printf(dev, "watchdog timeouts = %ld\n", 3717 adapter->watchdog_events); 3718 3719 device_printf(dev,"XON Rcvd = %llu\n", 3720 (long long)adapter->stats.lxonrxc); 3721 device_printf(dev,"XON Xmtd = %llu\n", 3722 (long long)adapter->stats.lxontxc); 3723 device_printf(dev,"XOFF Rcvd = %llu\n", 3724 (long long)adapter->stats.lxoffrxc); 3725 device_printf(dev,"XOFF Xmtd = %llu\n", 3726 (long long)adapter->stats.lxofftxc); 3727 3728 device_printf(dev,"Total Packets Rcvd = %llu\n", 3729 (long long)adapter->stats.tpr); 3730 device_printf(dev,"Good Packets Rcvd = %llu\n", 3731 (long long)adapter->stats.gprc); 3732 device_printf(dev,"Good Packets Xmtd = %llu\n", 3733 (long long)adapter->stats.gptc); 3734 device_printf(dev,"TSO Transmissions = %lu\n", 3735 adapter->tso_tx); 3736 3737 return; 3738} 3739 3740/********************************************************************** 3741 * 3742 * This routine is called only when em_display_debug_stats is enabled. 3743 * This routine provides a way to take a look at important statistics 3744 * maintained by the driver and hardware. 3745 * 3746 **********************************************************************/ 3747static void 3748ixgbe_print_debug_info(struct adapter *adapter) 3749{ 3750 device_t dev = adapter->dev; 3751 struct rx_ring *rxr = adapter->rx_rings; 3752 struct tx_ring *txr = adapter->tx_rings; 3753 struct ixgbe_hw *hw = &adapter->hw; 3754 3755 device_printf(dev,"Error Byte Count = %u \n", 3756 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 3757 3758 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3759 struct lro_ctrl *lro = &rxr->lro; 3760 device_printf(dev,"Queue[%d]: rdh = %d, hw rdt = %d\n", 3761 i, IXGBE_READ_REG(hw, IXGBE_RDH(i)), 3762 IXGBE_READ_REG(hw, IXGBE_RDT(i)));
| 3993 device_printf(dev,"Missed Packets = %llu\n", 3994 (long long)adapter->stats.mpc[0]); 3995 device_printf(dev,"Receive length errors = %llu\n", 3996 ((long long)adapter->stats.roc + 3997 (long long)adapter->stats.ruc)); 3998 device_printf(dev,"Crc errors = %llu\n", 3999 (long long)adapter->stats.crcerrs); 4000 device_printf(dev,"Driver dropped packets = %lu\n", 4001 adapter->dropped_pkts); 4002 device_printf(dev, "watchdog timeouts = %ld\n", 4003 adapter->watchdog_events); 4004 4005 device_printf(dev,"XON Rcvd = %llu\n", 4006 (long long)adapter->stats.lxonrxc); 4007 device_printf(dev,"XON Xmtd = %llu\n", 4008 (long long)adapter->stats.lxontxc); 4009 device_printf(dev,"XOFF Rcvd = %llu\n", 4010 (long long)adapter->stats.lxoffrxc); 4011 device_printf(dev,"XOFF Xmtd = %llu\n", 4012 (long long)adapter->stats.lxofftxc); 4013 4014 device_printf(dev,"Total Packets Rcvd = %llu\n", 4015 (long long)adapter->stats.tpr); 4016 device_printf(dev,"Good Packets Rcvd = %llu\n", 4017 (long long)adapter->stats.gprc); 4018 device_printf(dev,"Good Packets Xmtd = %llu\n", 4019 (long long)adapter->stats.gptc); 4020 device_printf(dev,"TSO Transmissions = %lu\n", 4021 adapter->tso_tx); 4022 4023 return; 4024} 4025 4026/********************************************************************** 4027 * 4028 * This routine is called only when em_display_debug_stats is enabled. 4029 * This routine provides a way to take a look at important statistics 4030 * maintained by the driver and hardware. 4031 * 4032 **********************************************************************/ 4033static void 4034ixgbe_print_debug_info(struct adapter *adapter) 4035{ 4036 device_t dev = adapter->dev; 4037 struct rx_ring *rxr = adapter->rx_rings; 4038 struct tx_ring *txr = adapter->tx_rings; 4039 struct ixgbe_hw *hw = &adapter->hw; 4040 4041 device_printf(dev,"Error Byte Count = %u \n", 4042 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 4043 4044 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 4045 struct lro_ctrl *lro = &rxr->lro; 4046 device_printf(dev,"Queue[%d]: rdh = %d, hw rdt = %d\n", 4047 i, IXGBE_READ_REG(hw, IXGBE_RDH(i)), 4048 IXGBE_READ_REG(hw, IXGBE_RDT(i)));
|
3763 device_printf(dev,"RX(%d) Packets Received: %lu\n", 3764 rxr->me, (long)rxr->packet_count);
| 4049 device_printf(dev,"RX(%d) Packets Received: %lld\n", 4050 rxr->me, (long long)rxr->rx_packets); 4051 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 4052 rxr->me, (long long)rxr->rx_split_packets);
|
3765 device_printf(dev,"RX(%d) Bytes Received: %lu\n",
| 4053 device_printf(dev,"RX(%d) Bytes Received: %lu\n",
|
3766 rxr->me, (long)rxr->byte_count);
| 4054 rxr->me, (long)rxr->rx_bytes);
|
3767 device_printf(dev,"RX(%d) IRQ Handled: %lu\n", 3768 rxr->me, (long)rxr->rx_irq); 3769 device_printf(dev,"RX(%d) LRO Queued= %d\n", 3770 rxr->me, lro->lro_queued); 3771 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 3772 rxr->me, lro->lro_flushed); 3773 } 3774 3775 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 3776 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i, 3777 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 3778 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 3779 device_printf(dev,"TX(%d) Packets Sent: %lu\n",
| 4055 device_printf(dev,"RX(%d) IRQ Handled: %lu\n", 4056 rxr->me, (long)rxr->rx_irq); 4057 device_printf(dev,"RX(%d) LRO Queued= %d\n", 4058 rxr->me, lro->lro_queued); 4059 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 4060 rxr->me, lro->lro_flushed); 4061 } 4062 4063 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 4064 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i, 4065 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 4066 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 4067 device_printf(dev,"TX(%d) Packets Sent: %lu\n",
|
3780 txr->me, (long)txr->tx_packets);
| 4068 txr->me, (long)txr->total_packets);
|
3781 device_printf(dev,"TX(%d) IRQ Handled: %lu\n", 3782 txr->me, (long)txr->tx_irq); 3783 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 3784 txr->me, (long)txr->no_tx_desc_avail); 3785 } 3786 3787 device_printf(dev,"Link IRQ Handled: %lu\n", 3788 (long)adapter->link_irq); 3789 return; 3790} 3791 3792static int 3793ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS) 3794{ 3795 int error; 3796 int result; 3797 struct adapter *adapter; 3798 3799 result = -1; 3800 error = sysctl_handle_int(oidp, &result, 0, req); 3801 3802 if (error || !req->newptr) 3803 return (error); 3804 3805 if (result == 1) { 3806 adapter = (struct adapter *) arg1; 3807 ixgbe_print_hw_stats(adapter); 3808 } 3809 return error; 3810} 3811 3812static int 3813ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS) 3814{ 3815 int error, result; 3816 struct adapter *adapter; 3817 3818 result = -1; 3819 error = sysctl_handle_int(oidp, &result, 0, req); 3820 3821 if (error || !req->newptr) 3822 return (error); 3823 3824 if (result == 1) { 3825 adapter = (struct adapter *) arg1; 3826 ixgbe_print_debug_info(adapter); 3827 } 3828 return error; 3829} 3830 3831/* 3832** Set flow control using sysctl: 3833** Flow control values: 3834** 0 - off 3835** 1 - rx pause 3836** 2 - tx pause 3837** 3 - full 3838*/ 3839static int 3840ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) 3841{ 3842 int error; 3843 struct adapter *adapter; 3844 3845 error = sysctl_handle_int(oidp, &ixgbe_flow_control, 0, req); 3846 3847 if (error) 3848 return (error); 3849 3850 adapter = (struct adapter *) arg1; 3851 switch (ixgbe_flow_control) { 3852 case ixgbe_fc_rx_pause: 3853 case ixgbe_fc_tx_pause: 3854 case ixgbe_fc_full:
| 4069 device_printf(dev,"TX(%d) IRQ Handled: %lu\n", 4070 txr->me, (long)txr->tx_irq); 4071 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 4072 txr->me, (long)txr->no_tx_desc_avail); 4073 } 4074 4075 device_printf(dev,"Link IRQ Handled: %lu\n", 4076 (long)adapter->link_irq); 4077 return; 4078} 4079 4080static int 4081ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS) 4082{ 4083 int error; 4084 int result; 4085 struct adapter *adapter; 4086 4087 result = -1; 4088 error = sysctl_handle_int(oidp, &result, 0, req); 4089 4090 if (error || !req->newptr) 4091 return (error); 4092 4093 if (result == 1) { 4094 adapter = (struct adapter *) arg1; 4095 ixgbe_print_hw_stats(adapter); 4096 } 4097 return error; 4098} 4099 4100static int 4101ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS) 4102{ 4103 int error, result; 4104 struct adapter *adapter; 4105 4106 result = -1; 4107 error = sysctl_handle_int(oidp, &result, 0, req); 4108 4109 if (error || !req->newptr) 4110 return (error); 4111 4112 if (result == 1) { 4113 adapter = (struct adapter *) arg1; 4114 ixgbe_print_debug_info(adapter); 4115 } 4116 return error; 4117} 4118 4119/* 4120** Set flow control using sysctl: 4121** Flow control values: 4122** 0 - off 4123** 1 - rx pause 4124** 2 - tx pause 4125** 3 - full 4126*/ 4127static int 4128ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) 4129{ 4130 int error; 4131 struct adapter *adapter; 4132 4133 error = sysctl_handle_int(oidp, &ixgbe_flow_control, 0, req); 4134 4135 if (error) 4136 return (error); 4137 4138 adapter = (struct adapter *) arg1; 4139 switch (ixgbe_flow_control) { 4140 case ixgbe_fc_rx_pause: 4141 case ixgbe_fc_tx_pause: 4142 case ixgbe_fc_full:
|
3855 adapter->hw.fc.type = ixgbe_flow_control;
| 4143 adapter->hw.fc.requested_mode = ixgbe_flow_control;
|
3856 break; 3857 case ixgbe_fc_none: 3858 default:
| 4144 break; 4145 case ixgbe_fc_none: 4146 default:
|
3859 adapter->hw.fc.type = ixgbe_fc_none;
| 4147 adapter->hw.fc.requested_mode = ixgbe_fc_none;
|
3860 } 3861 3862 ixgbe_setup_fc(&adapter->hw, 0); 3863 return error; 3864} 3865 3866static void 3867ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 3868 const char *description, int *limit, int value) 3869{ 3870 *limit = value; 3871 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 3872 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 3873 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 3874}
| 4148 } 4149 4150 ixgbe_setup_fc(&adapter->hw, 0); 4151 return error; 4152} 4153 4154static void 4155ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 4156 const char *description, int *limit, int value) 4157{ 4158 *limit = value; 4159 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4160 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4161 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4162}
|
3875 3876#ifndef NO_82598_A0_SUPPORT 3877/* 3878 * A0 Workaround: invert descriptor for hardware 3879 */ 3880void 3881desc_flip(void *desc) 3882{ 3883 struct dhack {u32 a1; u32 a2; u32 b1; u32 b2;}; 3884 struct dhack *d; 3885 3886 d = (struct dhack *)desc; 3887 d->a1 = ~(d->a1); 3888 d->a2 = ~(d->a2); 3889 d->b1 = ~(d->b1); 3890 d->b2 = ~(d->b2); 3891 d->b2 &= 0xFFFFFFF0; 3892 d->b1 &= ~IXGBE_ADVTXD_DCMD_RS; 3893} 3894#endif 3895 3896 3897
| |
| |