Deleted Added
sdiff udiff text old ( 193862 ) new ( 194865 )
full compact
if_igb.c (193862) if_igb.c (194865)
1/******************************************************************************
2
3 Copyright (c) 2001-2009, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8

--- 16 unchanged lines hidden (view full) ---

25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
1/******************************************************************************
2
3 Copyright (c) 2001-2009, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8

--- 16 unchanged lines hidden (view full) ---

25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
33/*$FreeBSD: head/sys/dev/e1000/if_igb.c 193862 2009-06-09 21:43:04Z bz $*/
33/*$FreeBSD: head/sys/dev/e1000/if_igb.c 194865 2009-06-24 17:41:29Z jfv $*/
34
34
35
35#ifdef HAVE_KERNEL_OPTION_HEADERS
36#include "opt_device_polling.h"
36#ifdef HAVE_KERNEL_OPTION_HEADERS
37#include "opt_device_polling.h"
37#include "opt_inet.h"
38#endif
39
40#include <sys/param.h>
41#include <sys/systm.h>
38#endif
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#if __FreeBSD_version >= 800000
43#include <sys/buf_ring.h>
44#endif
42#include <sys/bus.h>
43#include <sys/endian.h>
44#include <sys/kernel.h>
45#include <sys/kthread.h>
46#include <sys/malloc.h>
47#include <sys/mbuf.h>
48#include <sys/module.h>
49#include <sys/rman.h>
50#include <sys/socket.h>
51#include <sys/sockio.h>
52#include <sys/sysctl.h>
53#include <sys/taskqueue.h>
54#include <sys/eventhandler.h>
55#include <sys/pcpu.h>
45#include <sys/bus.h>
46#include <sys/endian.h>
47#include <sys/kernel.h>
48#include <sys/kthread.h>
49#include <sys/malloc.h>
50#include <sys/mbuf.h>
51#include <sys/module.h>
52#include <sys/rman.h>
53#include <sys/socket.h>
54#include <sys/sockio.h>
55#include <sys/sysctl.h>
56#include <sys/taskqueue.h>
57#include <sys/eventhandler.h>
58#include <sys/pcpu.h>
56#ifdef IGB_TIMESYNC
57#include <sys/ioccom.h>
58#include <sys/time.h>
59#endif
59#include <sys/smp.h>
60#include <machine/smp.h>
60#include <machine/bus.h>
61#include <machine/resource.h>
62
61#include <machine/bus.h>
62#include <machine/resource.h>
63
64#ifdef IGB_IEEE1588
65#include <sys/ieee1588.h>
66#endif
67
63#include <net/bpf.h>
64#include <net/ethernet.h>
65#include <net/if.h>
66#include <net/if_arp.h>
67#include <net/if_dl.h>
68#include <net/if_media.h>
69
70#include <net/if_types.h>

--- 19 unchanged lines hidden (view full) ---

90/*********************************************************************
91 * Set this to one to display debug statistics
92 *********************************************************************/
93int igb_display_debug_stats = 0;
94
95/*********************************************************************
96 * Driver version:
97 *********************************************************************/
68#include <net/bpf.h>
69#include <net/ethernet.h>
70#include <net/if.h>
71#include <net/if_arp.h>
72#include <net/if_dl.h>
73#include <net/if_media.h>
74
75#include <net/if_types.h>

--- 19 unchanged lines hidden (view full) ---

95/*********************************************************************
96 * Set this to one to display debug statistics
97 *********************************************************************/
98int igb_display_debug_stats = 0;
99
100/*********************************************************************
101 * Driver version:
102 *********************************************************************/
98char igb_driver_version[] = "version - 1.5.3";
103char igb_driver_version[] = "version - 1.7.3";
99
100
101/*********************************************************************
102 * PCI Device ID Table
103 *
104 * Used by probe to select devices to load on
105 * Last field stores an index into e1000_strings
106 * Last entry must be all 0s

--- 7 unchanged lines hidden (view full) ---

114 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
115 PCI_ANY_ID, PCI_ANY_ID, 0},
116 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
117 PCI_ANY_ID, PCI_ANY_ID, 0},
118 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0},
119 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0},
120 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
121 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
104
105
106/*********************************************************************
107 * PCI Device ID Table
108 *
109 * Used by probe to select devices to load on
110 * Last field stores an index into e1000_strings
111 * Last entry must be all 0s

--- 7 unchanged lines hidden (view full) ---

119 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
120 PCI_ANY_ID, PCI_ANY_ID, 0},
121 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
122 PCI_ANY_ID, PCI_ANY_ID, 0},
123 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0},
124 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
126 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
128 PCI_ANY_ID, PCI_ANY_ID, 0},
122 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
123 PCI_ANY_ID, PCI_ANY_ID, 0},
124 /* required last entry */
125 { 0, 0, 0, 0, 0}
126};
127
128/*********************************************************************
129 * Table of branding strings for all supported NICs.

--- 9 unchanged lines hidden (view full) ---

139static int igb_probe(device_t);
140static int igb_attach(device_t);
141static int igb_detach(device_t);
142static int igb_shutdown(device_t);
143static int igb_suspend(device_t);
144static int igb_resume(device_t);
145static void igb_start(struct ifnet *);
146static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
129 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
130 PCI_ANY_ID, PCI_ANY_ID, 0},
131 /* required last entry */
132 { 0, 0, 0, 0, 0}
133};
134
135/*********************************************************************
136 * Table of branding strings for all supported NICs.

--- 9 unchanged lines hidden (view full) ---

146static int igb_probe(device_t);
147static int igb_attach(device_t);
148static int igb_detach(device_t);
149static int igb_shutdown(device_t);
150static int igb_suspend(device_t);
151static int igb_resume(device_t);
152static void igb_start(struct ifnet *);
153static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
154#if __FreeBSD_version >= 800000
155static int igb_mq_start(struct ifnet *, struct mbuf *);
156static int igb_mq_start_locked(struct ifnet *,
157 struct tx_ring *, struct mbuf *);
158static void igb_qflush(struct ifnet *);
159#endif
147static int igb_ioctl(struct ifnet *, u_long, caddr_t);
148static void igb_watchdog(struct adapter *);
149static void igb_init(void *);
150static void igb_init_locked(struct adapter *);
151static void igb_stop(void *);
152static void igb_media_status(struct ifnet *, struct ifmediareq *);
153static int igb_media_change(struct ifnet *);
154static void igb_identify_hardware(struct adapter *);
155static int igb_allocate_pci_resources(struct adapter *);
156static int igb_allocate_msix(struct adapter *);
157static int igb_allocate_legacy(struct adapter *);
160static int igb_ioctl(struct ifnet *, u_long, caddr_t);
161static void igb_watchdog(struct adapter *);
162static void igb_init(void *);
163static void igb_init_locked(struct adapter *);
164static void igb_stop(void *);
165static void igb_media_status(struct ifnet *, struct ifmediareq *);
166static int igb_media_change(struct ifnet *);
167static void igb_identify_hardware(struct adapter *);
168static int igb_allocate_pci_resources(struct adapter *);
169static int igb_allocate_msix(struct adapter *);
170static int igb_allocate_legacy(struct adapter *);
158#if __FreeBSD_version >= 602105
159static int igb_setup_msix(struct adapter *);
171static int igb_setup_msix(struct adapter *);
160#endif
161static void igb_free_pci_resources(struct adapter *);
162static void igb_local_timer(void *);
163static int igb_hardware_init(struct adapter *);
164static void igb_setup_interface(device_t, struct adapter *);
165static int igb_allocate_queues(struct adapter *);
166static void igb_configure_queues(struct adapter *);
167
168static int igb_allocate_transmit_buffers(struct tx_ring *);

--- 19 unchanged lines hidden (view full) ---

188static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
189static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
190static void igb_set_promisc(struct adapter *);
191static void igb_disable_promisc(struct adapter *);
192static void igb_set_multi(struct adapter *);
193static void igb_print_hw_stats(struct adapter *);
194static void igb_update_link_status(struct adapter *);
195static int igb_get_buf(struct rx_ring *, int, u8);
172static void igb_free_pci_resources(struct adapter *);
173static void igb_local_timer(void *);
174static int igb_hardware_init(struct adapter *);
175static void igb_setup_interface(device_t, struct adapter *);
176static int igb_allocate_queues(struct adapter *);
177static void igb_configure_queues(struct adapter *);
178
179static int igb_allocate_transmit_buffers(struct tx_ring *);

--- 19 unchanged lines hidden (view full) ---

199static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
200static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
201static void igb_set_promisc(struct adapter *);
202static void igb_disable_promisc(struct adapter *);
203static void igb_set_multi(struct adapter *);
204static void igb_print_hw_stats(struct adapter *);
205static void igb_update_link_status(struct adapter *);
206static int igb_get_buf(struct rx_ring *, int, u8);
207
196static void igb_register_vlan(void *, struct ifnet *, u16);
197static void igb_unregister_vlan(void *, struct ifnet *, u16);
208static void igb_register_vlan(void *, struct ifnet *, u16);
209static void igb_unregister_vlan(void *, struct ifnet *, u16);
210static void igb_setup_vlan_hw_support(struct adapter *);
211
198static int igb_xmit(struct tx_ring *, struct mbuf **);
199static int igb_dma_malloc(struct adapter *, bus_size_t,
200 struct igb_dma_alloc *, int);
201static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
202static void igb_print_debug_info(struct adapter *);
203static void igb_print_nvm_info(struct adapter *);
204static int igb_is_valid_ether_addr(u8 *);
205static int igb_sysctl_stats(SYSCTL_HANDLER_ARGS);
206static int igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
207/* Management and WOL Support */
208static void igb_init_manageability(struct adapter *);
209static void igb_release_manageability(struct adapter *);
210static void igb_get_hw_control(struct adapter *);
211static void igb_release_hw_control(struct adapter *);
212static void igb_enable_wakeup(device_t);
213
212static int igb_xmit(struct tx_ring *, struct mbuf **);
213static int igb_dma_malloc(struct adapter *, bus_size_t,
214 struct igb_dma_alloc *, int);
215static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
216static void igb_print_debug_info(struct adapter *);
217static void igb_print_nvm_info(struct adapter *);
218static int igb_is_valid_ether_addr(u8 *);
219static int igb_sysctl_stats(SYSCTL_HANDLER_ARGS);
220static int igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
221/* Management and WOL Support */
222static void igb_init_manageability(struct adapter *);
223static void igb_release_manageability(struct adapter *);
224static void igb_get_hw_control(struct adapter *);
225static void igb_release_hw_control(struct adapter *);
226static void igb_enable_wakeup(device_t);
227
214#ifdef IGB_TIMESYNC
215/* Precision Time sync support */
216static int igb_tsync_init(struct adapter *);
217static void igb_tsync_disable(struct adapter *);
218#endif
219
220#if __FreeBSD_version > 700000
221static int igb_irq_fast(void *);
228static int igb_irq_fast(void *);
222#else
223static void igb_irq_fast(void *);
224#endif
225
226static void igb_add_rx_process_limit(struct adapter *, const char *,
227 const char *, int *, int);
228static void igb_handle_rxtx(void *context, int pending);
229static void igb_handle_tx(void *context, int pending);
230static void igb_handle_rx(void *context, int pending);
231
229static void igb_add_rx_process_limit(struct adapter *, const char *,
230 const char *, int *, int);
231static void igb_handle_rxtx(void *context, int pending);
232static void igb_handle_tx(void *context, int pending);
233static void igb_handle_rx(void *context, int pending);
234
232#if __FreeBSD_version >= 602105
233/* These are MSIX only irq handlers */
234static void igb_msix_rx(void *);
235static void igb_msix_tx(void *);
236static void igb_msix_link(void *);
235/* These are MSIX only irq handlers */
236static void igb_msix_rx(void *);
237static void igb_msix_tx(void *);
238static void igb_msix_link(void *);
237#endif
238
239/* Adaptive Interrupt Moderation */
240static void igb_update_aim(struct rx_ring *);
241
242/*********************************************************************
243 * FreeBSD Device Interface Entry Points
244 *********************************************************************/
245

--- 43 unchanged lines hidden (view full) ---

289static int igb_low_latency = IGB_LOW_LATENCY;
290TUNABLE_INT("hw.igb.low_latency", &igb_low_latency);
291static int igb_ave_latency = IGB_AVE_LATENCY;
292TUNABLE_INT("hw.igb.ave_latency", &igb_ave_latency);
293static int igb_bulk_latency = IGB_BULK_LATENCY;
294TUNABLE_INT("hw.igb.bulk_latency", &igb_bulk_latency);
295
296/*
239
240/* Adaptive Interrupt Moderation */
241static void igb_update_aim(struct rx_ring *);
242
243/*********************************************************************
244 * FreeBSD Device Interface Entry Points
245 *********************************************************************/
246

--- 43 unchanged lines hidden (view full) ---

290static int igb_low_latency = IGB_LOW_LATENCY;
291TUNABLE_INT("hw.igb.low_latency", &igb_low_latency);
292static int igb_ave_latency = IGB_AVE_LATENCY;
293TUNABLE_INT("hw.igb.ave_latency", &igb_ave_latency);
294static int igb_bulk_latency = IGB_BULK_LATENCY;
295TUNABLE_INT("hw.igb.bulk_latency", &igb_bulk_latency);
296
297/*
297** IF YOU CHANGE THESE: be sure and change IGB_MSIX_VEC in
298** if_igb.h to match. These can be autoconfigured if set to
299** 0, it will then be based on number of cpus.
298** This will autoconfigure based on the number
299** of CPUs if set to 0. Only a matched pair of
300** TX and RX rings are allowed.
300*/
301*/
301static int igb_tx_queues = 1;
302static int igb_rx_queues = 1;
303TUNABLE_INT("hw.igb.tx_queues", &igb_tx_queues);
304TUNABLE_INT("hw.igb.rx_queues", &igb_rx_queues);
302static int igb_num_queues = 1;
303TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
305
306/* How many packets rxeof tries to clean at a time */
307static int igb_rx_process_limit = 100;
308TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
309
310/* Flow control setting - default to FULL */
311static int igb_fc_setting = e1000_fc_full;
312TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
313
314/*
304
305/* How many packets rxeof tries to clean at a time */
306static int igb_rx_process_limit = 100;
307TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
308
309/* Flow control setting - default to FULL */
310static int igb_fc_setting = e1000_fc_full;
311TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
312
313/*
315 * Should the driver do LRO on the RX end
316 * this can be toggled on the fly, but the
317 * interface must be reset (down/up) for it
318 * to take effect.
319 */
320static int igb_enable_lro = 1;
321TUNABLE_INT("hw.igb.enable_lro", &igb_enable_lro);
314** Shadow VFTA table, this is needed because
315** the real filter table gets cleared during
316** a soft reset and the driver needs to be able
317** to repopulate it.
318*/
319static u32 igb_shadow_vfta[IGB_VFTA_SIZE];
322
320
323/*
324 * Enable RX Header Split
325 */
326static int igb_rx_hdr_split = 1;
327TUNABLE_INT("hw.igb.rx_hdr_split", &igb_rx_hdr_split);
328
321
329extern int mp_ncpus;
330/*********************************************************************
331 * Device identification routine
332 *
333 * igb_probe determines if the driver should be loaded on
334 * adapter based on PCI vendor/device id of the adapter.
335 *
336 * return BUS_PROBE_DEFAULT on success, positive on failure
337 *********************************************************************/

--- 74 unchanged lines hidden (view full) ---

412 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
413 igb_sysctl_stats, "I", "Statistics");
414
415 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
416 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
417 OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW,
418 &igb_fc_setting, 0, "Flow Control");
419
322/*********************************************************************
323 * Device identification routine
324 *
325 * igb_probe determines if the driver should be loaded on
326 * adapter based on PCI vendor/device id of the adapter.
327 *
328 * return BUS_PROBE_DEFAULT on success, positive on failure
329 *********************************************************************/

--- 74 unchanged lines hidden (view full) ---

404 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
405 igb_sysctl_stats, "I", "Statistics");
406
407 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
408 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
409 OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW,
410 &igb_fc_setting, 0, "Flow Control");
411
420 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
421 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
422 OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW,
423 &igb_enable_lro, 0, "Large Receive Offload");
424
425 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
426 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
427 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
428 &igb_enable_aim, 1, "Interrupt Moderation");
429
430 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
431 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
432 OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW,

--- 4 unchanged lines hidden (view full) ---

437 OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW,
438 &igb_ave_latency, 1, "Average Latency");
439
440 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
441 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
442 OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW,
443 &igb_bulk_latency, 1, "Bulk Latency");
444
412 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
413 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
414 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
415 &igb_enable_aim, 1, "Interrupt Moderation");
416
417 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
418 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
419 OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW,

--- 4 unchanged lines hidden (view full) ---

424 OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW,
425 &igb_ave_latency, 1, "Average Latency");
426
427 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
428 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
429 OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW,
430 &igb_bulk_latency, 1, "Bulk Latency");
431
445 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
446 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
447 OID_AUTO, "hdr_split", CTLTYPE_INT|CTLFLAG_RW,
448 &igb_rx_hdr_split, 0, "RX Header Split");
449
450 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
451
452 /* Determine hardware and mac info */
453 igb_identify_hardware(adapter);
454
455 /* Setup PCI resources */
456 if (igb_allocate_pci_resources(adapter)) {
457 device_printf(dev, "Allocation of PCI resources failed\n");

--- 114 unchanged lines hidden (view full) ---

572 else /* MSI or Legacy */
573 error = igb_allocate_legacy(adapter);
574 if (error)
575 goto err_late;
576
577 /* Setup OS specific network interface */
578 igb_setup_interface(dev, adapter);
579
432 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
433
434 /* Determine hardware and mac info */
435 igb_identify_hardware(adapter);
436
437 /* Setup PCI resources */
438 if (igb_allocate_pci_resources(adapter)) {
439 device_printf(dev, "Allocation of PCI resources failed\n");

--- 114 unchanged lines hidden (view full) ---

554 else /* MSI or Legacy */
555 error = igb_allocate_legacy(adapter);
556 if (error)
557 goto err_late;
558
559 /* Setup OS specific network interface */
560 igb_setup_interface(dev, adapter);
561
562#ifdef IGB_IEEE1588
563 /*
564 ** Setup the timer: IEEE 1588 support
565 */
566 adapter->cycles.read = igb_read_clock;
567 adapter->cycles.mask = (u64)-1;
568 adapter->cycles.mult = 1;
569 adapter->cycles.shift = IGB_TSYNC_SHIFT;
570 E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) |
571 IGB_TSYNC_CYCLE_TIME * IGB_TSYNC_SHIFT);
572 E1000_WRITE_REG(&adapter->hw, E1000_SYSTIML, 0x00000000);
573 E1000_WRITE_REG(&adapter->hw, E1000_SYSTIMH, 0xFF800000);
574
575 // JFV - this is not complete yet
576#endif
577
580 /* Initialize statistics */
581 igb_update_stats_counters(adapter);
582
583 adapter->hw.mac.get_link_status = 1;
584 igb_update_link_status(adapter);
585
586 /* Indicate SOL/IDER usage */
587 if (e1000_check_reset_block(&adapter->hw))

--- 49 unchanged lines hidden (view full) ---

637igb_detach(device_t dev)
638{
639 struct adapter *adapter = device_get_softc(dev);
640 struct ifnet *ifp = adapter->ifp;
641
642 INIT_DEBUGOUT("igb_detach: begin");
643
644 /* Make sure VLANS are not using driver */
578 /* Initialize statistics */
579 igb_update_stats_counters(adapter);
580
581 adapter->hw.mac.get_link_status = 1;
582 igb_update_link_status(adapter);
583
584 /* Indicate SOL/IDER usage */
585 if (e1000_check_reset_block(&adapter->hw))

--- 49 unchanged lines hidden (view full) ---

635igb_detach(device_t dev)
636{
637 struct adapter *adapter = device_get_softc(dev);
638 struct ifnet *ifp = adapter->ifp;
639
640 INIT_DEBUGOUT("igb_detach: begin");
641
642 /* Make sure VLANS are not using driver */
645#if __FreeBSD_version >= 700000
646 if (adapter->ifp->if_vlantrunk != NULL) {
643 if (adapter->ifp->if_vlantrunk != NULL) {
647#else
648 if (adapter->ifp->if_nvlans != 0) {
649#endif
650 device_printf(dev,"Vlan in use, detach first\n");
651 return (EBUSY);
652 }
653
654 IGB_CORE_LOCK(adapter);
655 adapter->in_detach = 1;
656 igb_stop(adapter);
657 IGB_CORE_UNLOCK(adapter);

--- 134 unchanged lines hidden (view full) ---

792 /* Send a copy of the frame to the BPF listener */
793 ETHER_BPF_MTAP(ifp, m_head);
794
795 /* Set timeout in case hardware has problems transmitting. */
796 txr->watchdog_timer = IGB_TX_TIMEOUT;
797 }
798}
799
644 device_printf(dev,"Vlan in use, detach first\n");
645 return (EBUSY);
646 }
647
648 IGB_CORE_LOCK(adapter);
649 adapter->in_detach = 1;
650 igb_stop(adapter);
651 IGB_CORE_UNLOCK(adapter);

--- 134 unchanged lines hidden (view full) ---

786 /* Send a copy of the frame to the BPF listener */
787 ETHER_BPF_MTAP(ifp, m_head);
788
789 /* Set timeout in case hardware has problems transmitting. */
790 txr->watchdog_timer = IGB_TX_TIMEOUT;
791 }
792}
793
794/*
795 * Legacy TX driver routine, called from the
796 * stack, always uses tx[0], and spins for it.
797 * Should not be used with multiqueue tx
798 */
800static void
801igb_start(struct ifnet *ifp)
802{
803 struct adapter *adapter = ifp->if_softc;
799static void
800igb_start(struct ifnet *ifp)
801{
802 struct adapter *adapter = ifp->if_softc;
804 struct tx_ring *txr;
805 u32 queue = 0;
803 struct tx_ring *txr = adapter->tx_rings;
806
804
807 /*
808 ** This is really just here for testing
809 ** TX multiqueue, ultimately what is
810 ** needed is the flow support in the stack
811 ** and appropriate logic here to deal with
812 ** it. -jfv
813 */
814 if (adapter->num_tx_queues > 1)
815 queue = (curcpu % adapter->num_tx_queues);
816
817 txr = &adapter->tx_rings[queue];
818 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
819 IGB_TX_LOCK(txr);
820 igb_start_locked(txr, ifp);
821 IGB_TX_UNLOCK(txr);
822 }
805 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
806 IGB_TX_LOCK(txr);
807 igb_start_locked(txr, ifp);
808 IGB_TX_UNLOCK(txr);
809 }
810 return;
823}
824
811}
812
813#if __FreeBSD_version >= 800000
814/*
815** Multiqueue Transmit driver
816**
817*/
818static int
819igb_mq_start(struct ifnet *ifp, struct mbuf *m)
820{
821 struct adapter *adapter = ifp->if_softc;
822 struct tx_ring *txr;
823 int i = 0, err = 0;
824
825 /* Which queue to use */
826 if ((m->m_flags & M_FLOWID) != 0)
827 i = m->m_pkthdr.flowid % adapter->num_queues;
828 txr = &adapter->tx_rings[i];
829
830 if (IGB_TX_TRYLOCK(txr)) {
831 err = igb_mq_start_locked(ifp, txr, m);
832 IGB_TX_UNLOCK(txr);
833 } else
834 err = drbr_enqueue(ifp, txr->br, m);
835
836 return (err);
837}
838
839static int
840igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
841{
842 struct adapter *adapter = txr->adapter;
843 struct mbuf *next;
844 int err = 0;
845
846 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
847 err = drbr_enqueue(ifp, txr->br, m);
848 return (err);
849 }
850
851 if (m == NULL) /* Called by tasklet */
852 goto process;
853
854 /* If nothing queued go right to xmit */
855 if (drbr_empty(ifp, txr->br)) {
856 if (igb_xmit(txr, &m)) {
857 if (m && (err = drbr_enqueue(ifp, txr->br, m)) != 0)
858 return (err);
859 } else {
860 /* Success, update stats */
861 drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
862 /* Send a copy of the frame to the BPF listener */
863 ETHER_BPF_MTAP(ifp, m);
864 /* Set the watchdog */
865 txr->watchdog_timer = IGB_TX_TIMEOUT;
866 }
867
868 } else if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
869 return (err);
870
871process:
872 if (drbr_empty(ifp, txr->br))
873 return (err);
874
875 /* Process the queue */
876 while (TRUE) {
877 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
878 break;
879 next = drbr_dequeue(ifp, txr->br);
880 if (next == NULL)
881 break;
882 if (igb_xmit(txr, &next))
883 break;
884 ETHER_BPF_MTAP(ifp, next);
885 /* Set the watchdog */
886 txr->watchdog_timer = IGB_TX_TIMEOUT;
887 }
888
889 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD)
890 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
891
892 return (err);
893}
894
895/*
896** Flush all ring buffers
897*/
898static void
899igb_qflush(struct ifnet *ifp)
900{
901 struct adapter *adapter = ifp->if_softc;
902 struct tx_ring *txr = adapter->tx_rings;
903 struct mbuf *m;
904
905 for (int i = 0; i < adapter->num_queues; i++, txr++) {
906 IGB_TX_LOCK(txr);
907 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
908 m_freem(m);
909 IGB_TX_UNLOCK(txr);
910 }
911 if_qflush(ifp);
912}
913#endif /* __FreeBSD_version >= 800000 */
914
825/*********************************************************************
826 * Ioctl entry point
827 *
828 * igb_ioctl is called when the user wants to configure the
829 * interface.
830 *
831 * return 0 on success, positive on failure
832 **********************************************************************/

--- 114 unchanged lines hidden (view full) ---

947 if (mask & IFCAP_TSO4) {
948 ifp->if_capenable ^= IFCAP_TSO4;
949 reinit = 1;
950 }
951 if (mask & IFCAP_VLAN_HWTAGGING) {
952 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
953 reinit = 1;
954 }
915/*********************************************************************
916 * Ioctl entry point
917 *
918 * igb_ioctl is called when the user wants to configure the
919 * interface.
920 *
921 * return 0 on success, positive on failure
922 **********************************************************************/

--- 114 unchanged lines hidden (view full) ---

1037 if (mask & IFCAP_TSO4) {
1038 ifp->if_capenable ^= IFCAP_TSO4;
1039 reinit = 1;
1040 }
1041 if (mask & IFCAP_VLAN_HWTAGGING) {
1042 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1043 reinit = 1;
1044 }
1045 if (mask & IFCAP_LRO) {
1046 ifp->if_capenable ^= IFCAP_LRO;
1047 reinit = 1;
1048 }
955 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
956 igb_init(adapter);
1049 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1050 igb_init(adapter);
957#if __FreeBSD_version >= 700000
958 VLAN_CAPABILITIES(ifp);
1051 VLAN_CAPABILITIES(ifp);
959#endif
960 break;
961 }
962
1052 break;
1053 }
1054
963#ifdef IGB_TIMESYNC
1055#ifdef IGB_IEEE1588
964 /*
965 ** IOCTL support for Precision Time (IEEE 1588) Support
966 */
1056 /*
1057 ** IOCTL support for Precision Time (IEEE 1588) Support
1058 */
967 case IGB_TIMESYNC_READTS:
968 {
969 u32 rx_ctl, tx_ctl;
970 struct igb_tsync_read *tdata;
1059 case SIOCSHWTSTAMP:
1060 error = igb_hwtstamp_ioctl(adapter, ifp);
1061 break;
1062#endif
971
1063
972 tdata = (struct igb_tsync_read *) ifr->ifr_data;
973
974 if (tdata->read_current_time) {
975 getnanotime(&tdata->system_time);
976 tdata->network_time = E1000_READ_REG(&adapter->hw,
977 E1000_SYSTIML);
978 tdata->network_time |=
979 (u64)E1000_READ_REG(&adapter->hw,
980 E1000_SYSTIMH ) << 32;
981 }
982
983 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
984 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
985
986 if (rx_ctl & 0x1) {
987 u32 tmp;
988 unsigned char *tmp_cp;
989
990 tdata->rx_valid = 1;
991 tdata->rx_stamp = E1000_READ_REG(&adapter->hw, E1000_RXSTMPL);
992 tdata->rx_stamp |= (u64)E1000_READ_REG(&adapter->hw,
993 E1000_RXSTMPH) << 32;
994
995 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRL);
996 tmp_cp = (unsigned char *) &tmp;
997 tdata->srcid[0] = tmp_cp[0];
998 tdata->srcid[1] = tmp_cp[1];
999 tdata->srcid[2] = tmp_cp[2];
1000 tdata->srcid[3] = tmp_cp[3];
1001 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRH);
1002 tmp_cp = (unsigned char *) &tmp;
1003 tdata->srcid[4] = tmp_cp[0];
1004 tdata->srcid[5] = tmp_cp[1];
1005 tdata->seqid = tmp >> 16;
1006 tdata->seqid = htons(tdata->seqid);
1007 } else
1008 tdata->rx_valid = 0;
1009
1010 if (tx_ctl & 0x1) {
1011 tdata->tx_valid = 1;
1012 tdata->tx_stamp = E1000_READ_REG(&adapter->hw, E1000_TXSTMPL);
1013 tdata->tx_stamp |= (u64) E1000_READ_REG(&adapter->hw,
1014 E1000_TXSTMPH) << 32;
1015 } else
1016 tdata->tx_valid = 0;
1017
1018 return (0);
1019 }
1020#endif /* IGB_TIMESYNC */
1021
1022 default:
1023 error = ether_ioctl(ifp, command, data);
1024 break;
1025 }
1026
1027 return (error);
1028}
1029

--- 20 unchanged lines hidden (view full) ---

1050 ** Then txeof keeps resetting it as long as it cleans at
1051 ** least one descriptor.
1052 ** Finally, anytime all descriptors are clean the timer is
1053 ** set to 0.
1054 **
1055 ** With TX Multiqueue we need to check every queue's timer,
1056 ** if any time out we do the reset.
1057 */
1064 default:
1065 error = ether_ioctl(ifp, command, data);
1066 break;
1067 }
1068
1069 return (error);
1070}
1071

--- 20 unchanged lines hidden (view full) ---

1092 ** Then txeof keeps resetting it as long as it cleans at
1093 ** least one descriptor.
1094 ** Finally, anytime all descriptors are clean the timer is
1095 ** set to 0.
1096 **
1097 ** With TX Multiqueue we need to check every queue's timer,
1098 ** if any time out we do the reset.
1099 */
1058 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
1100 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1059 IGB_TX_LOCK(txr);
1060 if (txr->watchdog_timer == 0 ||
1061 (--txr->watchdog_timer)) {
1062 IGB_TX_UNLOCK(txr);
1063 continue;
1064 } else {
1065 tx_hang = TRUE;
1066 IGB_TX_UNLOCK(txr);

--- 4 unchanged lines hidden (view full) ---

1071 return;
1072
1073 /* If we are in this routine because of pause frames, then
1074 * don't reset the hardware.
1075 */
1076 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1077 E1000_STATUS_TXOFF) {
1078 txr = adapter->tx_rings; /* reset pointer */
1101 IGB_TX_LOCK(txr);
1102 if (txr->watchdog_timer == 0 ||
1103 (--txr->watchdog_timer)) {
1104 IGB_TX_UNLOCK(txr);
1105 continue;
1106 } else {
1107 tx_hang = TRUE;
1108 IGB_TX_UNLOCK(txr);

--- 4 unchanged lines hidden (view full) ---

1113 return;
1114
1115 /* If we are in this routine because of pause frames, then
1116 * don't reset the hardware.
1117 */
1118 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1119 E1000_STATUS_TXOFF) {
1120 txr = adapter->tx_rings; /* reset pointer */
1079 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
1121 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1080 IGB_TX_LOCK(txr);
1081 txr->watchdog_timer = IGB_TX_TIMEOUT;
1082 IGB_TX_UNLOCK(txr);
1083 }
1084 return;
1085 }
1086
1087 if (e1000_check_for_link(&adapter->hw) == 0)
1088 device_printf(adapter->dev, "watchdog timeout -- resetting\n");
1089
1122 IGB_TX_LOCK(txr);
1123 txr->watchdog_timer = IGB_TX_TIMEOUT;
1124 IGB_TX_UNLOCK(txr);
1125 }
1126 return;
1127 }
1128
1129 if (e1000_check_for_link(&adapter->hw) == 0)
1130 device_printf(adapter->dev, "watchdog timeout -- resetting\n");
1131
1090 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
1132 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1091 device_printf(adapter->dev, "Queue(%d) tdh = %d, tdt = %d\n",
1092 i, E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
1093 E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
1094 device_printf(adapter->dev, "Queue(%d) desc avail = %d,"
1095 " Next Desc to Clean = %d\n", i, txr->tx_avail,
1096 txr->next_to_clean);
1097 }
1098

--- 51 unchanged lines hidden (view full) ---

1150 if (igb_hardware_init(adapter)) {
1151 device_printf(dev, "Unable to initialize the hardware\n");
1152 return;
1153 }
1154 igb_update_link_status(adapter);
1155
1156 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1157
1133 device_printf(adapter->dev, "Queue(%d) tdh = %d, tdt = %d\n",
1134 i, E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
1135 E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
1136 device_printf(adapter->dev, "Queue(%d) desc avail = %d,"
1137 " Next Desc to Clean = %d\n", i, txr->tx_avail,
1138 txr->next_to_clean);
1139 }
1140

--- 51 unchanged lines hidden (view full) ---

1192 if (igb_hardware_init(adapter)) {
1193 device_printf(dev, "Unable to initialize the hardware\n");
1194 return;
1195 }
1196 igb_update_link_status(adapter);
1197
1198 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1199
1158 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) &&
1159 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) {
1160 u32 ctrl;
1161 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1162 ctrl |= E1000_CTRL_VME;
1163 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1164 }
1165
1166 /* Set hardware offload abilities */
1167 ifp->if_hwassist = 0;
1168 if (ifp->if_capenable & IFCAP_TXCSUM) {
1169 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1170#if __FreeBSD_version >= 800000
1171 if (adapter->hw.mac.type == e1000_82576)
1172 ifp->if_hwassist |= CSUM_SCTP;
1173#endif

--- 36 unchanged lines hidden (view full) ---

1210 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1211
1212 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1213 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1214
1215 if (adapter->msix > 1) /* Set up queue routing */
1216 igb_configure_queues(adapter);
1217
1200 /* Set hardware offload abilities */
1201 ifp->if_hwassist = 0;
1202 if (ifp->if_capenable & IFCAP_TXCSUM) {
1203 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1204#if __FreeBSD_version >= 800000
1205 if (adapter->hw.mac.type == e1000_82576)
1206 ifp->if_hwassist |= CSUM_SCTP;
1207#endif

--- 36 unchanged lines hidden (view full) ---

1244 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1245
1246 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1247 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1248
1249 if (adapter->msix > 1) /* Set up queue routing */
1250 igb_configure_queues(adapter);
1251
1252 /* Set up VLAN tag offload and filter */
1253 igb_setup_vlan_hw_support(adapter);
1254
1218 /* Set default RX interrupt moderation */
1255 /* Set default RX interrupt moderation */
1219 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
1256 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
1220 E1000_WRITE_REG(&adapter->hw,
1221 E1000_EITR(rxr->msix), igb_ave_latency);
1222 rxr->eitr_setting = igb_ave_latency;
1223 }
1224
1225 /* Set TX interrupt rate & reset TX watchdog */
1257 E1000_WRITE_REG(&adapter->hw,
1258 E1000_EITR(rxr->msix), igb_ave_latency);
1259 rxr->eitr_setting = igb_ave_latency;
1260 }
1261
1262 /* Set TX interrupt rate & reset TX watchdog */
1226 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
1263 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1227 E1000_WRITE_REG(&adapter->hw,
1228 E1000_EITR(txr->msix), igb_ave_latency);
1229 txr->watchdog_timer = FALSE;
1230 }
1231
1264 E1000_WRITE_REG(&adapter->hw,
1265 E1000_EITR(txr->msix), igb_ave_latency);
1266 txr->watchdog_timer = FALSE;
1267 }
1268
1232 /* this clears any pending interrupts */
1233 E1000_READ_REG(&adapter->hw, E1000_ICR);
1234 igb_enable_intr(adapter);
1235 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1269 {
1270 /* this clears any pending interrupts */
1271 E1000_READ_REG(&adapter->hw, E1000_ICR);
1272 igb_enable_intr(adapter);
1273 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1274 }
1236
1275
1237#ifdef IGB_TIMESYNC
1238 /* Initialize IEEE 1588 Time sync if available */
1239 if (adapter->hw.mac.type == e1000_82576)
1240 igb_tsync_init(adapter);
1241#endif
1242
1243 /* Don't reset the phy next time init gets called */
1244 adapter->hw.phy.reset_disable = TRUE;
1245}
1246
1247static void
1248igb_init(void *arg)
1249{
1250 struct adapter *adapter = arg;

--- 15 unchanged lines hidden (view full) ---

1266 ifp = adapter->ifp;
1267
1268 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1269 if (igb_rxeof(rxr, adapter->rx_process_limit))
1270 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1271 IGB_TX_LOCK(txr);
1272 igb_txeof(txr);
1273
1276 /* Don't reset the phy next time init gets called */
1277 adapter->hw.phy.reset_disable = TRUE;
1278}
1279
1280static void
1281igb_init(void *arg)
1282{
1283 struct adapter *adapter = arg;

--- 15 unchanged lines hidden (view full) ---

1299 ifp = adapter->ifp;
1300
1301 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1302 if (igb_rxeof(rxr, adapter->rx_process_limit))
1303 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1304 IGB_TX_LOCK(txr);
1305 igb_txeof(txr);
1306
1307#if __FreeBSD_version >= 800000
1308 if (!drbr_empty(ifp, txr->br))
1309 igb_mq_start_locked(ifp, txr, NULL);
1310#else
1274 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1275 igb_start_locked(txr, ifp);
1311 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1312 igb_start_locked(txr, ifp);
1313#endif
1276 IGB_TX_UNLOCK(txr);
1277 }
1278
1279 igb_enable_intr(adapter);
1280}
1281
1282static void
1283igb_handle_rx(void *context, int pending)

--- 14 unchanged lines hidden (view full) ---

1298{
1299 struct tx_ring *txr = context;
1300 struct adapter *adapter = txr->adapter;
1301 struct ifnet *ifp = adapter->ifp;
1302
1303 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1304 IGB_TX_LOCK(txr);
1305 igb_txeof(txr);
1314 IGB_TX_UNLOCK(txr);
1315 }
1316
1317 igb_enable_intr(adapter);
1318}
1319
1320static void
1321igb_handle_rx(void *context, int pending)

--- 14 unchanged lines hidden (view full) ---

1336{
1337 struct tx_ring *txr = context;
1338 struct adapter *adapter = txr->adapter;
1339 struct ifnet *ifp = adapter->ifp;
1340
1341 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1342 IGB_TX_LOCK(txr);
1343 igb_txeof(txr);
1344#if __FreeBSD_version >= 800000
1345 if (!drbr_empty(ifp, txr->br))
1346 igb_mq_start_locked(ifp, txr, NULL);
1347#else
1306 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1307 igb_start_locked(txr, ifp);
1348 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1349 igb_start_locked(txr, ifp);
1350#endif
1308 IGB_TX_UNLOCK(txr);
1309 }
1310}
1311
1312
1313/*********************************************************************
1314 *
1315 * MSI/Legacy Deferred
1316 * Interrupt Service routine
1317 *
1318 *********************************************************************/
1351 IGB_TX_UNLOCK(txr);
1352 }
1353}
1354
1355
1356/*********************************************************************
1357 *
1358 * MSI/Legacy Deferred
1359 * Interrupt Service routine
1360 *
1361 *********************************************************************/
1319#if __FreeBSD_version < 700000
1320static void
1321#else
1322static int
1362static int
1323#endif
1324igb_irq_fast(void *arg)
1325{
1326 struct adapter *adapter = arg;
1327 uint32_t reg_icr;
1328
1329
1330 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1331

--- 23 unchanged lines hidden (view full) ---

1355 }
1356
1357 if (reg_icr & E1000_ICR_RXO)
1358 adapter->rx_overruns++;
1359 return FILTER_HANDLED;
1360}
1361
1362
1363igb_irq_fast(void *arg)
1364{
1365 struct adapter *adapter = arg;
1366 uint32_t reg_icr;
1367
1368
1369 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1370

--- 23 unchanged lines hidden (view full) ---

1394 }
1395
1396 if (reg_icr & E1000_ICR_RXO)
1397 adapter->rx_overruns++;
1398 return FILTER_HANDLED;
1399}
1400
1401
1363#if __FreeBSD_version >= 602105
1364/*********************************************************************
1365 *
1366 * MSIX TX Interrupt Service routine
1367 *
1368 **********************************************************************/
1369static void
1370igb_msix_tx(void *arg)
1371{

--- 71 unchanged lines hidden (view full) ---

1443 igb_update_link_status(adapter);
1444
1445spurious:
1446 /* Rearm */
1447 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1448 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1449 return;
1450}
1402/*********************************************************************
1403 *
1404 * MSIX TX Interrupt Service routine
1405 *
1406 **********************************************************************/
1407static void
1408igb_msix_tx(void *arg)
1409{

--- 71 unchanged lines hidden (view full) ---

1481 igb_update_link_status(adapter);
1482
1483spurious:
1484 /* Rearm */
1485 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1486 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1487 return;
1488}
1451#endif
1452
1453
1454/*
1455** Routine to adjust the RX EITR value based on traffic,
1456** its a simple three state model, but seems to help.
1457**
1458** Note that the three EITR values are tuneable using
1459** sysctl in real time. The feature can be effectively

--- 169 unchanged lines hidden (view full) ---

1629 struct adapter *adapter = txr->adapter;
1630 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1631 bus_dmamap_t map;
1632 struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped;
1633 union e1000_adv_tx_desc *txd = NULL;
1634 struct mbuf *m_head;
1635 u32 olinfo_status = 0, cmd_type_len = 0;
1636 int nsegs, i, j, error, first, last = 0;
1489
1490
1491/*
1492** Routine to adjust the RX EITR value based on traffic,
1493** its a simple three state model, but seems to help.
1494**
1495** Note that the three EITR values are tuneable using
1496** sysctl in real time. The feature can be effectively

--- 169 unchanged lines hidden (view full) ---

1666 struct adapter *adapter = txr->adapter;
1667 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1668 bus_dmamap_t map;
1669 struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped;
1670 union e1000_adv_tx_desc *txd = NULL;
1671 struct mbuf *m_head;
1672 u32 olinfo_status = 0, cmd_type_len = 0;
1673 int nsegs, i, j, error, first, last = 0;
1637 u32 hdrlen = 0, offload = 0;
1674 u32 hdrlen = 0;
1638
1639 m_head = *m_headp;
1640
1641
1642 /* Set basic descriptor constants */
1643 cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
1644 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
1645 if (m_head->m_flags & M_VLANTAG)

--- 79 unchanged lines hidden (view full) ---

1725 */
1726 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1727 if (igb_tso_setup(txr, m_head, &hdrlen)) {
1728 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1729 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
1730 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1731 } else
1732 return (ENXIO);
1675
1676 m_head = *m_headp;
1677
1678
1679 /* Set basic descriptor constants */
1680 cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
1681 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
1682 if (m_head->m_flags & M_VLANTAG)

--- 79 unchanged lines hidden (view full) ---

1762 */
1763 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1764 if (igb_tso_setup(txr, m_head, &hdrlen)) {
1765 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1766 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
1767 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1768 } else
1769 return (ENXIO);
1733 } else
1734 /* Do all other context descriptor setup */
1735 offload = igb_tx_ctx_setup(txr, m_head);
1736 if (offload == TRUE)
1770 } else if (igb_tx_ctx_setup(txr, m_head))
1737 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1771 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1738#ifdef IGB_TIMESYNC
1739 if (offload == IGB_TIMESTAMP)
1772
1773#ifdef IGB_IEEE1588
1774 /* This is changing soon to an mtag detection */
1775 if (we detect this mbuf has a TSTAMP mtag)
1740 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
1741#endif
1742 /* Calculate payload length */
1743 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
1744 << E1000_ADVTXD_PAYLEN_SHIFT);
1745
1746 /* Set up our transmit descriptors */
1747 i = txr->next_avail_desc;

--- 207 unchanged lines hidden (view full) ---

1955 } else if (!link_check && (adapter->link_active == 1)) {
1956 ifp->if_baudrate = adapter->link_speed = 0;
1957 adapter->link_duplex = 0;
1958 if (bootverbose)
1959 device_printf(dev, "Link is Down\n");
1960 adapter->link_active = 0;
1961 if_link_state_change(ifp, LINK_STATE_DOWN);
1962 /* Turn off watchdogs */
1776 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
1777#endif
1778 /* Calculate payload length */
1779 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
1780 << E1000_ADVTXD_PAYLEN_SHIFT);
1781
1782 /* Set up our transmit descriptors */
1783 i = txr->next_avail_desc;

--- 207 unchanged lines hidden (view full) ---

1991 } else if (!link_check && (adapter->link_active == 1)) {
1992 ifp->if_baudrate = adapter->link_speed = 0;
1993 adapter->link_duplex = 0;
1994 if (bootverbose)
1995 device_printf(dev, "Link is Down\n");
1996 adapter->link_active = 0;
1997 if_link_state_change(ifp, LINK_STATE_DOWN);
1998 /* Turn off watchdogs */
1963 for (int i = 0; i < adapter->num_tx_queues; i++, txr++)
1999 for (int i = 0; i < adapter->num_queues; i++, txr++)
1964 txr->watchdog_timer = FALSE;
1965 }
1966}
1967
1968/*********************************************************************
1969 *
1970 * This routine disables all traffic on the adapter by issuing a
1971 * global reset on the MAC and deallocates TX/RX buffers.

--- 12 unchanged lines hidden (view full) ---

1984
1985 igb_disable_intr(adapter);
1986
1987 callout_stop(&adapter->timer);
1988
1989 /* Tell the stack that the interface is no longer active */
1990 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1991
2000 txr->watchdog_timer = FALSE;
2001 }
2002}
2003
2004/*********************************************************************
2005 *
2006 * This routine disables all traffic on the adapter by issuing a
2007 * global reset on the MAC and deallocates TX/RX buffers.

--- 12 unchanged lines hidden (view full) ---

2020
2021 igb_disable_intr(adapter);
2022
2023 callout_stop(&adapter->timer);
2024
2025 /* Tell the stack that the interface is no longer active */
2026 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2027
1992#ifdef IGB_TIMESYNC
1993 /* Disable IEEE 1588 Time sync */
1994 if (adapter->hw.mac.type == e1000_82576)
1995 igb_tsync_disable(adapter);
1996#endif
1997
1998 e1000_reset_hw(&adapter->hw);
1999 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2000}
2001
2002
2003/*********************************************************************
2004 *
2005 * Determine hardware revision.

--- 31 unchanged lines hidden (view full) ---

2037 return;
2038 }
2039}
2040
2041static int
2042igb_allocate_pci_resources(struct adapter *adapter)
2043{
2044 device_t dev = adapter->dev;
2028 e1000_reset_hw(&adapter->hw);
2029 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2030}
2031
2032
2033/*********************************************************************
2034 *
2035 * Determine hardware revision.

--- 31 unchanged lines hidden (view full) ---

2067 return;
2068 }
2069}
2070
2071static int
2072igb_allocate_pci_resources(struct adapter *adapter)
2073{
2074 device_t dev = adapter->dev;
2045 int rid, error = 0;
2075 int rid;
2046
2047 rid = PCIR_BAR(0);
2048 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2049 &rid, RF_ACTIVE);
2050 if (adapter->pci_mem == NULL) {
2051 device_printf(dev, "Unable to allocate bus resource: memory\n");
2052 return (ENXIO);
2053 }
2054 adapter->osdep.mem_bus_space_tag =
2055 rman_get_bustag(adapter->pci_mem);
2056 adapter->osdep.mem_bus_space_handle =
2057 rman_get_bushandle(adapter->pci_mem);
2076
2077 rid = PCIR_BAR(0);
2078 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2079 &rid, RF_ACTIVE);
2080 if (adapter->pci_mem == NULL) {
2081 device_printf(dev, "Unable to allocate bus resource: memory\n");
2082 return (ENXIO);
2083 }
2084 adapter->osdep.mem_bus_space_tag =
2085 rman_get_bustag(adapter->pci_mem);
2086 adapter->osdep.mem_bus_space_handle =
2087 rman_get_bushandle(adapter->pci_mem);
2058 adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
2088 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2059
2089
2060 /*
2061 ** Init the resource arrays
2062 */
2063 for (int i = 0; i < IGB_MSIX_VEC; i++) {
2064 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2065 adapter->tag[i] = NULL;
2066 adapter->res[i] = NULL;
2067 }
2090 adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2068
2091
2069 adapter->num_tx_queues = 1; /* Defaults for Legacy or MSI */
2070 adapter->num_rx_queues = 1;
2071
2072#if __FreeBSD_version >= 602105
2073 /* This will setup either MSI/X or MSI */
2074 adapter->msix = igb_setup_msix(adapter);
2092 /* This will setup either MSI/X or MSI */
2093 adapter->msix = igb_setup_msix(adapter);
2075#endif
2076 adapter->hw.back = &adapter->osdep;
2077
2094 adapter->hw.back = &adapter->osdep;
2095
2078 return (error);
2096 return (0);
2079}
2080
2081/*********************************************************************
2082 *
2083 * Setup the Legacy or MSI Interrupt handler
2084 *
2085 **********************************************************************/
2086static int
2087igb_allocate_legacy(struct adapter *adapter)
2088{
2089 device_t dev = adapter->dev;
2097}
2098
2099/*********************************************************************
2100 *
2101 * Setup the Legacy or MSI Interrupt handler
2102 *
2103 **********************************************************************/
2104static int
2105igb_allocate_legacy(struct adapter *adapter)
2106{
2107 device_t dev = adapter->dev;
2090 int error;
2108 int error, rid = 0;
2091
2092 /* Turn off all interrupts */
2093 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2094
2109
2110 /* Turn off all interrupts */
2111 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2112
2095 /* Legacy RID at 0 */
2096 if (adapter->msix == 0)
2097 adapter->rid[0] = 0;
2113 /* MSI RID is 1 */
2114 if (adapter->msix == 1)
2115 rid = 1;
2098
2099 /* We allocate a single interrupt resource */
2116
2117 /* We allocate a single interrupt resource */
2100 adapter->res[0] = bus_alloc_resource_any(dev,
2101 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2102 if (adapter->res[0] == NULL) {
2118 adapter->res = bus_alloc_resource_any(dev,
2119 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2120 if (adapter->res == NULL) {
2103 device_printf(dev, "Unable to allocate bus resource: "
2104 "interrupt\n");
2105 return (ENXIO);
2106 }
2107
2108 /*
2109 * Try allocating a fast interrupt and the associated deferred
2110 * processing contexts.
2111 */
2112 TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter);
2113 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2114 taskqueue_thread_enqueue, &adapter->tq);
2115 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2116 device_get_nameunit(adapter->dev));
2121 device_printf(dev, "Unable to allocate bus resource: "
2122 "interrupt\n");
2123 return (ENXIO);
2124 }
2125
2126 /*
2127 * Try allocating a fast interrupt and the associated deferred
2128 * processing contexts.
2129 */
2130 TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter);
2131 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2132 taskqueue_thread_enqueue, &adapter->tq);
2133 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2134 device_get_nameunit(adapter->dev));
2117 if ((error = bus_setup_intr(dev, adapter->res[0],
2118 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast,
2119#if __FreeBSD_version >= 700000
2120 NULL,
2121#endif
2122 adapter, &adapter->tag[0])) != 0) {
2135 if ((error = bus_setup_intr(dev, adapter->res,
2136 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2137 adapter, &adapter->tag)) != 0) {
2123 device_printf(dev, "Failed to register fast interrupt "
2124 "handler: %d\n", error);
2125 taskqueue_free(adapter->tq);
2126 adapter->tq = NULL;
2127 return (error);
2128 }
2129
2130 return (0);
2131}
2132
2133
2138 device_printf(dev, "Failed to register fast interrupt "
2139 "handler: %d\n", error);
2140 taskqueue_free(adapter->tq);
2141 adapter->tq = NULL;
2142 return (error);
2143 }
2144
2145 return (0);
2146}
2147
2148
2134#if __FreeBSD_version >= 602105
2135/*********************************************************************
2136 *
2137 * Setup the MSIX Interrupt handlers:
2138 *
2139 **********************************************************************/
2140static int
2141igb_allocate_msix(struct adapter *adapter)
2142{
2143 device_t dev = adapter->dev;
2144 struct tx_ring *txr = adapter->tx_rings;
2145 struct rx_ring *rxr = adapter->rx_rings;
2149/*********************************************************************
2150 *
2151 * Setup the MSIX Interrupt handlers:
2152 *
2153 **********************************************************************/
2154static int
2155igb_allocate_msix(struct adapter *adapter)
2156{
2157 device_t dev = adapter->dev;
2158 struct tx_ring *txr = adapter->tx_rings;
2159 struct rx_ring *rxr = adapter->rx_rings;
2146 int error, vector = 0;
2160 int error, rid, vector = 0;
2147
2148 /*
2149 * Setup the interrupt handlers
2150 */
2151
2152 /* TX Setup */
2161
2162 /*
2163 * Setup the interrupt handlers
2164 */
2165
2166 /* TX Setup */
2153 for (int i = 0; i < adapter->num_tx_queues; i++, vector++, txr++) {
2154 adapter->res[vector] = bus_alloc_resource_any(dev,
2155 SYS_RES_IRQ, &adapter->rid[vector],
2156 RF_SHAREABLE | RF_ACTIVE);
2157 if (adapter->res[vector] == NULL) {
2167 for (int i = 0; i < adapter->num_queues; i++, vector++, txr++) {
2168 rid = vector +1;
2169 txr->res = bus_alloc_resource_any(dev,
2170 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2171 if (txr->res == NULL) {
2158 device_printf(dev,
2159 "Unable to allocate bus resource: "
2160 "MSIX TX Interrupt\n");
2161 return (ENXIO);
2162 }
2172 device_printf(dev,
2173 "Unable to allocate bus resource: "
2174 "MSIX TX Interrupt\n");
2175 return (ENXIO);
2176 }
2163 error = bus_setup_intr(dev, adapter->res[vector],
2164 INTR_TYPE_NET | INTR_MPSAFE,
2165#if __FreeBSD_version >= 700000
2166 NULL,
2167#endif
2168 igb_msix_tx, txr, &adapter->tag[vector]);
2177 error = bus_setup_intr(dev, txr->res,
2178 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2179 igb_msix_tx, txr, &txr->tag);
2169 if (error) {
2180 if (error) {
2170 adapter->res[vector] = NULL;
2181 txr->res = NULL;
2171 device_printf(dev, "Failed to register TX handler");
2172 return (error);
2173 }
2174 /* Make tasklet for deferred handling - one per queue */
2175 TASK_INIT(&txr->tx_task, 0, igb_handle_tx, txr);
2182 device_printf(dev, "Failed to register TX handler");
2183 return (error);
2184 }
2185 /* Make tasklet for deferred handling - one per queue */
2186 TASK_INIT(&txr->tx_task, 0, igb_handle_tx, txr);
2176 if (adapter->hw.mac.type == e1000_82575) {
2187 txr->msix = vector;
2188 if (adapter->hw.mac.type == e1000_82575)
2177 txr->eims = E1000_EICR_TX_QUEUE0 << i;
2189 txr->eims = E1000_EICR_TX_QUEUE0 << i;
2178 /* MSIXBM registers start at 0 */
2179 txr->msix = adapter->rid[vector] - 1;
2180 } else {
2190 else
2181 txr->eims = 1 << vector;
2191 txr->eims = 1 << vector;
2182 txr->msix = vector;
2183 }
2192#if __FreeBSD_version >= 800000
2193 /*
2194 ** Bind the msix vector, and thus the
2195 ** ring to the corresponding cpu.
2196 */
2197 intr_bind(rman_get_start(txr->res), i);
2198#endif
2184 }
2185
2186 /* RX Setup */
2199 }
2200
2201 /* RX Setup */
2187 for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rxr++) {
2188 adapter->res[vector] = bus_alloc_resource_any(dev,
2189 SYS_RES_IRQ, &adapter->rid[vector],
2190 RF_SHAREABLE | RF_ACTIVE);
2191 if (adapter->res[vector] == NULL) {
2202 for (int i = 0; i < adapter->num_queues; i++, vector++, rxr++) {
2203 rid = vector +1;
2204 rxr->res = bus_alloc_resource_any(dev,
2205 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2206 if (rxr->res == NULL) {
2192 device_printf(dev,
2193 "Unable to allocate bus resource: "
2194 "MSIX RX Interrupt\n");
2195 return (ENXIO);
2196 }
2207 device_printf(dev,
2208 "Unable to allocate bus resource: "
2209 "MSIX RX Interrupt\n");
2210 return (ENXIO);
2211 }
2197 error = bus_setup_intr(dev, adapter->res[vector],
2198 INTR_TYPE_NET | INTR_MPSAFE,
2199#if __FreeBSD_version >= 700000
2200 NULL,
2201#endif
2202 igb_msix_rx, rxr, &adapter->tag[vector]);
2212 error = bus_setup_intr(dev, rxr->res,
2213 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2214 igb_msix_rx, rxr, &rxr->tag);
2203 if (error) {
2215 if (error) {
2204 adapter->res[vector] = NULL;
2216 rxr->res = NULL;
2205 device_printf(dev, "Failed to register RX handler");
2206 return (error);
2207 }
2208 /* Make tasklet for deferred handling - one per queue */
2209 TASK_INIT(&rxr->rx_task, 0, igb_handle_rx, rxr);
2217 device_printf(dev, "Failed to register RX handler");
2218 return (error);
2219 }
2220 /* Make tasklet for deferred handling - one per queue */
2221 TASK_INIT(&rxr->rx_task, 0, igb_handle_rx, rxr);
2210 if (adapter->hw.mac.type == e1000_82575) {
2222 rxr->msix = vector;
2223 if (adapter->hw.mac.type == e1000_82575)
2211 rxr->eims = E1000_EICR_RX_QUEUE0 << i;
2224 rxr->eims = E1000_EICR_RX_QUEUE0 << i;
2212 rxr->msix = adapter->rid[vector] - 1;
2213 } else {
2225 else
2214 rxr->eims = 1 << vector;
2226 rxr->eims = 1 << vector;
2215 rxr->msix = vector;
2216 }
2217 /* Get a mask for local timer */
2218 adapter->rx_mask |= rxr->eims;
2227 /* Get a mask for local timer */
2228 adapter->rx_mask |= rxr->eims;
2229#if __FreeBSD_version >= 800000
2230 /*
2231 ** Bind the msix vector, and thus the
2232 ** ring to the corresponding cpu.
2233 ** Notice that this makes an RX/TX pair
2234 ** bound to each CPU, limited by the MSIX
2235 ** vectors.
2236 */
2237 intr_bind(rman_get_start(rxr->res), i);
2238#endif
2219 }
2220
2221 /* And Link */
2239 }
2240
2241 /* And Link */
2222 adapter->res[vector] = bus_alloc_resource_any(dev,
2223 SYS_RES_IRQ, &adapter->rid[vector],
2224 RF_SHAREABLE | RF_ACTIVE);
2225 if (adapter->res[vector] == NULL) {
2242 rid = vector +1;
2243 adapter->res = bus_alloc_resource_any(dev,
2244 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2245 if (adapter->res == NULL) {
2226 device_printf(dev,
2227 "Unable to allocate bus resource: "
2228 "MSIX Link Interrupt\n");
2229 return (ENXIO);
2230 }
2246 device_printf(dev,
2247 "Unable to allocate bus resource: "
2248 "MSIX Link Interrupt\n");
2249 return (ENXIO);
2250 }
2231 if ((error = bus_setup_intr(dev, adapter->res[vector],
2232 INTR_TYPE_NET | INTR_MPSAFE,
2233#if __FreeBSD_version >= 700000
2234 NULL,
2235#endif
2236 igb_msix_link, adapter, &adapter->tag[vector])) != 0) {
2251 if ((error = bus_setup_intr(dev, adapter->res,
2252 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2253 igb_msix_link, adapter, &adapter->tag)) != 0) {
2237 device_printf(dev, "Failed to register Link handler");
2238 return (error);
2239 }
2254 device_printf(dev, "Failed to register Link handler");
2255 return (error);
2256 }
2240 if (adapter->hw.mac.type == e1000_82575)
2241 adapter->linkvec = adapter->rid[vector] - 1;
2242 else
2243 adapter->linkvec = vector;
2244
2257 adapter->linkvec = vector;
2245 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2246 taskqueue_thread_enqueue, &adapter->tq);
2247 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2248 device_get_nameunit(adapter->dev));
2249
2250 return (0);
2251}
2258 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2259 taskqueue_thread_enqueue, &adapter->tq);
2260 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2261 device_get_nameunit(adapter->dev));
2262
2263 return (0);
2264}
2252#else /* FreeBSD 6.1/2 */
2253static int
2254igb_allocate_msix(struct adapter *adapter)
2255{
2256 return (1);
2257}
2258#endif
2259
2260
2261static void
2262igb_configure_queues(struct adapter *adapter)
2263{
2264 struct e1000_hw *hw = &adapter->hw;
2265 struct tx_ring *txr;
2266 struct rx_ring *rxr;

--- 7 unchanged lines hidden (view full) ---

2274 if (adapter->hw.mac.type == e1000_82576) {
2275 u32 ivar = 0;
2276 /* First turn on the capability */
2277 E1000_WRITE_REG(hw, E1000_GPIE,
2278 E1000_GPIE_MSIX_MODE |
2279 E1000_GPIE_EIAME |
2280 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2281 /* RX */
2265
2266
2267static void
2268igb_configure_queues(struct adapter *adapter)
2269{
2270 struct e1000_hw *hw = &adapter->hw;
2271 struct tx_ring *txr;
2272 struct rx_ring *rxr;

--- 7 unchanged lines hidden (view full) ---

2280 if (adapter->hw.mac.type == e1000_82576) {
2281 u32 ivar = 0;
2282 /* First turn on the capability */
2283 E1000_WRITE_REG(hw, E1000_GPIE,
2284 E1000_GPIE_MSIX_MODE |
2285 E1000_GPIE_EIAME |
2286 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2287 /* RX */
2282 for (int i = 0; i < adapter->num_rx_queues; i++) {
2288 for (int i = 0; i < adapter->num_queues; i++) {
2283 u32 index = i & 0x7; /* Each IVAR has two entries */
2284 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2285 rxr = &adapter->rx_rings[i];
2286 if (i < 8) {
2287 ivar &= 0xFFFFFF00;
2288 ivar |= rxr->msix | E1000_IVAR_VALID;
2289 } else {
2290 ivar &= 0xFF00FFFF;
2291 ivar |= (rxr->msix | E1000_IVAR_VALID) << 16;
2292 }
2293 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2294 adapter->eims_mask |= rxr->eims;
2295 }
2296 /* TX */
2289 u32 index = i & 0x7; /* Each IVAR has two entries */
2290 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2291 rxr = &adapter->rx_rings[i];
2292 if (i < 8) {
2293 ivar &= 0xFFFFFF00;
2294 ivar |= rxr->msix | E1000_IVAR_VALID;
2295 } else {
2296 ivar &= 0xFF00FFFF;
2297 ivar |= (rxr->msix | E1000_IVAR_VALID) << 16;
2298 }
2299 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2300 adapter->eims_mask |= rxr->eims;
2301 }
2302 /* TX */
2297 for (int i = 0; i < adapter->num_tx_queues; i++) {
2303 for (int i = 0; i < adapter->num_queues; i++) {
2298 u32 index = i & 0x7; /* Each IVAR has two entries */
2299 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2300 txr = &adapter->tx_rings[i];
2301 if (i < 8) {
2302 ivar &= 0xFFFF00FF;
2303 ivar |= (txr->msix | E1000_IVAR_VALID) << 8;
2304 } else {
2305 ivar &= 0x00FFFFFF;

--- 16 unchanged lines hidden (view full) ---

2322 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2323 tmp |= E1000_CTRL_EXT_PBA_CLR;
2324 /* Auto-Mask interrupts upon ICR read. */
2325 tmp |= E1000_CTRL_EXT_EIAME;
2326 tmp |= E1000_CTRL_EXT_IRCA;
2327 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2328
2329 /* TX */
2304 u32 index = i & 0x7; /* Each IVAR has two entries */
2305 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2306 txr = &adapter->tx_rings[i];
2307 if (i < 8) {
2308 ivar &= 0xFFFF00FF;
2309 ivar |= (txr->msix | E1000_IVAR_VALID) << 8;
2310 } else {
2311 ivar &= 0x00FFFFFF;

--- 16 unchanged lines hidden (view full) ---

2328 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2329 tmp |= E1000_CTRL_EXT_PBA_CLR;
2330 /* Auto-Mask interrupts upon ICR read. */
2331 tmp |= E1000_CTRL_EXT_EIAME;
2332 tmp |= E1000_CTRL_EXT_IRCA;
2333 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2334
2335 /* TX */
2330 for (int i = 0; i < adapter->num_tx_queues; i++) {
2336 for (int i = 0; i < adapter->num_queues; i++) {
2331 txr = &adapter->tx_rings[i];
2332 E1000_WRITE_REG(hw, E1000_MSIXBM(txr->msix),
2333 txr->eims);
2334 adapter->eims_mask |= txr->eims;
2335 }
2336
2337 /* RX */
2337 txr = &adapter->tx_rings[i];
2338 E1000_WRITE_REG(hw, E1000_MSIXBM(txr->msix),
2339 txr->eims);
2340 adapter->eims_mask |= txr->eims;
2341 }
2342
2343 /* RX */
2338 for (int i = 0; i < adapter->num_rx_queues; i++) {
2344 for (int i = 0; i < adapter->num_queues; i++) {
2339 rxr = &adapter->rx_rings[i];
2340 E1000_WRITE_REG(hw, E1000_MSIXBM(rxr->msix),
2341 rxr->eims);
2342 adapter->eims_mask |= rxr->eims;
2343 }
2344
2345 /* Link */
2346 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2347 E1000_EIMS_OTHER);
2348 adapter->link_mask |= E1000_EIMS_OTHER;
2349 adapter->eims_mask |= adapter->link_mask;
2350 }
2351 return;
2352}
2353
2354
2355static void
2356igb_free_pci_resources(struct adapter *adapter)
2357{
2345 rxr = &adapter->rx_rings[i];
2346 E1000_WRITE_REG(hw, E1000_MSIXBM(rxr->msix),
2347 rxr->eims);
2348 adapter->eims_mask |= rxr->eims;
2349 }
2350
2351 /* Link */
2352 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2353 E1000_EIMS_OTHER);
2354 adapter->link_mask |= E1000_EIMS_OTHER;
2355 adapter->eims_mask |= adapter->link_mask;
2356 }
2357 return;
2358}
2359
2360
2361static void
2362igb_free_pci_resources(struct adapter *adapter)
2363{
2358 device_t dev = adapter->dev;
2364 struct tx_ring *txr = adapter->tx_rings;
2365 struct rx_ring *rxr = adapter->rx_rings;
2366 device_t dev = adapter->dev;
2367 int rid;
2359
2368
2360 /* Make sure the for loop below runs once */
2361 if (adapter->msix == 0)
2362 adapter->msix = 1;
2369 /*
2370 ** There is a slight possibility of a failure mode
2371 ** in attach that will result in entering this function
2372 ** before interrupt resources have been initialized, and
2373 ** in that case we do not want to execute the loops below
2374 ** We can detect this reliably by the state of the adapter
2375 ** res pointer.
2376 */
2377 if (adapter->res == NULL)
2378 goto mem;
2363
2364 /*
2379
2380 /*
2365 * First release all the interrupt resources:
2366 * notice that since these are just kept
2367 * in an array we can do the same logic
2368 * whether its MSIX or just legacy.
2381 * First release all the TX/RX interrupt resources:
2369 */
2382 */
2370 for (int i = 0; i < adapter->msix; i++) {
2371 if (adapter->tag[i] != NULL) {
2372 bus_teardown_intr(dev, adapter->res[i],
2373 adapter->tag[i]);
2374 adapter->tag[i] = NULL;
2383 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2384 rid = txr->msix + 1;
2385 if (txr->tag != NULL) {
2386 bus_teardown_intr(dev, txr->res, txr->tag);
2387 txr->tag = NULL;
2375 }
2388 }
2376 if (adapter->res[i] != NULL) {
2377 bus_release_resource(dev, SYS_RES_IRQ,
2378 adapter->rid[i], adapter->res[i]);
2389 if (txr->res != NULL)
2390 bus_release_resource(dev, SYS_RES_IRQ, rid, txr->res);
2391 }
2392
2393 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
2394 rid = rxr->msix + 1;
2395 if (rxr->tag != NULL) {
2396 bus_teardown_intr(dev, rxr->res, rxr->tag);
2397 rxr->tag = NULL;
2379 }
2398 }
2399 if (rxr->res != NULL)
2400 bus_release_resource(dev, SYS_RES_IRQ, rid, rxr->res);
2380 }
2381
2401 }
2402
2382#if __FreeBSD_version >= 602105
2403 /* Clean the Legacy or Link interrupt last */
2404 if (adapter->linkvec) /* we are doing MSIX */
2405 rid = adapter->linkvec + 1;
2406 else
2407 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2408
2409 if (adapter->tag != NULL) {
2410 bus_teardown_intr(dev, adapter->res, adapter->tag);
2411 adapter->tag = NULL;
2412 }
2413 if (adapter->res != NULL)
2414 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2415
2416mem:
2383 if (adapter->msix)
2384 pci_release_msi(dev);
2385
2386 if (adapter->msix_mem != NULL)
2387 bus_release_resource(dev, SYS_RES_MEMORY,
2388 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2417 if (adapter->msix)
2418 pci_release_msi(dev);
2419
2420 if (adapter->msix_mem != NULL)
2421 bus_release_resource(dev, SYS_RES_MEMORY,
2422 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2389#endif
2390
2391 if (adapter->pci_mem != NULL)
2392 bus_release_resource(dev, SYS_RES_MEMORY,
2393 PCIR_BAR(0), adapter->pci_mem);
2394
2395}
2396
2423
2424 if (adapter->pci_mem != NULL)
2425 bus_release_resource(dev, SYS_RES_MEMORY,
2426 PCIR_BAR(0), adapter->pci_mem);
2427
2428}
2429
2397#if __FreeBSD_version >= 602105
2398/*
2399 * Setup Either MSI/X or MSI
2400 */
2401static int
2402igb_setup_msix(struct adapter *adapter)
2403{
2404 device_t dev = adapter->dev;
2405 int rid, want, queues, msgs;

--- 12 unchanged lines hidden (view full) ---

2418 msgs = pci_msix_count(dev);
2419 if (msgs == 0) { /* system has msix disabled */
2420 bus_release_resource(dev, SYS_RES_MEMORY,
2421 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2422 adapter->msix_mem = NULL;
2423 goto msi;
2424 }
2425
2430/*
2431 * Setup Either MSI/X or MSI
2432 */
2433static int
2434igb_setup_msix(struct adapter *adapter)
2435{
2436 device_t dev = adapter->dev;
2437 int rid, want, queues, msgs;

--- 12 unchanged lines hidden (view full) ---

2450 msgs = pci_msix_count(dev);
2451 if (msgs == 0) { /* system has msix disabled */
2452 bus_release_resource(dev, SYS_RES_MEMORY,
2453 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2454 adapter->msix_mem = NULL;
2455 goto msi;
2456 }
2457
2426 /* Limit by the number set in header */
2427 if (msgs > IGB_MSIX_VEC)
2428 msgs = IGB_MSIX_VEC;
2429
2430 /* Figure out a reasonable auto config value */
2431 queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus;
2432
2458 /* Figure out a reasonable auto config value */
2459 queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus;
2460
2433 if (igb_tx_queues == 0)
2434 igb_tx_queues = queues;
2435 if (igb_rx_queues == 0)
2436 igb_rx_queues = queues;
2437 want = igb_tx_queues + igb_rx_queues + 1;
2461 if (igb_num_queues == 0)
2462 igb_num_queues = queues;
2463 /*
2464 ** Two vectors (RX/TX pair) per queue
2465 ** plus an additional for Link interrupt
2466 */
2467 want = (igb_num_queues * 2) + 1;
2438 if (msgs >= want)
2439 msgs = want;
2440 else {
2441 device_printf(adapter->dev,
2442 "MSIX Configuration Problem, "
2443 "%d vectors configured, but %d queues wanted!\n",
2444 msgs, want);
2445 return (ENXIO);
2446 }
2447 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2448 device_printf(adapter->dev,
2449 "Using MSIX interrupts with %d vectors\n", msgs);
2468 if (msgs >= want)
2469 msgs = want;
2470 else {
2471 device_printf(adapter->dev,
2472 "MSIX Configuration Problem, "
2473 "%d vectors configured, but %d queues wanted!\n",
2474 msgs, want);
2475 return (ENXIO);
2476 }
2477 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2478 device_printf(adapter->dev,
2479 "Using MSIX interrupts with %d vectors\n", msgs);
2450 adapter->num_tx_queues = igb_tx_queues;
2451 adapter->num_rx_queues = igb_rx_queues;
2480 adapter->num_queues = igb_num_queues;
2452 return (msgs);
2453 }
2454msi:
2455 msgs = pci_msi_count(dev);
2456 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2457 device_printf(adapter->dev,"Using MSI interrupt\n");
2458 return (msgs);
2459}
2481 return (msgs);
2482 }
2483msi:
2484 msgs = pci_msi_count(dev);
2485 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2486 device_printf(adapter->dev,"Using MSI interrupt\n");
2487 return (msgs);
2488}
2460#endif /* __FreeBSD_version >= 602105 */
2461
2462/*********************************************************************
2463 *
2464 * Initialize the hardware to a configuration
2465 * as specified by the adapter structure.
2466 *
2467 **********************************************************************/
2468static int

--- 71 unchanged lines hidden (view full) ---

2540 panic("%s: can not if_alloc()", device_get_nameunit(dev));
2541 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2542 ifp->if_mtu = ETHERMTU;
2543 ifp->if_init = igb_init;
2544 ifp->if_softc = adapter;
2545 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2546 ifp->if_ioctl = igb_ioctl;
2547 ifp->if_start = igb_start;
2489
2490/*********************************************************************
2491 *
2492 * Initialize the hardware to a configuration
2493 * as specified by the adapter structure.
2494 *
2495 **********************************************************************/
2496static int

--- 71 unchanged lines hidden (view full) ---

2568 panic("%s: can not if_alloc()", device_get_nameunit(dev));
2569 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2570 ifp->if_mtu = ETHERMTU;
2571 ifp->if_init = igb_init;
2572 ifp->if_softc = adapter;
2573 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2574 ifp->if_ioctl = igb_ioctl;
2575 ifp->if_start = igb_start;
2576#if __FreeBSD_version >= 800000
2577 ifp->if_transmit = igb_mq_start;
2578 ifp->if_qflush = igb_qflush;
2579#endif
2548 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2549 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2550 IFQ_SET_READY(&ifp->if_snd);
2551
2552 ether_ifattach(ifp, adapter->hw.mac.addr);
2553
2554 ifp->if_capabilities = ifp->if_capenable = 0;
2555
2580 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2581 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2582 IFQ_SET_READY(&ifp->if_snd);
2583
2584 ether_ifattach(ifp, adapter->hw.mac.addr);
2585
2586 ifp->if_capabilities = ifp->if_capenable = 0;
2587
2556 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2557 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_VLAN_MTU;
2588 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU;
2589 ifp->if_capabilities |= IFCAP_TSO4;
2558 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2559 ifp->if_capenable = ifp->if_capabilities;
2560
2561 /*
2562 * Tell the upper layer(s) we support long frames.
2563 */
2564 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2590 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2591 ifp->if_capenable = ifp->if_capabilities;
2592
2593 /*
2594 * Tell the upper layer(s) we support long frames.
2595 */
2596 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2565 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER;
2566 ifp->if_capabilities |= IFCAP_VLAN_MTU;
2567 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER;
2568 ifp->if_capenable |= IFCAP_VLAN_MTU;
2597 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2598 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2569
2570 /*
2571 * Specify the media types supported by this adapter and register
2572 * callbacks to update media and link information
2573 */
2574 ifmedia_init(&adapter->media, IFM_IMASK,
2575 igb_media_change, igb_media_status);
2576 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||

--- 120 unchanged lines hidden (view full) ---

2697 struct tx_ring *txr;
2698 struct rx_ring *rxr;
2699 int rsize, tsize, error = E1000_SUCCESS;
2700 int txconf = 0, rxconf = 0;
2701
2702 /* First allocate the TX ring struct memory */
2703 if (!(adapter->tx_rings =
2704 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2599
2600 /*
2601 * Specify the media types supported by this adapter and register
2602 * callbacks to update media and link information
2603 */
2604 ifmedia_init(&adapter->media, IFM_IMASK,
2605 igb_media_change, igb_media_status);
2606 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||

--- 120 unchanged lines hidden (view full) ---

2727 struct tx_ring *txr;
2728 struct rx_ring *rxr;
2729 int rsize, tsize, error = E1000_SUCCESS;
2730 int txconf = 0, rxconf = 0;
2731
2732 /* First allocate the TX ring struct memory */
2733 if (!(adapter->tx_rings =
2734 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2705 adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2735 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2706 device_printf(dev, "Unable to allocate TX ring memory\n");
2707 error = ENOMEM;
2708 goto fail;
2709 }
2710 txr = adapter->tx_rings;
2711
2712 /* Next allocate the RX */
2713 if (!(adapter->rx_rings =
2714 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2736 device_printf(dev, "Unable to allocate TX ring memory\n");
2737 error = ENOMEM;
2738 goto fail;
2739 }
2740 txr = adapter->tx_rings;
2741
2742 /* Next allocate the RX */
2743 if (!(adapter->rx_rings =
2744 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2715 adapter->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2745 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2716 device_printf(dev, "Unable to allocate RX ring memory\n");
2717 error = ENOMEM;
2718 goto rx_fail;
2719 }
2720 rxr = adapter->rx_rings;
2721
2722 tsize = roundup2(adapter->num_tx_desc *
2723 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
2724 /*
2725 * Now set up the TX queues, txconf is needed to handle the
2726 * possibility that things fail midcourse and we need to
2727 * undo memory gracefully
2728 */
2746 device_printf(dev, "Unable to allocate RX ring memory\n");
2747 error = ENOMEM;
2748 goto rx_fail;
2749 }
2750 rxr = adapter->rx_rings;
2751
2752 tsize = roundup2(adapter->num_tx_desc *
2753 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
2754 /*
2755 * Now set up the TX queues, txconf is needed to handle the
2756 * possibility that things fail midcourse and we need to
2757 * undo memory gracefully
2758 */
2729 for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) {
2759 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2730 /* Set up some basics */
2731 txr = &adapter->tx_rings[i];
2732 txr->adapter = adapter;
2733 txr->me = i;
2734
2735 /* Initialize the TX lock */
2736 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2737 device_get_nameunit(dev), txr->me);

--- 11 unchanged lines hidden (view full) ---

2749
2750 /* Now allocate transmit buffers for the ring */
2751 if (igb_allocate_transmit_buffers(txr)) {
2752 device_printf(dev,
2753 "Critical Failure setting up transmit buffers\n");
2754 error = ENOMEM;
2755 goto err_tx_desc;
2756 }
2760 /* Set up some basics */
2761 txr = &adapter->tx_rings[i];
2762 txr->adapter = adapter;
2763 txr->me = i;
2764
2765 /* Initialize the TX lock */
2766 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2767 device_get_nameunit(dev), txr->me);

--- 11 unchanged lines hidden (view full) ---

2779
2780 /* Now allocate transmit buffers for the ring */
2781 if (igb_allocate_transmit_buffers(txr)) {
2782 device_printf(dev,
2783 "Critical Failure setting up transmit buffers\n");
2784 error = ENOMEM;
2785 goto err_tx_desc;
2786 }
2757
2787#if __FreeBSD_version >= 800000
2788 /* Allocate a buf ring */
2789 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF,
2790 M_WAITOK, &txr->tx_mtx);
2791#endif
2758 }
2759
2760 /*
2761 * Next the RX queues...
2762 */
2763 rsize = roundup2(adapter->num_rx_desc *
2764 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
2792 }
2793
2794 /*
2795 * Next the RX queues...
2796 */
2797 rsize = roundup2(adapter->num_rx_desc *
2798 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
2765 for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) {
2799 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2766 rxr = &adapter->rx_rings[i];
2767 rxr->adapter = adapter;
2768 rxr->me = i;
2769
2770 /* Initialize the RX lock */
2771 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2772 device_get_nameunit(dev), txr->me);
2773 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);

--- 98 unchanged lines hidden (view full) ---

2872 **********************************************************************/
2873static void
2874igb_setup_transmit_ring(struct tx_ring *txr)
2875{
2876 struct adapter *adapter = txr->adapter;
2877 struct igb_tx_buffer *txbuf;
2878 int i;
2879
2800 rxr = &adapter->rx_rings[i];
2801 rxr->adapter = adapter;
2802 rxr->me = i;
2803
2804 /* Initialize the RX lock */
2805 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2806 device_get_nameunit(dev), txr->me);
2807 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);

--- 98 unchanged lines hidden (view full) ---

2906 **********************************************************************/
2907static void
2908igb_setup_transmit_ring(struct tx_ring *txr)
2909{
2910 struct adapter *adapter = txr->adapter;
2911 struct igb_tx_buffer *txbuf;
2912 int i;
2913
2880 /* Clear the old ring contents */
2914 /* Clear the old descriptor contents */
2881 bzero((void *)txr->tx_base,
2882 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
2883 /* Reset indices */
2884 txr->next_avail_desc = 0;
2885 txr->next_to_clean = 0;
2886
2887 /* Free any existing tx buffers. */
2888 txbuf = txr->tx_buffers;

--- 22 unchanged lines hidden (view full) ---

2911 * Initialize all transmit rings.
2912 *
2913 **********************************************************************/
2914static void
2915igb_setup_transmit_structures(struct adapter *adapter)
2916{
2917 struct tx_ring *txr = adapter->tx_rings;
2918
2915 bzero((void *)txr->tx_base,
2916 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
2917 /* Reset indices */
2918 txr->next_avail_desc = 0;
2919 txr->next_to_clean = 0;
2920
2921 /* Free any existing tx buffers. */
2922 txbuf = txr->tx_buffers;

--- 22 unchanged lines hidden (view full) ---

2945 * Initialize all transmit rings.
2946 *
2947 **********************************************************************/
2948static void
2949igb_setup_transmit_structures(struct adapter *adapter)
2950{
2951 struct tx_ring *txr = adapter->tx_rings;
2952
2919 for (int i = 0; i < adapter->num_tx_queues; i++, txr++)
2953 for (int i = 0; i < adapter->num_queues; i++, txr++)
2920 igb_setup_transmit_ring(txr);
2921
2922 return;
2923}
2924
2925/*********************************************************************
2926 *
2927 * Enable transmit unit.
2928 *
2929 **********************************************************************/
2930static void
2931igb_initialize_transmit_units(struct adapter *adapter)
2932{
2933 struct tx_ring *txr = adapter->tx_rings;
2934 u32 tctl, txdctl;
2935
2936 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
2937
2938 /* Setup the Base and Length of the Tx Descriptor Rings */
2954 igb_setup_transmit_ring(txr);
2955
2956 return;
2957}
2958
2959/*********************************************************************
2960 *
2961 * Enable transmit unit.
2962 *
2963 **********************************************************************/
2964static void
2965igb_initialize_transmit_units(struct adapter *adapter)
2966{
2967 struct tx_ring *txr = adapter->tx_rings;
2968 u32 tctl, txdctl;
2969
2970 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
2971
2972 /* Setup the Base and Length of the Tx Descriptor Rings */
2939 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
2973 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2940 u64 bus_addr = txr->txdma.dma_paddr;
2941
2942 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(i),
2943 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
2944 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(i),
2945 (uint32_t)(bus_addr >> 32));
2946 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(i),
2947 (uint32_t)bus_addr);

--- 32 unchanged lines hidden (view full) ---

2980 * Free all transmit rings.
2981 *
2982 **********************************************************************/
2983static void
2984igb_free_transmit_structures(struct adapter *adapter)
2985{
2986 struct tx_ring *txr = adapter->tx_rings;
2987
2974 u64 bus_addr = txr->txdma.dma_paddr;
2975
2976 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(i),
2977 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
2978 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(i),
2979 (uint32_t)(bus_addr >> 32));
2980 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(i),
2981 (uint32_t)bus_addr);

--- 32 unchanged lines hidden (view full) ---

3014 * Free all transmit rings.
3015 *
3016 **********************************************************************/
3017static void
3018igb_free_transmit_structures(struct adapter *adapter)
3019{
3020 struct tx_ring *txr = adapter->tx_rings;
3021
2988 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
3022 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2989 IGB_TX_LOCK(txr);
2990 igb_free_transmit_buffers(txr);
2991 igb_dma_free(adapter, &txr->txdma);
2992 IGB_TX_UNLOCK(txr);
2993 IGB_TX_LOCK_DESTROY(txr);
2994 }
2995 free(adapter->tx_rings, M_DEVBUF);
2996}

--- 32 unchanged lines hidden (view full) ---

3029 } else if (tx_buffer->map != NULL) {
3030 bus_dmamap_unload(txr->txtag,
3031 tx_buffer->map);
3032 bus_dmamap_destroy(txr->txtag,
3033 tx_buffer->map);
3034 tx_buffer->map = NULL;
3035 }
3036 }
3023 IGB_TX_LOCK(txr);
3024 igb_free_transmit_buffers(txr);
3025 igb_dma_free(adapter, &txr->txdma);
3026 IGB_TX_UNLOCK(txr);
3027 IGB_TX_LOCK_DESTROY(txr);
3028 }
3029 free(adapter->tx_rings, M_DEVBUF);
3030}

--- 32 unchanged lines hidden (view full) ---

3063 } else if (tx_buffer->map != NULL) {
3064 bus_dmamap_unload(txr->txtag,
3065 tx_buffer->map);
3066 bus_dmamap_destroy(txr->txtag,
3067 tx_buffer->map);
3068 tx_buffer->map = NULL;
3069 }
3070 }
3037
3071#if __FreeBSD_version >= 800000
3072 buf_ring_free(txr->br, M_DEVBUF);
3073#endif
3038 if (txr->tx_buffers != NULL) {
3039 free(txr->tx_buffers, M_DEVBUF);
3040 txr->tx_buffers = NULL;
3041 }
3042 if (txr->txtag != NULL) {
3043 bus_dma_tag_destroy(txr->txtag);
3044 txr->txtag = NULL;
3045 }
3046 return;
3047}
3048
3074 if (txr->tx_buffers != NULL) {
3075 free(txr->tx_buffers, M_DEVBUF);
3076 txr->tx_buffers = NULL;
3077 }
3078 if (txr->txtag != NULL) {
3079 bus_dma_tag_destroy(txr->txtag);
3080 txr->txtag = NULL;
3081 }
3082 return;
3083}
3084
3049#if __FreeBSD_version >= 700000
3050/**********************************************************************
3051 *
3052 * Setup work for hardware segmentation offload (TSO) on
3053 * adapters using advanced tx descriptors (82575)
3054 *
3055 **********************************************************************/
3056static boolean_t
3057igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)

--- 71 unchanged lines hidden (view full) ---

3129
3130 if (++ctxd == adapter->num_tx_desc)
3131 ctxd = 0;
3132
3133 txr->tx_avail--;
3134 txr->next_avail_desc = ctxd;
3135 return TRUE;
3136}
3085/**********************************************************************
3086 *
3087 * Setup work for hardware segmentation offload (TSO) on
3088 * adapters using advanced tx descriptors (82575)
3089 *
3090 **********************************************************************/
3091static boolean_t
3092igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)

--- 71 unchanged lines hidden (view full) ---

3164
3165 if (++ctxd == adapter->num_tx_desc)
3166 ctxd = 0;
3167
3168 txr->tx_avail--;
3169 txr->next_avail_desc = ctxd;
3170 return TRUE;
3171}
3137#else /* fake out for 6.2 */
3138static boolean_t
3139igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
3140{
3141 return (FALSE);
3142}
3143#endif
3144
3172
3173
3145/*********************************************************************
3146 *
3147 * Context Descriptor setup for VLAN or CSUM
3148 *
3149 **********************************************************************/
3150
3174/*********************************************************************
3175 *
3176 * Context Descriptor setup for VLAN or CSUM
3177 *
3178 **********************************************************************/
3179
3151static int
3180static bool
3152igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3153{
3154 struct adapter *adapter = txr->adapter;
3155 struct e1000_adv_tx_context_desc *TXD;
3156 struct igb_tx_buffer *tx_buffer;
3157 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3158 struct ether_vlan_header *eh;
3159 struct ip *ip = NULL;
3160 struct ip6_hdr *ip6;
3181igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3182{
3183 struct adapter *adapter = txr->adapter;
3184 struct e1000_adv_tx_context_desc *TXD;
3185 struct igb_tx_buffer *tx_buffer;
3186 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3187 struct ether_vlan_header *eh;
3188 struct ip *ip = NULL;
3189 struct ip6_hdr *ip6;
3161 int ehdrlen, ip_hlen = 0;
3162 u16 etype;
3190 int ehdrlen, ctxd, ip_hlen = 0;
3191 u16 etype, vtag = 0;
3163 u8 ipproto = 0;
3192 u8 ipproto = 0;
3164 bool offload = FALSE;
3165#if __FreeBSD_version >= 700000
3166 u16 vtag = 0;
3167#else
3168 struct m_tag *mtag;
3169#endif
3193 bool offload = TRUE;
3170
3194
3171 int ctxd = txr->next_avail_desc;
3195 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3196 offload = FALSE;
3197
3198 ctxd = txr->next_avail_desc;
3172 tx_buffer = &txr->tx_buffers[ctxd];
3173 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3174
3199 tx_buffer = &txr->tx_buffers[ctxd];
3200 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3201
3175 if (mp->m_pkthdr.csum_flags & CSUM_OFFLOAD)
3176 offload = TRUE;
3177
3178 /*
3179 ** In advanced descriptors the vlan tag must
3202 /*
3203 ** In advanced descriptors the vlan tag must
3180 ** be placed into the descriptor itself.
3204 ** be placed into the context descriptor, thus
3205 ** we need to be here just for that setup.
3181 */
3206 */
3182#if __FreeBSD_version < 700000
3183 mtag = VLAN_OUTPUT_TAG(ifp, mp);
3184 if (mtag != NULL) {
3185 vlan_macip_lens |=
3186 htole16(VLAN_TAG_VALUE(mtag)) << E1000_ADVTXD_VLAN_SHIFT;
3187 offload = TRUE;
3188 }
3189#else
3190 if (mp->m_flags & M_VLANTAG) {
3191 vtag = htole16(mp->m_pkthdr.ether_vtag);
3192 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3207 if (mp->m_flags & M_VLANTAG) {
3208 vtag = htole16(mp->m_pkthdr.ether_vtag);
3209 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3193 offload = TRUE;
3194 }
3195#endif
3210 } else if (offload == FALSE)
3211 return FALSE;
3212
3196 /*
3197 * Determine where frame payload starts.
3198 * Jump over vlan headers if already present,
3199 * helpful for QinQ too.
3200 */
3201 eh = mtod(mp, struct ether_vlan_header *);
3202 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3203 etype = ntohs(eh->evl_proto);

--- 5 unchanged lines hidden (view full) ---

3209
3210 /* Set the ether header length */
3211 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3212
3213 switch (etype) {
3214 case ETHERTYPE_IP:
3215 ip = (struct ip *)(mp->m_data + ehdrlen);
3216 ip_hlen = ip->ip_hl << 2;
3213 /*
3214 * Determine where frame payload starts.
3215 * Jump over vlan headers if already present,
3216 * helpful for QinQ too.
3217 */
3218 eh = mtod(mp, struct ether_vlan_header *);
3219 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3220 etype = ntohs(eh->evl_proto);

--- 5 unchanged lines hidden (view full) ---

3226
3227 /* Set the ether header length */
3228 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3229
3230 switch (etype) {
3231 case ETHERTYPE_IP:
3232 ip = (struct ip *)(mp->m_data + ehdrlen);
3233 ip_hlen = ip->ip_hl << 2;
3217 if (mp->m_len < ehdrlen + ip_hlen)
3218 return FALSE;
3234 if (mp->m_len < ehdrlen + ip_hlen) {
3235 offload = FALSE;
3236 break;
3237 }
3219 ipproto = ip->ip_p;
3220 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3221 break;
3222 case ETHERTYPE_IPV6:
3223 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3224 ip_hlen = sizeof(struct ip6_hdr);
3225 if (mp->m_len < ehdrlen + ip_hlen)
3238 ipproto = ip->ip_p;
3239 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3240 break;
3241 case ETHERTYPE_IPV6:
3242 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3243 ip_hlen = sizeof(struct ip6_hdr);
3244 if (mp->m_len < ehdrlen + ip_hlen)
3226 return FALSE; /* failure */
3245 return (FALSE);
3227 ipproto = ip6->ip6_nxt;
3228 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3229 break;
3246 ipproto = ip6->ip6_nxt;
3247 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3248 break;
3230#ifdef IGB_TIMESYNC
3231 case ETHERTYPE_IEEE1588:
3232 return (IGB_TIMESTAMP);
3233#endif
3234 default:
3249 default:
3235 return (FALSE);
3250 offload = FALSE;
3251 break;
3236 }
3237
3238 vlan_macip_lens |= ip_hlen;
3239 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3240
3241 switch (ipproto) {
3242 case IPPROTO_TCP:
3252 }
3253
3254 vlan_macip_lens |= ip_hlen;
3255 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3256
3257 switch (ipproto) {
3258 case IPPROTO_TCP:
3243 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3259 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3244 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3260 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3245 offload = TRUE;
3246 }
3247 break;
3248 case IPPROTO_UDP:
3261 break;
3262 case IPPROTO_UDP:
3249 {
3250#ifdef IGB_TIMESYNC
3251 void *hdr = (caddr_t) ip + ip_hlen;
3252 struct udphdr *uh = (struct udphdr *)hdr;
3253
3254 if (uh->uh_dport == htons(TSYNC_PORT))
3255 return (IGB_TIMESTAMP);
3256#endif
3257 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3263 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3258 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3264 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3259 offload = TRUE;
3260 }
3261 break;
3265 break;
3262 }
3263#if __FreeBSD_version >= 800000
3264 case IPPROTO_SCTP:
3266#if __FreeBSD_version >= 800000
3267 case IPPROTO_SCTP:
3265 {
3266 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) {
3268 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3267 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3269 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3268 offload = TRUE;
3269 }
3270 break;
3270 break;
3271 }
3272#endif
3273 default:
3271#endif
3272 default:
3274 return (FALSE);
3273 offload = FALSE;
3274 break;
3275 }
3276
3275 }
3276
3277 if (offload != TRUE)
3278 return (FALSE);
3279
3280 /* Now copy bits into descriptor */
3281 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3282 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3283 TXD->seqnum_seed = htole32(0);
3284 TXD->mss_l4len_idx = htole32(0);
3285
3286 tx_buffer->m_head = NULL;
3287 tx_buffer->next_eop = -1;
3288
3289 /* We've consumed the first desc, adjust counters */
3290 if (++ctxd == adapter->num_tx_desc)
3291 ctxd = 0;
3292 txr->next_avail_desc = ctxd;
3293 --txr->tx_avail;
3294
3277 /* Now copy bits into descriptor */
3278 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3279 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3280 TXD->seqnum_seed = htole32(0);
3281 TXD->mss_l4len_idx = htole32(0);
3282
3283 tx_buffer->m_head = NULL;
3284 tx_buffer->next_eop = -1;
3285
3286 /* We've consumed the first desc, adjust counters */
3287 if (++ctxd == adapter->num_tx_desc)
3288 ctxd = 0;
3289 txr->next_avail_desc = ctxd;
3290 --txr->tx_avail;
3291
3295 return (TRUE);
3292 return (offload);
3296}
3297
3298
3299/**********************************************************************
3300 *
3301 * Examine each tx_buffer in the used queue. If the hardware is done
3302 * processing the packet then free associated resources. The
3303 * tx_buffer is put back on the free queue.
3304 *
3305 * TRUE return means there's work in the ring to clean, FALSE its empty.
3306 **********************************************************************/
3307static bool
3308igb_txeof(struct tx_ring *txr)
3309{
3310 struct adapter *adapter = txr->adapter;
3311 int first, last, done, num_avail;
3293}
3294
3295
3296/**********************************************************************
3297 *
3298 * Examine each tx_buffer in the used queue. If the hardware is done
3299 * processing the packet then free associated resources. The
3300 * tx_buffer is put back on the free queue.
3301 *
3302 * TRUE return means there's work in the ring to clean, FALSE its empty.
3303 **********************************************************************/
3304static bool
3305igb_txeof(struct tx_ring *txr)
3306{
3307 struct adapter *adapter = txr->adapter;
3308 int first, last, done, num_avail;
3312 u32 cleaned = 0;
3309 u32 cleaned = 0;
3313 struct igb_tx_buffer *tx_buffer;
3314 struct e1000_tx_desc *tx_desc, *eop_desc;
3315 struct ifnet *ifp = adapter->ifp;
3316
3317 IGB_TX_LOCK_ASSERT(txr);
3318
3319 if (txr->tx_avail == adapter->num_tx_desc)
3320 return FALSE;

--- 56 unchanged lines hidden (view full) ---

3377 break;
3378 }
3379 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3380 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3381
3382 txr->next_to_clean = first;
3383
3384 /*
3310 struct igb_tx_buffer *tx_buffer;
3311 struct e1000_tx_desc *tx_desc, *eop_desc;
3312 struct ifnet *ifp = adapter->ifp;
3313
3314 IGB_TX_LOCK_ASSERT(txr);
3315
3316 if (txr->tx_avail == adapter->num_tx_desc)
3317 return FALSE;

--- 56 unchanged lines hidden (view full) ---

3374 break;
3375 }
3376 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3377 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3378
3379 txr->next_to_clean = first;
3380
3381 /*
3385 * If we have enough room, clear IFF_DRV_OACTIVE to
3386 * tell the stack that it is OK to send packets.
3387 * If there are no pending descriptors, clear the timeout.
3382 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack
3383 * that it is OK to send packets.
3384 * If there are no pending descriptors, clear the timeout. Otherwise,
3385 * if some descriptors have been freed, restart the timeout.
3388 */
3389 if (num_avail > IGB_TX_CLEANUP_THRESHOLD) {
3390 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3386 */
3387 if (num_avail > IGB_TX_CLEANUP_THRESHOLD) {
3388 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3389 /* All clean, turn off the timer */
3391 if (num_avail == adapter->num_tx_desc) {
3392 txr->watchdog_timer = 0;
3393 txr->tx_avail = num_avail;
3394 return FALSE;
3395 }
3396 }
3390 if (num_avail == adapter->num_tx_desc) {
3391 txr->watchdog_timer = 0;
3392 txr->tx_avail = num_avail;
3393 return FALSE;
3394 }
3395 }
3397 /* Some descriptors cleaned, reset the watchdog */
3396
3397 /* Some cleaned, reset the timer */
3398 if (cleaned)
3399 txr->watchdog_timer = IGB_TX_TIMEOUT;
3400 txr->tx_avail = num_avail;
3401 return TRUE;
3402}
3403
3404
3405/*********************************************************************

--- 186 unchanged lines hidden (view full) ---

3592 *
3593 * Initialize a receive ring and its buffers.
3594 *
3595 **********************************************************************/
3596static int
3597igb_setup_receive_ring(struct rx_ring *rxr)
3598{
3599 struct adapter *adapter;
3398 if (cleaned)
3399 txr->watchdog_timer = IGB_TX_TIMEOUT;
3400 txr->tx_avail = num_avail;
3401 return TRUE;
3402}
3403
3404
3405/*********************************************************************

--- 186 unchanged lines hidden (view full) ---

3592 *
3593 * Initialize a receive ring and its buffers.
3594 *
3595 **********************************************************************/
3596static int
3597igb_setup_receive_ring(struct rx_ring *rxr)
3598{
3599 struct adapter *adapter;
3600 struct ifnet *ifp;
3600 device_t dev;
3601 struct igb_rx_buffer *rxbuf;
3602 struct lro_ctrl *lro = &rxr->lro;
3603 int j, rsize;
3604
3605 adapter = rxr->adapter;
3606 dev = adapter->dev;
3601 device_t dev;
3602 struct igb_rx_buffer *rxbuf;
3603 struct lro_ctrl *lro = &rxr->lro;
3604 int j, rsize;
3605
3606 adapter = rxr->adapter;
3607 dev = adapter->dev;
3608 ifp = adapter->ifp;
3609 rxr->lro_enabled = FALSE;
3610 rxr->hdr_split = FALSE;
3607
3608 /* Clear the ring contents */
3609 rsize = roundup2(adapter->num_rx_desc *
3610 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3611 bzero((void *)rxr->rx_base, rsize);
3612
3613 /*
3614 ** Free current RX buffer structures and their mbufs

--- 24 unchanged lines hidden (view full) ---

3639
3640 /* Setup our descriptor indices */
3641 rxr->next_to_check = 0;
3642 rxr->last_cleaned = 0;
3643
3644 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3645 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3646
3611
3612 /* Clear the ring contents */
3613 rsize = roundup2(adapter->num_rx_desc *
3614 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3615 bzero((void *)rxr->rx_base, rsize);
3616
3617 /*
3618 ** Free current RX buffer structures and their mbufs

--- 24 unchanged lines hidden (view full) ---

3643
3644 /* Setup our descriptor indices */
3645 rxr->next_to_check = 0;
3646 rxr->last_cleaned = 0;
3647
3648 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3649 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3650
3647 /* Now set up the LRO interface */
3648 if (igb_enable_lro) {
3651 /*
3652 ** Now set up the LRO interface, we
3653 ** also only do head split when LRO
3654 ** is enabled, since so often they
3655 ** are undesireable in similar setups.
3656 */
3657 if (ifp->if_capenable & IFCAP_LRO) {
3649 int err = tcp_lro_init(lro);
3650 if (err) {
3651 device_printf(dev,"LRO Initialization failed!\n");
3652 goto fail;
3653 }
3654 INIT_DEBUGOUT("RX LRO Initialized\n");
3658 int err = tcp_lro_init(lro);
3659 if (err) {
3660 device_printf(dev,"LRO Initialization failed!\n");
3661 goto fail;
3662 }
3663 INIT_DEBUGOUT("RX LRO Initialized\n");
3664 rxr->lro_enabled = TRUE;
3665 rxr->hdr_split = TRUE;
3655 lro->ifp = adapter->ifp;
3656 }
3657
3658 return (0);
3659fail:
3660 /*
3661 * We need to clean up any buffers allocated
3662 * so far, 'j' is the failing index.

--- 17 unchanged lines hidden (view full) ---

3680 *
3681 **********************************************************************/
3682static int
3683igb_setup_receive_structures(struct adapter *adapter)
3684{
3685 struct rx_ring *rxr = adapter->rx_rings;
3686 int i, j;
3687
3666 lro->ifp = adapter->ifp;
3667 }
3668
3669 return (0);
3670fail:
3671 /*
3672 * We need to clean up any buffers allocated
3673 * so far, 'j' is the failing index.

--- 17 unchanged lines hidden (view full) ---

3691 *
3692 **********************************************************************/
3693static int
3694igb_setup_receive_structures(struct adapter *adapter)
3695{
3696 struct rx_ring *rxr = adapter->rx_rings;
3697 int i, j;
3698
3688 for (i = 0; i < adapter->num_rx_queues; i++, rxr++)
3699 for (i = 0; i < adapter->num_queues; i++, rxr++)
3689 if (igb_setup_receive_ring(rxr))
3690 goto fail;
3691
3692 return (0);
3693fail:
3694 /*
3695 * Free RX buffers allocated so far, we will only handle
3696 * the rings that completed, the failing case will have

--- 37 unchanged lines hidden (view full) ---

3734 * up the descriptor ring
3735 */
3736 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
3737 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3738
3739 /*
3740 ** Set up for header split
3741 */
3700 if (igb_setup_receive_ring(rxr))
3701 goto fail;
3702
3703 return (0);
3704fail:
3705 /*
3706 * Free RX buffers allocated so far, we will only handle
3707 * the rings that completed, the failing case will have

--- 37 unchanged lines hidden (view full) ---

3745 * up the descriptor ring
3746 */
3747 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
3748 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3749
3750 /*
3751 ** Set up for header split
3752 */
3742 if (igb_rx_hdr_split) {
3753 if (rxr->hdr_split) {
3743 /* Use a standard mbuf for the header */
3744 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3745 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3746 } else
3747 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3748
3749 /*
3750 ** Set up for jumbo frames
3751 */
3752 if (ifp->if_mtu > ETHERMTU) {
3753 rctl |= E1000_RCTL_LPE;
3754 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3755 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
3756
3757 /* Set maximum packet len */
3758 psize = adapter->max_frame_size;
3759 /* are we on a vlan? */
3754 /* Use a standard mbuf for the header */
3755 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3756 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3757 } else
3758 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3759
3760 /*
3761 ** Set up for jumbo frames
3762 */
3763 if (ifp->if_mtu > ETHERMTU) {
3764 rctl |= E1000_RCTL_LPE;
3765 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3766 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
3767
3768 /* Set maximum packet len */
3769 psize = adapter->max_frame_size;
3770 /* are we on a vlan? */
3760#if __FreeBSD_version >= 700000
3761 if (adapter->ifp->if_vlantrunk != NULL)
3771 if (adapter->ifp->if_vlantrunk != NULL)
3762#else
3763 if (adapter->ifp->if_nvlans != 0)
3764#endif
3765 psize += VLAN_TAG_SIZE;
3766 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
3767 } else {
3768 rctl &= ~E1000_RCTL_LPE;
3769 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3770 rctl |= E1000_RCTL_SZ_2048;
3771 }
3772
3773 /* Setup the Base and Length of the Rx Descriptor Rings */
3772 psize += VLAN_TAG_SIZE;
3773 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
3774 } else {
3775 rctl &= ~E1000_RCTL_LPE;
3776 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3777 rctl |= E1000_RCTL_SZ_2048;
3778 }
3779
3780 /* Setup the Base and Length of the Rx Descriptor Rings */
3774 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
3781 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3775 u64 bus_addr = rxr->rxdma.dma_paddr;
3776 u32 rxdctl;
3777
3778 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(i),
3779 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3780 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(i),
3781 (uint32_t)(bus_addr >> 32));
3782 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(i),

--- 8 unchanged lines hidden (view full) ---

3791 rxdctl |= IGB_RX_WTHRESH << 16;
3792 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(i), rxdctl);
3793 }
3794
3795 /*
3796 ** Setup for RX MultiQueue
3797 */
3798 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
3782 u64 bus_addr = rxr->rxdma.dma_paddr;
3783 u32 rxdctl;
3784
3785 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(i),
3786 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3787 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(i),
3788 (uint32_t)(bus_addr >> 32));
3789 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(i),

--- 8 unchanged lines hidden (view full) ---

3798 rxdctl |= IGB_RX_WTHRESH << 16;
3799 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(i), rxdctl);
3800 }
3801
3802 /*
3803 ** Setup for RX MultiQueue
3804 */
3805 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
3799 if (adapter->num_rx_queues >1) {
3806 if (adapter->num_queues >1) {
3800 u32 random[10], mrqc, shift = 0;
3801 union igb_reta {
3802 u32 dword;
3803 u8 bytes[4];
3804 } reta;
3805
3806 arc4rand(&random, sizeof(random), 0);
3807 if (adapter->hw.mac.type == e1000_82575)
3808 shift = 6;
3809 /* Warning FM follows */
3810 for (int i = 0; i < 128; i++) {
3811 reta.bytes[i & 3] =
3807 u32 random[10], mrqc, shift = 0;
3808 union igb_reta {
3809 u32 dword;
3810 u8 bytes[4];
3811 } reta;
3812
3813 arc4rand(&random, sizeof(random), 0);
3814 if (adapter->hw.mac.type == e1000_82575)
3815 shift = 6;
3816 /* Warning FM follows */
3817 for (int i = 0; i < 128; i++) {
3818 reta.bytes[i & 3] =
3812 (i % adapter->num_rx_queues) << shift;
3819 (i % adapter->num_queues) << shift;
3813 if ((i & 3) == 3)
3814 E1000_WRITE_REG(&adapter->hw,
3815 E1000_RETA(i >> 2), reta.dword);
3816 }
3817 /* Now fill in hash table */
3818 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3819 for (int i = 0; i < 10; i++)
3820 E1000_WRITE_REG_ARRAY(&adapter->hw,

--- 49 unchanged lines hidden (view full) ---

3870
3871 /* Enable Receives */
3872 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
3873
3874 /*
3875 * Setup the HW Rx Head and Tail Descriptor Pointers
3876 * - needs to be after enable
3877 */
3820 if ((i & 3) == 3)
3821 E1000_WRITE_REG(&adapter->hw,
3822 E1000_RETA(i >> 2), reta.dword);
3823 }
3824 /* Now fill in hash table */
3825 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3826 for (int i = 0; i < 10; i++)
3827 E1000_WRITE_REG_ARRAY(&adapter->hw,

--- 49 unchanged lines hidden (view full) ---

3877
3878 /* Enable Receives */
3879 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
3880
3881 /*
3882 * Setup the HW Rx Head and Tail Descriptor Pointers
3883 * - needs to be after enable
3884 */
3878 for (int i = 0; i < adapter->num_rx_queues; i++) {
3885 for (int i = 0; i < adapter->num_queues; i++) {
3879 E1000_WRITE_REG(&adapter->hw, E1000_RDH(i), 0);
3880 E1000_WRITE_REG(&adapter->hw, E1000_RDT(i),
3881 adapter->num_rx_desc - 1);
3882 }
3883 return;
3884}
3885
3886/*********************************************************************
3887 *
3888 * Free receive rings.
3889 *
3890 **********************************************************************/
3891static void
3892igb_free_receive_structures(struct adapter *adapter)
3893{
3894 struct rx_ring *rxr = adapter->rx_rings;
3895
3886 E1000_WRITE_REG(&adapter->hw, E1000_RDH(i), 0);
3887 E1000_WRITE_REG(&adapter->hw, E1000_RDT(i),
3888 adapter->num_rx_desc - 1);
3889 }
3890 return;
3891}
3892
3893/*********************************************************************
3894 *
3895 * Free receive rings.
3896 *
3897 **********************************************************************/
3898static void
3899igb_free_receive_structures(struct adapter *adapter)
3900{
3901 struct rx_ring *rxr = adapter->rx_rings;
3902
3896 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
3903 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3897 struct lro_ctrl *lro = &rxr->lro;
3898 igb_free_receive_buffers(rxr);
3899 tcp_lro_free(lro);
3900 igb_dma_free(adapter, &rxr->rxdma);
3901 }
3902
3903 free(adapter->rx_rings, M_DEVBUF);
3904}

--- 86 unchanged lines hidden (view full) ---

3991 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3992 BUS_DMASYNC_POSTREAD);
3993
3994 /* Main clean loop */
3995 while ((staterr & E1000_RXD_STAT_DD) &&
3996 (count != 0) &&
3997 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3998 struct mbuf *sendmp, *mh, *mp;
3904 struct lro_ctrl *lro = &rxr->lro;
3905 igb_free_receive_buffers(rxr);
3906 tcp_lro_free(lro);
3907 igb_dma_free(adapter, &rxr->rxdma);
3908 }
3909
3910 free(adapter->rx_rings, M_DEVBUF);
3911}

--- 86 unchanged lines hidden (view full) ---

3998 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3999 BUS_DMASYNC_POSTREAD);
4000
4001 /* Main clean loop */
4002 while ((staterr & E1000_RXD_STAT_DD) &&
4003 (count != 0) &&
4004 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4005 struct mbuf *sendmp, *mh, *mp;
3999 u16 hlen, plen, hdr, ptype, len_adj;
4006 u16 hlen, plen, hdr, ptype, len_adj, vtag;
4000 u8 dopayload, accept_frame, eop;
4001
4002 accept_frame = 1;
4007 u8 dopayload, accept_frame, eop;
4008
4009 accept_frame = 1;
4003 hlen = plen = len_adj = 0;
4010 hlen = plen = len_adj = vtag = 0;
4004 sendmp = mh = mp = NULL;
4005 ptype = (u16)(cur->wb.lower.lo_dword.data >> 4);
4006
4007 /* Sync the buffers */
4008 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map,
4009 BUS_DMASYNC_POSTREAD);
4010
4011 /*
4012 ** The way the hardware is configured to
4013 ** split, it will ONLY use the header buffer
4014 ** when header split is enabled, otherwise we
4015 ** get normal behavior, ie, both header and
4016 ** payload are DMA'd into the payload buffer.
4017 **
4018 ** The fmp test is to catch the case where a
4019 ** packet spans multiple descriptors, in that
4020 ** case only the first header is valid.
4021 */
4011 sendmp = mh = mp = NULL;
4012 ptype = (u16)(cur->wb.lower.lo_dword.data >> 4);
4013
4014 /* Sync the buffers */
4015 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map,
4016 BUS_DMASYNC_POSTREAD);
4017
4018 /*
4019 ** The way the hardware is configured to
4020 ** split, it will ONLY use the header buffer
4021 ** when header split is enabled, otherwise we
4022 ** get normal behavior, ie, both header and
4023 ** payload are DMA'd into the payload buffer.
4024 **
4025 ** The fmp test is to catch the case where a
4026 ** packet spans multiple descriptors, in that
4027 ** case only the first header is valid.
4028 */
4022 if ((igb_rx_hdr_split) && (rxr->fmp == NULL)){
4029 if ((rxr->hdr_split) && (rxr->fmp == NULL)){
4023 hdr = le16toh(cur->
4024 wb.lower.lo_dword.hs_rss.hdr_info);
4025 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4026 E1000_RXDADV_HDRBUFLEN_SHIFT;
4027 if (hlen > IGB_HDR_BUF)
4028 hlen = IGB_HDR_BUF;
4029 plen = le16toh(cur->wb.upper.length);
4030 /* Handle the header mbuf */

--- 50 unchanged lines hidden (view full) ---

4081 } else
4082 mh->m_len -= ETHER_CRC_LEN;
4083 }
4084 } else
4085 eop = 0;
4086
4087 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)
4088 accept_frame = 0;
4030 hdr = le16toh(cur->
4031 wb.lower.lo_dword.hs_rss.hdr_info);
4032 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4033 E1000_RXDADV_HDRBUFLEN_SHIFT;
4034 if (hlen > IGB_HDR_BUF)
4035 hlen = IGB_HDR_BUF;
4036 plen = le16toh(cur->wb.upper.length);
4037 /* Handle the header mbuf */

--- 50 unchanged lines hidden (view full) ---

4088 } else
4089 mh->m_len -= ETHER_CRC_LEN;
4090 }
4091 } else
4092 eop = 0;
4093
4094 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)
4095 accept_frame = 0;
4096#ifdef IGB_IEEE1588
4097 This linux code needs to be converted to work here
4098 -----------------------------------------------------
4099 if (unlikely(staterr & E1000_RXD_STAT_TS)) {
4100 u64 regval;
4101 u64 ns;
4102// Create an mtag and set it up
4103 struct skb_shared_hwtstamps *shhwtstamps =
4104 skb_hwtstamps(skb);
4089
4105
4106 rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID),
4107 "igb: no RX time stamp available for time stamped packet");
4108 regval = rd32(E1000_RXSTMPL);
4109 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4110// Do time conversion from the register
4111 ns = timecounter_cyc2time(&adapter->clock, regval);
4112 clocksync_update(&adapter->sync, ns);
4113 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
4114 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4115 shhwtstamps->syststamp =
4116 clocksync_hw2sys(&adapter->sync, ns);
4117 }
4118#endif
4090 if (accept_frame) {
4119 if (accept_frame) {
4120 /*
4121 ** get_buf will overwrite the writeback
4122 ** descriptor so save the VLAN tag now.
4123 */
4124 vtag = le16toh(cur->wb.upper.vlan);
4091 if (igb_get_buf(rxr, i, dopayload) != 0) {
4092 ifp->if_iqdrops++;
4093 goto discard;
4094 }
4095 /* Initial frame - setup */
4096 if (rxr->fmp == NULL) {
4097 mh->m_flags |= M_PKTHDR;
4098 mh->m_pkthdr.len = mh->m_len;
4099 rxr->fmp = mh; /* Store the first mbuf */
4100 rxr->lmp = mh;
4101 if (mp) { /* Add payload if split */
4102 mh->m_pkthdr.len += mp->m_len;
4103 rxr->lmp = mh->m_next;
4104 }
4105 } else {
4125 if (igb_get_buf(rxr, i, dopayload) != 0) {
4126 ifp->if_iqdrops++;
4127 goto discard;
4128 }
4129 /* Initial frame - setup */
4130 if (rxr->fmp == NULL) {
4131 mh->m_flags |= M_PKTHDR;
4132 mh->m_pkthdr.len = mh->m_len;
4133 rxr->fmp = mh; /* Store the first mbuf */
4134 rxr->lmp = mh;
4135 if (mp) { /* Add payload if split */
4136 mh->m_pkthdr.len += mp->m_len;
4137 rxr->lmp = mh->m_next;
4138 }
4139 } else {
4106 /* Adjust for CRC frag */
4107 if (len_adj) {
4108 rxr->lmp->m_len -= len_adj;
4109 rxr->fmp->m_pkthdr.len -= len_adj;
4110 }
4111 /* Chain mbuf's together */
4112 mh->m_flags &= ~M_PKTHDR;
4113 rxr->lmp->m_next = mh;
4114 rxr->lmp = rxr->lmp->m_next;
4115 rxr->fmp->m_pkthdr.len += mh->m_len;
4140 /* Chain mbuf's together */
4141 mh->m_flags &= ~M_PKTHDR;
4142 rxr->lmp->m_next = mh;
4143 rxr->lmp = rxr->lmp->m_next;
4144 rxr->fmp->m_pkthdr.len += mh->m_len;
4145 /* Adjust for CRC frag */
4146 if (len_adj) {
4147 rxr->lmp->m_len -= len_adj;
4148 rxr->fmp->m_pkthdr.len -= len_adj;
4149 }
4116 }
4117
4118 if (eop) {
4119 bool sctp = ((ptype & 0x40) != 0);
4120 rxr->fmp->m_pkthdr.rcvif = ifp;
4121 ifp->if_ipackets++;
4122 rxr->rx_packets++;
4123 /* capture data for AIM */
4124 rxr->bytes += rxr->fmp->m_pkthdr.len;
4125 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4126
4127 igb_rx_checksum(staterr, rxr->fmp, sctp);
4128 if (staterr & E1000_RXD_STAT_VP) {
4150 }
4151
4152 if (eop) {
4153 bool sctp = ((ptype & 0x40) != 0);
4154 rxr->fmp->m_pkthdr.rcvif = ifp;
4155 ifp->if_ipackets++;
4156 rxr->rx_packets++;
4157 /* capture data for AIM */
4158 rxr->bytes += rxr->fmp->m_pkthdr.len;
4159 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4160
4161 igb_rx_checksum(staterr, rxr->fmp, sctp);
4162 if (staterr & E1000_RXD_STAT_VP) {
4129#if __FreeBSD_version >= 700000
4130 rxr->fmp->m_pkthdr.ether_vtag =
4131 le16toh(cur->wb.upper.vlan);
4163 rxr->fmp->m_pkthdr.ether_vtag = vtag;
4132 rxr->fmp->m_flags |= M_VLANTAG;
4164 rxr->fmp->m_flags |= M_VLANTAG;
4133#else
4134 VLAN_INPUT_TAG_NEW(ifp, rxr->fmp,
4135 (le16toh(cur->wb.upper.vlan) &
4136 E1000_RXD_SPC_VLAN_MASK));
4137#endif
4138 }
4165 }
4166#if __FreeBSD_version >= 800000
4167 rxr->fmp->m_pkthdr.flowid = curcpu;
4168 rxr->fmp->m_flags |= M_FLOWID;
4169#endif
4139 sendmp = rxr->fmp;
4140 rxr->fmp = NULL;
4141 rxr->lmp = NULL;
4142 }
4143 } else {
4144 ifp->if_ierrors++;
4145discard:
4146 /* Reuse loaded DMA map and just update mbuf chain */

--- 28 unchanged lines hidden (view full) ---

4175 i = 0;
4176
4177 /*
4178 ** Note that we hold the RX lock thru
4179 ** the following call so this ring's
4180 ** next_to_check is not gonna change.
4181 */
4182 if (sendmp != NULL) {
4170 sendmp = rxr->fmp;
4171 rxr->fmp = NULL;
4172 rxr->lmp = NULL;
4173 }
4174 } else {
4175 ifp->if_ierrors++;
4176discard:
4177 /* Reuse loaded DMA map and just update mbuf chain */

--- 28 unchanged lines hidden (view full) ---

4206 i = 0;
4207
4208 /*
4209 ** Note that we hold the RX lock thru
4210 ** the following call so this ring's
4211 ** next_to_check is not gonna change.
4212 */
4213 if (sendmp != NULL) {
4183 /* Use LRO if possible */
4184 if ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0)))
4185 /* Pass up to the stack */
4186 (*ifp->if_input)(ifp, sendmp);
4187 }
4214 /*
4215 ** Send to the stack if:
4216 ** - LRO not enabled, or
4217 ** - no LRO resources, or
4218 ** - lro enqueue fails
4219 */
4220 if ((!rxr->lro_enabled) ||
4221 ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0))))
4222 (*ifp->if_input)(ifp, sendmp);
4223 }
4188
4189 /* Get the next descriptor */
4190 cur = &rxr->rx_base[i];
4191 staterr = cur->wb.upper.status_error;
4192 }
4193 rxr->next_to_check = i;
4194
4195 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */

--- 71 unchanged lines hidden (view full) ---

4267/*
4268 * This routine is run via an vlan
4269 * config EVENT
4270 */
4271static void
4272igb_register_vlan(void *unused, struct ifnet *ifp, u16 vtag)
4273{
4274 struct adapter *adapter = ifp->if_softc;
4224
4225 /* Get the next descriptor */
4226 cur = &rxr->rx_base[i];
4227 staterr = cur->wb.upper.status_error;
4228 }
4229 rxr->next_to_check = i;
4230
4231 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */

--- 71 unchanged lines hidden (view full) ---

4303/*
4304 * This routine is run via an vlan
4305 * config EVENT
4306 */
4307static void
4308igb_register_vlan(void *unused, struct ifnet *ifp, u16 vtag)
4309{
4310 struct adapter *adapter = ifp->if_softc;
4275 u32 ctrl, rctl, index, vfta;
4311 u32 index, bit;
4276
4312
4277 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4278 ctrl |= E1000_CTRL_VME;
4279 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4313 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4314 return;
4280
4315
4281 /* Setup for Hardware Filter */
4282 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4283 rctl |= E1000_RCTL_VFE;
4284 rctl &= ~E1000_RCTL_CFIEN;
4285 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4286
4287 /* Make entry in the hardware filter table */
4288 index = ((vtag >> 5) & 0x7F);
4289 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index);
4290 vfta |= (1 << (vtag & 0x1F));
4291 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta);
4292
4293 /* Update the frame size */
4294 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4295 adapter->max_frame_size + VLAN_TAG_SIZE);
4296
4316 index = (vtag >> 5) & 0x7F;
4317 bit = vtag & 0x1F;
4318 igb_shadow_vfta[index] |= (1 << bit);
4319 ++adapter->num_vlans;
4320 /* Re-init to load the changes */
4321 igb_init(adapter);
4297}
4298
4299/*
4300 * This routine is run via an vlan
4301 * unconfig EVENT
4302 */
4303static void
4304igb_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag)
4305{
4306 struct adapter *adapter = ifp->if_softc;
4322}
4323
4324/*
4325 * This routine is run via an vlan
4326 * unconfig EVENT
4327 */
4328static void
4329igb_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag)
4330{
4331 struct adapter *adapter = ifp->if_softc;
4307 u32 index, vfta;
4332 u32 index, bit;
4308
4333
4309 /* Remove entry in the hardware filter table */
4310 index = ((vtag >> 5) & 0x7F);
4311 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index);
4312 vfta &= ~(1 << (vtag & 0x1F));
4313 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta);
4314 /* Have all vlans unregistered? */
4315 if (adapter->ifp->if_vlantrunk == NULL) {
4316 u32 rctl;
4317 /* Turn off the filter table */
4318 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4319 rctl &= ~E1000_RCTL_VFE;
4320 rctl |= E1000_RCTL_CFIEN;
4321 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4322 /* Reset the frame size */
4323 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4324 adapter->max_frame_size);
4325 }
4334 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4335 return;
4336
4337 index = (vtag >> 5) & 0x7F;
4338 bit = vtag & 0x1F;
4339 igb_shadow_vfta[index] &= ~(1 << bit);
4340 --adapter->num_vlans;
4341 /* Re-init to load the changes */
4342 igb_init(adapter);
4326}
4327
4328static void
4343}
4344
4345static void
4346igb_setup_vlan_hw_support(struct adapter *adapter)
4347{
4348 struct e1000_hw *hw = &adapter->hw;
4349 u32 reg;
4350
4351 /*
4352 ** We get here thru init_locked, meaning
4353 ** a soft reset, this has already cleared
4354 ** the VFTA and other state, so if there
4355 ** have been no vlan's registered do nothing.
4356 */
4357 if (adapter->num_vlans == 0)
4358 return;
4359
4360 /*
4361 ** A soft reset zero's out the VFTA, so
4362 ** we need to repopulate it now.
4363 */
4364 for (int i = 0; i < IGB_VFTA_SIZE; i++)
4365 if (igb_shadow_vfta[i] != 0)
4366 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4367 i, igb_shadow_vfta[i]);
4368
4369 reg = E1000_READ_REG(hw, E1000_CTRL);
4370 reg |= E1000_CTRL_VME;
4371 E1000_WRITE_REG(hw, E1000_CTRL, reg);
4372
4373 /* Enable the Filter Table */
4374 reg = E1000_READ_REG(hw, E1000_RCTL);
4375 reg &= ~E1000_RCTL_CFIEN;
4376 reg |= E1000_RCTL_VFE;
4377 E1000_WRITE_REG(hw, E1000_RCTL, reg);
4378
4379 /* Update the frame size */
4380 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4381 adapter->max_frame_size + VLAN_TAG_SIZE);
4382}
4383
4384static void
4329igb_enable_intr(struct adapter *adapter)
4330{
4331 /* With RSS set up what to auto clear */
4332 if (adapter->msix_mem) {
4333 E1000_WRITE_REG(&adapter->hw, E1000_EIAC,
4334 adapter->eims_mask);
4335 E1000_WRITE_REG(&adapter->hw, E1000_EIAM,
4336 adapter->eims_mask);

--- 259 unchanged lines hidden (view full) ---

4596
4597 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
4598 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
4599 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
4600 device_printf(dev, "Flow control watermarks high = %d low = %d\n",
4601 adapter->hw.fc.high_water,
4602 adapter->hw.fc.low_water);
4603
4385igb_enable_intr(struct adapter *adapter)
4386{
4387 /* With RSS set up what to auto clear */
4388 if (adapter->msix_mem) {
4389 E1000_WRITE_REG(&adapter->hw, E1000_EIAC,
4390 adapter->eims_mask);
4391 E1000_WRITE_REG(&adapter->hw, E1000_EIAM,
4392 adapter->eims_mask);

--- 259 unchanged lines hidden (view full) ---

4652
4653 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
4654 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
4655 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
4656 device_printf(dev, "Flow control watermarks high = %d low = %d\n",
4657 adapter->hw.fc.high_water,
4658 adapter->hw.fc.low_water);
4659
4604 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
4660 for (int i = 0; i < adapter->num_queues; i++, txr++) {
4605 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i,
4606 E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
4607 E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
4661 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i,
4662 E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
4663 E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
4608 device_printf(dev, "no descriptors avail event = %lld\n",
4609 (long long)txr->no_desc_avail);
4664 device_printf(dev, "no descriptors avail event = %lu\n",
4665 txr->no_desc_avail);
4610 device_printf(dev, "TX(%d) MSIX IRQ Handled = %lld\n", txr->me,
4611 (long long)txr->tx_irq);
4612 device_printf(dev, "TX(%d) Packets sent = %lld\n", txr->me,
4613 (long long)txr->tx_packets);
4614 }
4615
4666 device_printf(dev, "TX(%d) MSIX IRQ Handled = %lld\n", txr->me,
4667 (long long)txr->tx_irq);
4668 device_printf(dev, "TX(%d) Packets sent = %lld\n", txr->me,
4669 (long long)txr->tx_packets);
4670 }
4671
4616 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
4672 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4617 struct lro_ctrl *lro = &rxr->lro;
4618 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i,
4619 E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
4620 E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
4621 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me,
4622 (long long)rxr->rx_packets);
4623 device_printf(dev, "RX(%d) Split Packets = %lld\n", rxr->me,
4624 (long long)rxr->rx_split_packets);

--- 157 unchanged lines hidden (view full) ---

4782 const char *description, int *limit, int value)
4783{
4784 *limit = value;
4785 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4786 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4787 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
4788}
4789
4673 struct lro_ctrl *lro = &rxr->lro;
4674 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i,
4675 E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
4676 E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
4677 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me,
4678 (long long)rxr->rx_packets);
4679 device_printf(dev, "RX(%d) Split Packets = %lld\n", rxr->me,
4680 (long long)rxr->rx_split_packets);

--- 157 unchanged lines hidden (view full) ---

4838 const char *description, int *limit, int value)
4839{
4840 *limit = value;
4841 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4842 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4843 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
4844}
4845
4790#ifdef IGB_TIMESYNC
4846#ifdef IGB_IEEE1588
4791/*
4847/*
4792 * Initialize the Time Sync Feature
4793 */
4848** igb_hwtstamp_ioctl - control hardware time stamping
4849**
4850** Outgoing time stamping can be enabled and disabled. Play nice and
4851** disable it when requested, although it shouldn't case any overhead
4852** when no packet needs it. At most one packet in the queue may be
4853** marked for time stamping, otherwise it would be impossible to tell
4854** for sure to which packet the hardware time stamp belongs.
4855**
4856** Incoming time stamping has to be configured via the hardware
4857** filters. Not all combinations are supported, in particular event
4858** type has to be specified. Matching the kind of event packet is
4859** not supported, with the exception of "all V2 events regardless of
4860** level 2 or 4".
4861**
4862*/
4794static int
4863static int
4795igb_tsync_init(struct adapter *adapter)
4864igb_hwtstamp_ioctl(struct adapter *adapter, struct ifreq *ifr)
4796{
4865{
4797 device_t dev = adapter->dev;
4798 u32 tx_ctl, rx_ctl, val;
4866 struct e1000_hw *hw = &adapter->hw;
4867 struct hwtstamp_ctrl *config;
4868 u32 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
4869 u32 tsync_rx_ctl_bit = E1000_TSYNCRXCTL_ENABLED;
4870 u32 tsync_rx_ctl_type = 0;
4871 u32 tsync_rx_cfg = 0;
4872 int is_l4 = 0;
4873 int is_l2 = 0;
4874 u16 port = 319; /* PTP */
4875 u32 regval;
4799
4876
4877 config = (struct hwtstamp_ctrl *) ifr->ifr_data;
4800
4878
4801 E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) |
4802 20833/PICOSECS_PER_TICK);
4879 /* reserved for future extensions */
4880 if (config->flags)
4881 return (EINVAL);
4803
4882
4804 adapter->last_stamp = E1000_READ_REG(&adapter->hw, E1000_SYSTIML);
4805 adapter->last_stamp |= (u64)E1000_READ_REG(&adapter->hw,
4806 E1000_SYSTIMH) << 32ULL;
4883 switch (config->tx_type) {
4884 case HWTSTAMP_TX_OFF:
4885 tsync_tx_ctl_bit = 0;
4886 break;
4887 case HWTSTAMP_TX_ON:
4888 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
4889 break;
4890 default:
4891 return (ERANGE);
4892 }
4807
4893
4808 /* Enable the TX side */
4809 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
4810 tx_ctl |= 0x10;
4811 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl);
4812 E1000_WRITE_FLUSH(&adapter->hw);
4894 switch (config->rx_filter) {
4895 case HWTSTAMP_FILTER_NONE:
4896 tsync_rx_ctl_bit = 0;
4897 break;
4898 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
4899 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
4900 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
4901 case HWTSTAMP_FILTER_ALL:
4902 /*
4903 * register TSYNCRXCFG must be set, therefore it is not
4904 * possible to time stamp both Sync and Delay_Req messages
4905 * => fall back to time stamping all packets
4906 */
4907 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_ALL;
4908 config->rx_filter = HWTSTAMP_FILTER_ALL;
4909 break;
4910 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
4911 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
4912 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
4913 is_l4 = 1;
4914 break;
4915 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
4916 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
4917 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
4918 is_l4 = 1;
4919 break;
4920 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
4921 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
4922 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
4923 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
4924 is_l2 = 1;
4925 is_l4 = 1;
4926 config->rx_filter = HWTSTAMP_FILTER_SOME;
4927 break;
4928 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
4929 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
4930 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
4931 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
4932 is_l2 = 1;
4933 is_l4 = 1;
4934 config->rx_filter = HWTSTAMP_FILTER_SOME;
4935 break;
4936 case HWTSTAMP_FILTER_PTP_V2_EVENT:
4937 case HWTSTAMP_FILTER_PTP_V2_SYNC:
4938 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
4939 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_EVENT_V2;
4940 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
4941 is_l2 = 1;
4942 break;
4943 default:
4944 return -ERANGE;
4945 }
4813
4946
4814 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
4815 if ((tx_ctl & 0x10) == 0) {
4816 device_printf(dev, "Failed to enable TX timestamping\n");
4817 return (ENXIO);
4818 }
4947 /* enable/disable TX */
4948 regval = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4949 regval = (regval & ~E1000_TSYNCTXCTL_ENABLED) | tsync_tx_ctl_bit;
4950 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, regval);
4819
4951
4820 /* Enable RX */
4821 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
4822 rx_ctl |= 0x10; /* Enable the feature */
4823 rx_ctl |= 0x04; /* This value turns on Ver 1 and 2 */
4824 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl);
4952 /* enable/disable RX, define which PTP packets are time stamped */
4953 regval = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4954 regval = (regval & ~E1000_TSYNCRXCTL_ENABLED) | tsync_rx_ctl_bit;
4955 regval = (regval & ~0xE) | tsync_rx_ctl_type;
4956 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, regval);
4957 E1000_WRITE_REG(hw, E1000_TSYNCRXCFG, tsync_rx_cfg);
4825
4826 /*
4958
4959 /*
4827 * Ethertype Filter Queue Filter[0][15:0] = 0x88F7 (Ethertype)
4960 * Ethertype Filter Queue Filter[0][15:0] = 0x88F7
4961 * (Ethertype to filter on)
4828 * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter)
4962 * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter)
4829 * Ethertype Filter Queue Filter[0][31] = 0x1 (Enable Timestamping)
4963 * Ethertype Filter Queue Filter[0][30] = 0x1 (Enable Timestamping)
4830 */
4964 */
4831 E1000_WRITE_REG(&adapter->hw, E1000_ETQF(0), 0x440088f7);
4832 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCFG, 0x0);
4965 E1000_WRITE_REG(hw, E1000_ETQF0, is_l2 ? 0x440088f7 : 0);
4833
4966
4834 /*
4835 * Source Port Queue Filter Setup:
4836 * this is for UDP port filtering
4837 */
4838 E1000_WRITE_REG(&adapter->hw, E1000_SPQF(0), TSYNC_PORT);
4839 /* Protocol = UDP, enable Timestamp, and filter on source/protocol */
4840 val = (0x11 | (1 << 27) | (6 << 28));
4841 E1000_WRITE_REG(&adapter->hw, E1000_FTQF(0), val);
4967 /* L4 Queue Filter[0]: only filter by source and destination port */
4968 E1000_WRITE_REG(hw, E1000_SPQF0, htons(port));
4969 E1000_WRITE_REG(hw, E1000_IMIREXT(0), is_l4 ?
4970 ((1<<12) | (1<<19) /* bypass size and control flags */) : 0);
4971 E1000_WRITE_REG(hw, E1000_IMIR(0), is_l4 ?
4972 (htons(port)
4973 | (0<<16) /* immediate interrupt disabled */
4974 | 0 /* (1<<17) bit cleared: do not bypass
4975 destination port check */)
4976 : 0);
4977 E1000_WRITE_REG(hw, E1000_FTQF0, is_l4 ?
4978 (0x11 /* UDP */
4979 | (1<<15) /* VF not compared */
4980 | (1<<27) /* Enable Timestamping */
4981 | (7<<28) /* only source port filter enabled,
4982 source/target address and protocol
4983 masked */)
4984 : ((1<<15) | (15<<28) /* all mask bits set = filter not
4985 enabled */));
4842
4986
4843 E1000_WRITE_FLUSH(&adapter->hw);
4987 wrfl();
4844
4988
4845 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
4846 if ((rx_ctl & 0x10) == 0) {
4847 device_printf(dev, "Failed to enable RX timestamping\n");
4848 return (ENXIO);
4849 }
4989 adapter->hwtstamp_ctrl = config;
4850
4990
4851 device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n");
4991 /* clear TX/RX time stamp registers, just to be sure */
4992 regval = E1000_READ_REG(hw, E1000_TXSTMPH);
4993 regval = E1000_READ_REG(hw, E1000_RXSTMPH);
4852
4994
4853 return (0);
4995 return (error);
4854}
4855
4856/*
4996}
4997
4998/*
4857 * Disable the Time Sync Feature
4858 */
4859static void
4860igb_tsync_disable(struct adapter *adapter)
4999** igb_read_clock - read raw cycle counter (to be used by time counter)
5000*/
5001static cycle_t igb_read_clock(const struct cyclecounter *tc)
4861{
5002{
4862 u32 tx_ctl, rx_ctl;
4863
4864 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
4865 tx_ctl &= ~0x10;
4866 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl);
4867 E1000_WRITE_FLUSH(&adapter->hw);
4868
4869 /* Invalidate TX Timestamp */
4870 E1000_READ_REG(&adapter->hw, E1000_TXSTMPH);
4871
4872 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
4873 if (tx_ctl & 0x10)
4874 HW_DEBUGOUT("Failed to disable TX timestamping\n");
4875
4876 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
4877 rx_ctl &= ~0x10;
4878
4879 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl);
4880 E1000_WRITE_FLUSH(&adapter->hw);
4881
4882 /* Invalidate RX Timestamp */
4883 E1000_READ_REG(&adapter->hw, E1000_RXSATRH);
4884
4885 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
4886 if (rx_ctl & 0x10)
4887 HW_DEBUGOUT("Failed to disable RX timestamping\n");
4888
4889 return;
5003 struct igb_adapter *adapter =
5004 container_of(tc, struct igb_adapter, cycles);
5005 struct e1000_hw *hw = &adapter->hw;
5006 u64 stamp;
5007
5008 stamp = E1000_READ_REG(hw, E1000_SYSTIML);
5009 stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIMH) << 32ULL;
5010
5011 return (stamp);
4890}
4891
5012}
5013
4892#endif /* IGB_TIMESYNC */
5014#endif /* IGB_IEEE1588 */