Deleted Added
sdiff udiff text old ( 172162 ) new ( 175365 )
full compact
if_mxge.c (172162) if_mxge.c (175365)
1/******************************************************************************
2
1/******************************************************************************
2
3Copyright (c) 2006-2007, Myricom Inc.
3Copyright (c) 2006-2008, Myricom Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11

--- 11 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11

--- 11 unchanged lines hidden (view full) ---

23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/mxge/if_mxge.c 172162 2007-09-13 21:29:02Z gallatin $");
31__FBSDID("$FreeBSD: head/sys/dev/mxge/if_mxge.c 175365 2008-01-15 20:34:49Z gallatin $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/linker.h>
36#include <sys/firmware.h>
37#include <sys/endian.h>
38#include <sys/sockio.h>
39#include <sys/mbuf.h>

--- 24 unchanged lines hidden (view full) ---

64#include <netinet/ip.h>
65#include <netinet/tcp.h>
66
67#include <machine/bus.h>
68#include <machine/in_cksum.h>
69#include <machine/resource.h>
70#include <sys/bus.h>
71#include <sys/rman.h>
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/linker.h>
36#include <sys/firmware.h>
37#include <sys/endian.h>
38#include <sys/sockio.h>
39#include <sys/mbuf.h>

--- 24 unchanged lines hidden (view full) ---

64#include <netinet/ip.h>
65#include <netinet/tcp.h>
66
67#include <machine/bus.h>
68#include <machine/in_cksum.h>
69#include <machine/resource.h>
70#include <sys/bus.h>
71#include <sys/rman.h>
72#include <sys/smp.h>
72
73#include <dev/pci/pcireg.h>
74#include <dev/pci/pcivar.h>
75
76#include <vm/vm.h> /* for pmap_mapdev() */
77#include <vm/pmap.h>
78
79#if defined(__i386) || defined(__amd64)
80#include <machine/specialreg.h>
81#endif
82
83#include <dev/mxge/mxge_mcp.h>
84#include <dev/mxge/mcp_gen_header.h>
73
74#include <dev/pci/pcireg.h>
75#include <dev/pci/pcivar.h>
76
77#include <vm/vm.h> /* for pmap_mapdev() */
78#include <vm/pmap.h>
79
80#if defined(__i386) || defined(__amd64)
81#include <machine/specialreg.h>
82#endif
83
84#include <dev/mxge/mxge_mcp.h>
85#include <dev/mxge/mcp_gen_header.h>
86/*#define MXGE_FAKE_IFP*/
85#include <dev/mxge/if_mxge_var.h>
86
87/* tunable params */
88static int mxge_nvidia_ecrc_enable = 1;
89static int mxge_force_firmware = 0;
90static int mxge_intr_coal_delay = 30;
91static int mxge_deassert_wait = 1;
92static int mxge_flow_control = 1;
93static int mxge_verbose = 0;
94static int mxge_lro_cnt = 8;
95static int mxge_ticks;
87#include <dev/mxge/if_mxge_var.h>
88
89/* tunable params */
90static int mxge_nvidia_ecrc_enable = 1;
91static int mxge_force_firmware = 0;
92static int mxge_intr_coal_delay = 30;
93static int mxge_deassert_wait = 1;
94static int mxge_flow_control = 1;
95static int mxge_verbose = 0;
96static int mxge_lro_cnt = 8;
97static int mxge_ticks;
98static int mxge_max_slices = 1;
99static int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_PORT;
100static int mxge_always_promisc = 0;
96static char *mxge_fw_unaligned = "mxge_ethp_z8e";
97static char *mxge_fw_aligned = "mxge_eth_z8e";
101static char *mxge_fw_unaligned = "mxge_ethp_z8e";
102static char *mxge_fw_aligned = "mxge_eth_z8e";
103static char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e";
104static char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e";
98
99static int mxge_probe(device_t dev);
100static int mxge_attach(device_t dev);
101static int mxge_detach(device_t dev);
102static int mxge_shutdown(device_t dev);
103static void mxge_intr(void *arg);
104
105static device_method_t mxge_methods[] =

--- 15 unchanged lines hidden (view full) ---

121
122static devclass_t mxge_devclass;
123
124/* Declare ourselves to be a child of the PCI bus.*/
125DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0);
126MODULE_DEPEND(mxge, firmware, 1, 1, 1);
127MODULE_DEPEND(mxge, zlib, 1, 1, 1);
128
105
106static int mxge_probe(device_t dev);
107static int mxge_attach(device_t dev);
108static int mxge_detach(device_t dev);
109static int mxge_shutdown(device_t dev);
110static void mxge_intr(void *arg);
111
112static device_method_t mxge_methods[] =

--- 15 unchanged lines hidden (view full) ---

128
129static devclass_t mxge_devclass;
130
131/* Declare ourselves to be a child of the PCI bus.*/
132DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0);
133MODULE_DEPEND(mxge, firmware, 1, 1, 1);
134MODULE_DEPEND(mxge, zlib, 1, 1, 1);
135
129static int mxge_load_firmware(mxge_softc_t *sc);
136static int mxge_load_firmware(mxge_softc_t *sc, int adopt);
130static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data);
131static int mxge_close(mxge_softc_t *sc);
132static int mxge_open(mxge_softc_t *sc);
133static void mxge_tick(void *arg);
134
135static int
136mxge_probe(device_t dev)
137{

--- 52 unchanged lines hidden (view full) ---

190}
191
192static int
193mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes,
194 bus_size_t alignment)
195{
196 int err;
197 device_t dev = sc->dev;
137static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data);
138static int mxge_close(mxge_softc_t *sc);
139static int mxge_open(mxge_softc_t *sc);
140static void mxge_tick(void *arg);
141
142static int
143mxge_probe(device_t dev)
144{

--- 52 unchanged lines hidden (view full) ---

197}
198
199static int
200mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes,
201 bus_size_t alignment)
202{
203 int err;
204 device_t dev = sc->dev;
205 bus_size_t boundary, maxsegsize;
198
206
207 if (bytes > 4096 && alignment == 4096) {
208 boundary = 0;
209 maxsegsize = bytes;
210 } else {
211 boundary = 4096;
212 maxsegsize = 4096;
213 }
214
199 /* allocate DMAable memory tags */
200 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
201 alignment, /* alignment */
215 /* allocate DMAable memory tags */
216 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
217 alignment, /* alignment */
202 4096, /* boundary */
218 boundary, /* boundary */
203 BUS_SPACE_MAXADDR, /* low */
204 BUS_SPACE_MAXADDR, /* high */
205 NULL, NULL, /* filter */
206 bytes, /* maxsize */
207 1, /* num segs */
219 BUS_SPACE_MAXADDR, /* low */
220 BUS_SPACE_MAXADDR, /* high */
221 NULL, NULL, /* filter */
222 bytes, /* maxsize */
223 1, /* num segs */
208 4096, /* maxsegsize */
224 maxsegsize, /* maxsegsize */
209 BUS_DMA_COHERENT, /* flags */
210 NULL, NULL, /* lock */
211 &dma->dmat); /* tag */
212 if (err != 0) {
213 device_printf(dev, "couldn't alloc tag (err = %d)\n", err);
214 return err;
215 }
216

--- 231 unchanged lines hidden (view full) ---

448 * The magic multipliers to the length tell the firmware
449 * to do DMA read, write, or read+write tests. The
450 * results are returned in cmd.data0. The upper 16
451 * bits of the return is the number of transfers completed.
452 * The lower 16 bits is the time in 0.5us ticks that the
453 * transfers took to complete.
454 */
455
225 BUS_DMA_COHERENT, /* flags */
226 NULL, NULL, /* lock */
227 &dma->dmat); /* tag */
228 if (err != 0) {
229 device_printf(dev, "couldn't alloc tag (err = %d)\n", err);
230 return err;
231 }
232

--- 231 unchanged lines hidden (view full) ---

464 * The magic multipliers to the length tell the firmware
465 * to do DMA read, write, or read+write tests. The
466 * results are returned in cmd.data0. The upper 16
467 * bits of the return is the number of transfers completed.
468 * The lower 16 bits is the time in 0.5us ticks that the
469 * transfers took to complete.
470 */
471
456 len = sc->tx.boundary;
472 len = sc->tx_boundary;
457
458 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
459 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
460 cmd.data2 = len * 0x10000;
461 status = mxge_send_cmd(sc, test_type, &cmd);
462 if (status != 0) {
463 test = "read";
464 goto abort;

--- 39 unchanged lines hidden (view full) ---

504 *
505 * When PCI-E Completion packets are not aligned, it is actually more
506 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
507 *
508 * If the driver can neither enable ECRC nor verify that it has
509 * already been enabled, then it must use a firmware image which works
510 * around unaligned completion packets (ethp_z8e.dat), and it should
511 * also ensure that it never gives the device a Read-DMA which is
473
474 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
475 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
476 cmd.data2 = len * 0x10000;
477 status = mxge_send_cmd(sc, test_type, &cmd);
478 if (status != 0) {
479 test = "read";
480 goto abort;

--- 39 unchanged lines hidden (view full) ---

520 *
521 * When PCI-E Completion packets are not aligned, it is actually more
522 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
523 *
524 * If the driver can neither enable ECRC nor verify that it has
525 * already been enabled, then it must use a firmware image which works
526 * around unaligned completion packets (ethp_z8e.dat), and it should
527 * also ensure that it never gives the device a Read-DMA which is
512 * larger than 2KB by setting the tx.boundary to 2KB. If ECRC is
528 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is
513 * enabled, then the driver should use the aligned (eth_z8e.dat)
529 * enabled, then the driver should use the aligned (eth_z8e.dat)
514 * firmware image, and set tx.boundary to 4KB.
530 * firmware image, and set tx_boundary to 4KB.
515 */
516
517static int
518mxge_firmware_probe(mxge_softc_t *sc)
519{
520 device_t dev = sc->dev;
521 int reg, status;
522 uint16_t pectl;
523
531 */
532
533static int
534mxge_firmware_probe(mxge_softc_t *sc)
535{
536 device_t dev = sc->dev;
537 int reg, status;
538 uint16_t pectl;
539
524 sc->tx.boundary = 4096;
540 sc->tx_boundary = 4096;
525 /*
526 * Verify the max read request size was set to 4KB
527 * before trying the test with 4KB.
528 */
529 if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
530 pectl = pci_read_config(dev, reg + 0x8, 2);
531 if ((pectl & (5 << 12)) != (5 << 12)) {
532 device_printf(dev, "Max Read Req. size != 4k (0x%x\n",
533 pectl);
541 /*
542 * Verify the max read request size was set to 4KB
543 * before trying the test with 4KB.
544 */
545 if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
546 pectl = pci_read_config(dev, reg + 0x8, 2);
547 if ((pectl & (5 << 12)) != (5 << 12)) {
548 device_printf(dev, "Max Read Req. size != 4k (0x%x\n",
549 pectl);
534 sc->tx.boundary = 2048;
550 sc->tx_boundary = 2048;
535 }
536 }
537
538 /*
539 * load the optimized firmware (which assumes aligned PCIe
540 * completions) in order to see if it works on this host.
541 */
542 sc->fw_name = mxge_fw_aligned;
551 }
552 }
553
554 /*
555 * load the optimized firmware (which assumes aligned PCIe
556 * completions) in order to see if it works on this host.
557 */
558 sc->fw_name = mxge_fw_aligned;
543 status = mxge_load_firmware(sc);
559 status = mxge_load_firmware(sc, 1);
544 if (status != 0) {
545 return status;
546 }
547
548 /*
549 * Enable ECRC if possible
550 */
551 mxge_enable_nvidia_ecrc(sc);

--- 44 unchanged lines hidden (view full) ---

596 }
597
598 if (0 == mxge_firmware_probe(sc))
599 return 0;
600
601abort:
602 if (aligned) {
603 sc->fw_name = mxge_fw_aligned;
560 if (status != 0) {
561 return status;
562 }
563
564 /*
565 * Enable ECRC if possible
566 */
567 mxge_enable_nvidia_ecrc(sc);

--- 44 unchanged lines hidden (view full) ---

612 }
613
614 if (0 == mxge_firmware_probe(sc))
615 return 0;
616
617abort:
618 if (aligned) {
619 sc->fw_name = mxge_fw_aligned;
604 sc->tx.boundary = 4096;
620 sc->tx_boundary = 4096;
605 } else {
606 sc->fw_name = mxge_fw_unaligned;
621 } else {
622 sc->fw_name = mxge_fw_unaligned;
607 sc->tx.boundary = 2048;
623 sc->tx_boundary = 2048;
608 }
624 }
609 return (mxge_load_firmware(sc));
625 return (mxge_load_firmware(sc, 0));
610}
611
612union qualhack
613{
614 const char *ro_char;
615 char *rw_char;
616};
617

--- 300 unchanged lines hidden (view full) ---

918 sc->fw_ver_tiny);
919 }
920
921 return status;
922}
923
924
925static int
626}
627
628union qualhack
629{
630 const char *ro_char;
631 char *rw_char;
632};
633

--- 300 unchanged lines hidden (view full) ---

934 sc->fw_ver_tiny);
935 }
936
937 return status;
938}
939
940
941static int
926mxge_load_firmware(mxge_softc_t *sc)
942mxge_load_firmware(mxge_softc_t *sc, int adopt)
927{
928 volatile uint32_t *confirm;
929 volatile char *submit;
930 char buf_bytes[72];
931 uint32_t *buf, size, dma_low, dma_high;
932 int status, i;
933
934 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
935
936 size = sc->sram_size;
937 status = mxge_load_firmware_helper(sc, &size);
938 if (status) {
943{
944 volatile uint32_t *confirm;
945 volatile char *submit;
946 char buf_bytes[72];
947 uint32_t *buf, size, dma_low, dma_high;
948 int status, i;
949
950 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
951
952 size = sc->sram_size;
953 status = mxge_load_firmware_helper(sc, &size);
954 if (status) {
955 if (!adopt)
956 return status;
939 /* Try to use the currently running firmware, if
940 it is new enough */
941 status = mxge_adopt_running_firmware(sc);
942 if (status) {
943 device_printf(sc->dev,
944 "failed to adopt running firmware\n");
945 return status;
946 }
947 device_printf(sc->dev,
948 "Successfully adopted running firmware\n");
957 /* Try to use the currently running firmware, if
958 it is new enough */
959 status = mxge_adopt_running_firmware(sc);
960 if (status) {
961 device_printf(sc->dev,
962 "failed to adopt running firmware\n");
963 return status;
964 }
965 device_printf(sc->dev,
966 "Successfully adopted running firmware\n");
949 if (sc->tx.boundary == 4096) {
967 if (sc->tx_boundary == 4096) {
950 device_printf(sc->dev,
951 "Using firmware currently running on NIC"
952 ". For optimal\n");
953 device_printf(sc->dev,
954 "performance consider loading optimized "
955 "firmware\n");
956 }
957 sc->fw_name = mxge_fw_unaligned;
968 device_printf(sc->dev,
969 "Using firmware currently running on NIC"
970 ". For optimal\n");
971 device_printf(sc->dev,
972 "performance consider loading optimized "
973 "firmware\n");
974 }
975 sc->fw_name = mxge_fw_unaligned;
958 sc->tx.boundary = 2048;
976 sc->tx_boundary = 2048;
959 return 0;
960 }
961 /* clear confirmation addr */
962 confirm = (volatile uint32_t *)sc->cmd;
963 *confirm = 0;
964 mb();
965 /* send a reload command to the bootstrap MCP, and wait for the
966 response in the confirmation address. The firmware should

--- 77 unchanged lines hidden (view full) ---

1044}
1045
1046static void
1047mxge_change_promisc(mxge_softc_t *sc, int promisc)
1048{
1049 mxge_cmd_t cmd;
1050 int status;
1051
977 return 0;
978 }
979 /* clear confirmation addr */
980 confirm = (volatile uint32_t *)sc->cmd;
981 *confirm = 0;
982 mb();
983 /* send a reload command to the bootstrap MCP, and wait for the
984 response in the confirmation address. The firmware should

--- 77 unchanged lines hidden (view full) ---

1062}
1063
1064static void
1065mxge_change_promisc(mxge_softc_t *sc, int promisc)
1066{
1067 mxge_cmd_t cmd;
1068 int status;
1069
1070 if (mxge_always_promisc)
1071 promisc = 1;
1072
1052 if (promisc)
1053 status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC,
1054 &cmd);
1055 else
1056 status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC,
1057 &cmd);
1058
1059 if (status) {

--- 88 unchanged lines hidden (view full) ---

1148
1149 /* otherwise, we're limited to MJUMPAGESIZE */
1150 return MJUMPAGESIZE - MXGEFW_PAD;
1151}
1152
1153static int
1154mxge_reset(mxge_softc_t *sc, int interrupts_setup)
1155{
1073 if (promisc)
1074 status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC,
1075 &cmd);
1076 else
1077 status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC,
1078 &cmd);
1079
1080 if (status) {

--- 88 unchanged lines hidden (view full) ---

1169
1170 /* otherwise, we're limited to MJUMPAGESIZE */
1171 return MJUMPAGESIZE - MXGEFW_PAD;
1172}
1173
1174static int
1175mxge_reset(mxge_softc_t *sc, int interrupts_setup)
1176{
1156
1177 struct mxge_slice_state *ss;
1178 mxge_rx_done_t *rx_done;
1179 volatile uint32_t *irq_claim;
1157 mxge_cmd_t cmd;
1180 mxge_cmd_t cmd;
1158 size_t bytes;
1159 int status;
1181 int slice, status;
1160
1161 /* try to send a reset command to the card to see if it
1162 is alive */
1163 memset(&cmd, 0, sizeof (cmd));
1164 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
1165 if (status != 0) {
1166 device_printf(sc->dev, "failed reset\n");
1167 return ENXIO;
1168 }
1169
1170 mxge_dummy_rdma(sc, 1);
1171
1182
1183 /* try to send a reset command to the card to see if it
1184 is alive */
1185 memset(&cmd, 0, sizeof (cmd));
1186 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
1187 if (status != 0) {
1188 device_printf(sc->dev, "failed reset\n");
1189 return ENXIO;
1190 }
1191
1192 mxge_dummy_rdma(sc, 1);
1193
1194
1195 /* set the intrq size */
1196 cmd.data0 = sc->rx_ring_size;
1197 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
1198
1199 /*
1200 * Even though we already know how many slices are supported
1201 * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES
1202 * has magic side effects, and must be called after a reset.
1203 * It must be called prior to calling any RSS related cmds,
1204 * including assigning an interrupt queue for anything but
1205 * slice 0. It must also be called *after*
1206 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
1207 * the firmware to compute offsets.
1208 */
1209
1210 if (sc->num_slices > 1) {
1211 /* ask the maximum number of slices it supports */
1212 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
1213 &cmd);
1214 if (status != 0) {
1215 device_printf(sc->dev,
1216 "failed to get number of slices\n");
1217 return status;
1218 }
1219 /*
1220 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
1221 * to setting up the interrupt queue DMA
1222 */
1223 cmd.data0 = sc->num_slices;
1224 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
1225 status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES,
1226 &cmd);
1227 if (status != 0) {
1228 device_printf(sc->dev,
1229 "failed to set number of slices\n");
1230 return status;
1231 }
1232 }
1233
1234
1172 if (interrupts_setup) {
1173 /* Now exchange information about interrupts */
1235 if (interrupts_setup) {
1236 /* Now exchange information about interrupts */
1174 bytes = (sc->rx_done.mask + 1) * sizeof (*sc->rx_done.entry);
1175 memset(sc->rx_done.entry, 0, bytes);
1176 cmd.data0 = (uint32_t)bytes;
1177 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
1178 cmd.data0 = MXGE_LOWPART_TO_U32(sc->rx_done.dma.bus_addr);
1179 cmd.data1 = MXGE_HIGHPART_TO_U32(sc->rx_done.dma.bus_addr);
1180 status |= mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_DMA, &cmd);
1237 for (slice = 0; slice < sc->num_slices; slice++) {
1238 rx_done = &sc->ss[slice].rx_done;
1239 memset(rx_done->entry, 0, sc->rx_ring_size);
1240 cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr);
1241 cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr);
1242 cmd.data2 = slice;
1243 status |= mxge_send_cmd(sc,
1244 MXGEFW_CMD_SET_INTRQ_DMA,
1245 &cmd);
1246 }
1181 }
1182
1183 status |= mxge_send_cmd(sc,
1184 MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd);
1185
1186
1187 sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0);
1188
1189 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
1247 }
1248
1249 status |= mxge_send_cmd(sc,
1250 MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd);
1251
1252
1253 sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0);
1254
1255 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
1190 sc->irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0);
1256 irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0);
1191
1192
1193 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET,
1194 &cmd);
1195 sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0);
1196 if (status != 0) {
1197 device_printf(sc->dev, "failed set interrupt parameters\n");
1198 return status;
1199 }
1200
1201
1202 *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay);
1203
1204
1205 /* run a DMA benchmark */
1206 (void) mxge_dma_test(sc, MXGEFW_DMA_TEST);
1207
1257
1258
1259 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET,
1260 &cmd);
1261 sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0);
1262 if (status != 0) {
1263 device_printf(sc->dev, "failed set interrupt parameters\n");
1264 return status;
1265 }
1266
1267
1268 *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay);
1269
1270
1271 /* run a DMA benchmark */
1272 (void) mxge_dma_test(sc, MXGEFW_DMA_TEST);
1273
1208 /* reset mcp/driver shared state back to 0 */
1209 sc->rx_done.idx = 0;
1210 sc->rx_done.cnt = 0;
1211 sc->tx.req = 0;
1212 sc->tx.done = 0;
1213 sc->tx.pkt_done = 0;
1214 sc->tx.wake = 0;
1215 sc->tx_defrag = 0;
1216 sc->tx.stall = 0;
1217 sc->rx_big.cnt = 0;
1218 sc->rx_small.cnt = 0;
1274 for (slice = 0; slice < sc->num_slices; slice++) {
1275 ss = &sc->ss[slice];
1276
1277 ss->irq_claim = irq_claim + (2 * slice);
1278 /* reset mcp/driver shared state back to 0 */
1279 ss->rx_done.idx = 0;
1280 ss->rx_done.cnt = 0;
1281 ss->tx.req = 0;
1282 ss->tx.done = 0;
1283 ss->tx.pkt_done = 0;
1284 ss->tx.wake = 0;
1285 ss->tx.defrag = 0;
1286 ss->tx.stall = 0;
1287 ss->rx_big.cnt = 0;
1288 ss->rx_small.cnt = 0;
1289 ss->lro_bad_csum = 0;
1290 ss->lro_queued = 0;
1291 ss->lro_flushed = 0;
1292 if (ss->fw_stats != NULL) {
1293 ss->fw_stats->valid = 0;
1294 ss->fw_stats->send_done_count = 0;
1295 }
1296 }
1219 sc->rdma_tags_available = 15;
1297 sc->rdma_tags_available = 15;
1220 sc->fw_stats->valid = 0;
1221 sc->fw_stats->send_done_count = 0;
1222 sc->lro_bad_csum = 0;
1223 sc->lro_queued = 0;
1224 sc->lro_flushed = 0;
1225 status = mxge_update_mac_address(sc);
1226 mxge_change_promisc(sc, 0);
1227 mxge_change_pause(sc, sc->pause);
1228 mxge_set_multicast_list(sc);
1229 return status;
1230}
1231
1232static int

--- 102 unchanged lines hidden (view full) ---

1335 arg2 = be32toh(*(int *)arg1);
1336 arg1 = NULL;
1337 err = sysctl_handle_int(oidp, arg1, arg2, req);
1338
1339 return err;
1340}
1341
1342static void
1298 status = mxge_update_mac_address(sc);
1299 mxge_change_promisc(sc, 0);
1300 mxge_change_pause(sc, sc->pause);
1301 mxge_set_multicast_list(sc);
1302 return status;
1303}
1304
1305static int

--- 102 unchanged lines hidden (view full) ---

1408 arg2 = be32toh(*(int *)arg1);
1409 arg1 = NULL;
1410 err = sysctl_handle_int(oidp, arg1, arg2, req);
1411
1412 return err;
1413}
1414
1415static void
1416mxge_rem_sysctls(mxge_softc_t *sc)
1417{
1418 struct mxge_slice_state *ss;
1419 int slice;
1420
1421 if (sc->slice_sysctl_tree == NULL)
1422 return;
1423
1424 for (slice = 0; slice < sc->num_slices; slice++) {
1425 ss = &sc->ss[slice];
1426 if (ss == NULL || ss->sysctl_tree == NULL)
1427 continue;
1428 sysctl_ctx_free(&ss->sysctl_ctx);
1429 ss->sysctl_tree = NULL;
1430 }
1431 sysctl_ctx_free(&sc->slice_sysctl_ctx);
1432 sc->slice_sysctl_tree = NULL;
1433}
1434
1435static void
1343mxge_add_sysctls(mxge_softc_t *sc)
1344{
1345 struct sysctl_ctx_list *ctx;
1346 struct sysctl_oid_list *children;
1347 mcp_irq_data_t *fw;
1436mxge_add_sysctls(mxge_softc_t *sc)
1437{
1438 struct sysctl_ctx_list *ctx;
1439 struct sysctl_oid_list *children;
1440 mcp_irq_data_t *fw;
1441 struct mxge_slice_state *ss;
1442 int slice;
1443 char slice_num[8];
1348
1349 ctx = device_get_sysctl_ctx(sc->dev);
1350 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
1444
1445 ctx = device_get_sysctl_ctx(sc->dev);
1446 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
1351 fw = sc->fw_stats;
1447 fw = sc->ss[0].fw_stats;
1352
1353 /* random information */
1354 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1355 "firmware_version",
1356 CTLFLAG_RD, &sc->fw_version,
1357 0, "firmware version");
1358 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1359 "serial_number",

--- 4 unchanged lines hidden (view full) ---

1364 CTLFLAG_RD, &sc->product_code_string,
1365 0, "product_code");
1366 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1367 "pcie_link_width",
1368 CTLFLAG_RD, &sc->link_width,
1369 0, "tx_boundary");
1370 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1371 "tx_boundary",
1448
1449 /* random information */
1450 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1451 "firmware_version",
1452 CTLFLAG_RD, &sc->fw_version,
1453 0, "firmware version");
1454 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1455 "serial_number",

--- 4 unchanged lines hidden (view full) ---

1460 CTLFLAG_RD, &sc->product_code_string,
1461 0, "product_code");
1462 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1463 "pcie_link_width",
1464 CTLFLAG_RD, &sc->link_width,
1465 0, "tx_boundary");
1466 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1467 "tx_boundary",
1372 CTLFLAG_RD, &sc->tx.boundary,
1468 CTLFLAG_RD, &sc->tx_boundary,
1373 0, "tx_boundary");
1374 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1375 "write_combine",
1376 CTLFLAG_RD, &sc->wc,
1377 0, "write combining PIO?");
1378 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1379 "read_dma_MBs",
1380 CTLFLAG_RD, &sc->read_dma,

--- 96 unchanged lines hidden (view full) ---

1477 "I", "dropped_runt");
1478
1479 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1480 "dropped_unicast_filtered",
1481 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered,
1482 0, mxge_handle_be32,
1483 "I", "dropped_unicast_filtered");
1484
1469 0, "tx_boundary");
1470 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1471 "write_combine",
1472 CTLFLAG_RD, &sc->wc,
1473 0, "write combining PIO?");
1474 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1475 "read_dma_MBs",
1476 CTLFLAG_RD, &sc->read_dma,

--- 96 unchanged lines hidden (view full) ---

1573 "I", "dropped_runt");
1574
1575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1576 "dropped_unicast_filtered",
1577 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered,
1578 0, mxge_handle_be32,
1579 "I", "dropped_unicast_filtered");
1580
1485 /* host counters exported for debugging */
1486 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1487 "rx_small_cnt",
1488 CTLFLAG_RD, &sc->rx_small.cnt,
1489 0, "rx_small_cnt");
1490 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1491 "rx_big_cnt",
1492 CTLFLAG_RD, &sc->rx_big.cnt,
1493 0, "rx_small_cnt");
1494 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1495 "tx_req",
1496 CTLFLAG_RD, &sc->tx.req,
1497 0, "tx_req");
1498 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1499 "tx_done",
1500 CTLFLAG_RD, &sc->tx.done,
1501 0, "tx_done");
1502 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1503 "tx_pkt_done",
1504 CTLFLAG_RD, &sc->tx.pkt_done,
1505 0, "tx_done");
1506 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1507 "tx_stall",
1508 CTLFLAG_RD, &sc->tx.stall,
1509 0, "tx_stall");
1510 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1511 "tx_wake",
1512 CTLFLAG_RD, &sc->tx.wake,
1513 0, "tx_wake");
1514 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1515 "tx_defrag",
1516 CTLFLAG_RD, &sc->tx_defrag,
1517 0, "tx_defrag");
1518
1519 /* verbose printing? */
1520 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1521 "verbose",
1522 CTLFLAG_RW, &mxge_verbose,
1523 0, "verbose printing");
1524
1525 /* lro */
1526 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1527 "lro_cnt",
1528 CTLTYPE_INT|CTLFLAG_RW, sc,
1529 0, mxge_change_lro,
1530 "I", "number of lro merge queues");
1531
1581 /* verbose printing? */
1582 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1583 "verbose",
1584 CTLFLAG_RW, &mxge_verbose,
1585 0, "verbose printing");
1586
1587 /* lro */
1588 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1589 "lro_cnt",
1590 CTLTYPE_INT|CTLFLAG_RW, sc,
1591 0, mxge_change_lro,
1592 "I", "number of lro merge queues");
1593
1532 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1533 "lro_flushed", CTLFLAG_RD, &sc->lro_flushed,
1534 0, "number of lro merge queues flushed");
1535
1594
1536 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1537 "lro_queued", CTLFLAG_RD, &sc->lro_queued,
1538 0, "number of frames appended to lro merge queues");
1595 /* add counters exported for debugging from all slices */
1596 sysctl_ctx_init(&sc->slice_sysctl_ctx);
1597 sc->slice_sysctl_tree =
1598 SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO,
1599 "slice", CTLFLAG_RD, 0, "");
1539
1600
1601 for (slice = 0; slice < sc->num_slices; slice++) {
1602 ss = &sc->ss[slice];
1603 sysctl_ctx_init(&ss->sysctl_ctx);
1604 ctx = &ss->sysctl_ctx;
1605 children = SYSCTL_CHILDREN(sc->slice_sysctl_tree);
1606 sprintf(slice_num, "%d", slice);
1607 ss->sysctl_tree =
1608 SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num,
1609 CTLFLAG_RD, 0, "");
1610 children = SYSCTL_CHILDREN(ss->sysctl_tree);
1611 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1612 "rx_small_cnt",
1613 CTLFLAG_RD, &ss->rx_small.cnt,
1614 0, "rx_small_cnt");
1615 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1616 "rx_big_cnt",
1617 CTLFLAG_RD, &ss->rx_big.cnt,
1618 0, "rx_small_cnt");
1619 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1620 "tx_req",
1621 CTLFLAG_RD, &ss->tx.req,
1622 0, "tx_req");
1623 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1624 "lro_flushed", CTLFLAG_RD, &ss->lro_flushed,
1625 0, "number of lro merge queues flushed");
1626
1627 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1628 "lro_queued", CTLFLAG_RD, &ss->lro_queued,
1629 0, "number of frames appended to lro merge"
1630 "queues");
1631
1632 /* only transmit from slice 0 for now */
1633 if (slice > 0)
1634 continue;
1635
1636 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1637 "tx_done",
1638 CTLFLAG_RD, &ss->tx.done,
1639 0, "tx_done");
1640 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1641 "tx_pkt_done",
1642 CTLFLAG_RD, &ss->tx.pkt_done,
1643 0, "tx_done");
1644 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1645 "tx_stall",
1646 CTLFLAG_RD, &ss->tx.stall,
1647 0, "tx_stall");
1648 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1649 "tx_wake",
1650 CTLFLAG_RD, &ss->tx.wake,
1651 0, "tx_wake");
1652 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1653 "tx_defrag",
1654 CTLFLAG_RD, &ss->tx.defrag,
1655 0, "tx_defrag");
1656 }
1540}
1541
1542/* copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1543 backwards one at a time and handle ring wraps */
1544
1545static inline void
1657}
1658
1659/* copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1660 backwards one at a time and handle ring wraps */
1661
1662static inline void
1546mxge_submit_req_backwards(mxge_tx_buf_t *tx,
1663mxge_submit_req_backwards(mxge_tx_ring_t *tx,
1547 mcp_kreq_ether_send_t *src, int cnt)
1548{
1549 int idx, starting_slot;
1550 starting_slot = tx->req;
1551 while (cnt > 1) {
1552 cnt--;
1553 idx = (starting_slot + cnt) & tx->mask;
1554 mxge_pio_copy(&tx->lanai[idx],

--- 5 unchanged lines hidden (view full) ---

1560/*
1561 * copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1562 * at most 32 bytes at a time, so as to avoid involving the software
1563 * pio handler in the nic. We re-write the first segment's flags
1564 * to mark them valid only after writing the entire chain
1565 */
1566
1567static inline void
1664 mcp_kreq_ether_send_t *src, int cnt)
1665{
1666 int idx, starting_slot;
1667 starting_slot = tx->req;
1668 while (cnt > 1) {
1669 cnt--;
1670 idx = (starting_slot + cnt) & tx->mask;
1671 mxge_pio_copy(&tx->lanai[idx],

--- 5 unchanged lines hidden (view full) ---

1677/*
1678 * copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1679 * at most 32 bytes at a time, so as to avoid involving the software
1680 * pio handler in the nic. We re-write the first segment's flags
1681 * to mark them valid only after writing the entire chain
1682 */
1683
1684static inline void
1568mxge_submit_req(mxge_tx_buf_t *tx, mcp_kreq_ether_send_t *src,
1685mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src,
1569 int cnt)
1570{
1571 int idx, i;
1572 uint32_t *src_ints;
1573 volatile uint32_t *dst_ints;
1574 mcp_kreq_ether_send_t *srcp;
1575 volatile mcp_kreq_ether_send_t *dstp, *dst;
1576 uint8_t last_flags;

--- 32 unchanged lines hidden (view full) ---

1609 dst_ints = (volatile uint32_t *)dst;
1610 dst_ints+=3;
1611 *dst_ints = *src_ints;
1612 tx->req += cnt;
1613 mb();
1614}
1615
1616static void
1686 int cnt)
1687{
1688 int idx, i;
1689 uint32_t *src_ints;
1690 volatile uint32_t *dst_ints;
1691 mcp_kreq_ether_send_t *srcp;
1692 volatile mcp_kreq_ether_send_t *dstp, *dst;
1693 uint8_t last_flags;

--- 32 unchanged lines hidden (view full) ---

1726 dst_ints = (volatile uint32_t *)dst;
1727 dst_ints+=3;
1728 *dst_ints = *src_ints;
1729 tx->req += cnt;
1730 mb();
1731}
1732
1733static void
1617mxge_encap_tso(mxge_softc_t *sc, struct mbuf *m, int busdma_seg_cnt,
1618 int ip_off)
1734mxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m,
1735 int busdma_seg_cnt, int ip_off)
1619{
1736{
1620 mxge_tx_buf_t *tx;
1737 mxge_tx_ring_t *tx;
1621 mcp_kreq_ether_send_t *req;
1622 bus_dma_segment_t *seg;
1623 struct ip *ip;
1624 struct tcphdr *tcp;
1625 uint32_t low, high_swapped;
1626 int len, seglen, cum_len, cum_len_next;
1627 int next_is_first, chop, cnt, rdma_count, small;
1628 uint16_t pseudo_hdr_offset, cksum_offset, mss;
1629 uint8_t flags, flags_next;
1630 static int once;
1738 mcp_kreq_ether_send_t *req;
1739 bus_dma_segment_t *seg;
1740 struct ip *ip;
1741 struct tcphdr *tcp;
1742 uint32_t low, high_swapped;
1743 int len, seglen, cum_len, cum_len_next;
1744 int next_is_first, chop, cnt, rdma_count, small;
1745 uint16_t pseudo_hdr_offset, cksum_offset, mss;
1746 uint8_t flags, flags_next;
1747 static int once;
1631
1748
1632 mss = m->m_pkthdr.tso_segsz;
1633
1634 /* negative cum_len signifies to the
1635 * send loop that we are still in the
1636 * header portion of the TSO packet.
1637 */
1638
1639 /* ensure we have the ethernet, IP and TCP
1640 header together in the first mbuf, copy
1641 it to a scratch buffer if not */
1642 if (__predict_false(m->m_len < ip_off + sizeof (*ip))) {
1643 m_copydata(m, 0, ip_off + sizeof (*ip),
1749 mss = m->m_pkthdr.tso_segsz;
1750
1751 /* negative cum_len signifies to the
1752 * send loop that we are still in the
1753 * header portion of the TSO packet.
1754 */
1755
1756 /* ensure we have the ethernet, IP and TCP
1757 header together in the first mbuf, copy
1758 it to a scratch buffer if not */
1759 if (__predict_false(m->m_len < ip_off + sizeof (*ip))) {
1760 m_copydata(m, 0, ip_off + sizeof (*ip),
1644 sc->scratch);
1645 ip = (struct ip *)(sc->scratch + ip_off);
1761 ss->scratch);
1762 ip = (struct ip *)(ss->scratch + ip_off);
1646 } else {
1647 ip = (struct ip *)(mtod(m, char *) + ip_off);
1648 }
1649 if (__predict_false(m->m_len < ip_off + (ip->ip_hl << 2)
1650 + sizeof (*tcp))) {
1651 m_copydata(m, 0, ip_off + (ip->ip_hl << 2)
1763 } else {
1764 ip = (struct ip *)(mtod(m, char *) + ip_off);
1765 }
1766 if (__predict_false(m->m_len < ip_off + (ip->ip_hl << 2)
1767 + sizeof (*tcp))) {
1768 m_copydata(m, 0, ip_off + (ip->ip_hl << 2)
1652 + sizeof (*tcp), sc->scratch);
1769 + sizeof (*tcp), ss->scratch);
1653 ip = (struct ip *)(mtod(m, char *) + ip_off);
1654 }
1655
1656 tcp = (struct tcphdr *)((char *)ip + (ip->ip_hl << 2));
1657 cum_len = -(ip_off + ((ip->ip_hl + tcp->th_off) << 2));
1658
1659 /* TSO implies checksum offload on this hardware */
1660 cksum_offset = ip_off + (ip->ip_hl << 2);
1661 flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST;
1662
1663
1664 /* for TSO, pseudo_hdr_offset holds mss.
1665 * The firmware figures out where to put
1666 * the checksum by parsing the header. */
1667 pseudo_hdr_offset = htobe16(mss);
1668
1770 ip = (struct ip *)(mtod(m, char *) + ip_off);
1771 }
1772
1773 tcp = (struct tcphdr *)((char *)ip + (ip->ip_hl << 2));
1774 cum_len = -(ip_off + ((ip->ip_hl + tcp->th_off) << 2));
1775
1776 /* TSO implies checksum offload on this hardware */
1777 cksum_offset = ip_off + (ip->ip_hl << 2);
1778 flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST;
1779
1780
1781 /* for TSO, pseudo_hdr_offset holds mss.
1782 * The firmware figures out where to put
1783 * the checksum by parsing the header. */
1784 pseudo_hdr_offset = htobe16(mss);
1785
1669 tx = &sc->tx;
1786 tx = &ss->tx;
1670 req = tx->req_list;
1671 seg = tx->seg_list;
1672 cnt = 0;
1673 rdma_count = 0;
1674 /* "rdma_count" is the number of RDMAs belonging to the
1675 * current packet BEFORE the current send request. For
1676 * non-TSO packets, this is equal to "count".
1677 * For TSO packets, rdma_count needs to be reset

--- 78 unchanged lines hidden (view full) ---

1756
1757 tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
1758 mxge_submit_req(tx, tx->req_list, cnt);
1759 return;
1760
1761drop:
1762 bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map);
1763 m_freem(m);
1787 req = tx->req_list;
1788 seg = tx->seg_list;
1789 cnt = 0;
1790 rdma_count = 0;
1791 /* "rdma_count" is the number of RDMAs belonging to the
1792 * current packet BEFORE the current send request. For
1793 * non-TSO packets, this is equal to "count".
1794 * For TSO packets, rdma_count needs to be reset

--- 78 unchanged lines hidden (view full) ---

1873
1874 tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
1875 mxge_submit_req(tx, tx->req_list, cnt);
1876 return;
1877
1878drop:
1879 bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map);
1880 m_freem(m);
1764 sc->ifp->if_oerrors++;
1881 ss->sc->ifp->if_oerrors++;
1765 if (!once) {
1766 printf("tx->max_desc exceeded via TSO!\n");
1767 printf("mss = %d, %ld, %d!\n", mss,
1768 (long)seg - (long)tx->seg_list, tx->max_desc);
1769 once = 1;
1770 }
1771 return;
1772

--- 27 unchanged lines hidden (view full) ---

1800 (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
1801 evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1802 evl->evl_tag = htons(m->m_pkthdr.ether_vtag);
1803 m->m_flags &= ~M_VLANTAG;
1804 return m;
1805}
1806
1807static void
1882 if (!once) {
1883 printf("tx->max_desc exceeded via TSO!\n");
1884 printf("mss = %d, %ld, %d!\n", mss,
1885 (long)seg - (long)tx->seg_list, tx->max_desc);
1886 once = 1;
1887 }
1888 return;
1889

--- 27 unchanged lines hidden (view full) ---

1917 (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
1918 evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1919 evl->evl_tag = htons(m->m_pkthdr.ether_vtag);
1920 m->m_flags &= ~M_VLANTAG;
1921 return m;
1922}
1923
1924static void
1808mxge_encap(mxge_softc_t *sc, struct mbuf *m)
1925mxge_encap(struct mxge_slice_state *ss, struct mbuf *m)
1809{
1926{
1927 mxge_softc_t *sc;
1810 mcp_kreq_ether_send_t *req;
1811 bus_dma_segment_t *seg;
1812 struct mbuf *m_tmp;
1813 struct ifnet *ifp;
1928 mcp_kreq_ether_send_t *req;
1929 bus_dma_segment_t *seg;
1930 struct mbuf *m_tmp;
1931 struct ifnet *ifp;
1814 mxge_tx_buf_t *tx;
1932 mxge_tx_ring_t *tx;
1815 struct ip *ip;
1816 int cnt, cum_len, err, i, idx, odd_flag, ip_off;
1817 uint16_t pseudo_hdr_offset;
1818 uint8_t flags, cksum_offset;
1819
1820
1933 struct ip *ip;
1934 int cnt, cum_len, err, i, idx, odd_flag, ip_off;
1935 uint16_t pseudo_hdr_offset;
1936 uint8_t flags, cksum_offset;
1937
1938
1821
1939 sc = ss->sc;
1822 ifp = sc->ifp;
1940 ifp = sc->ifp;
1823 tx = &sc->tx;
1941 tx = &ss->tx;
1824
1825 ip_off = sizeof (struct ether_header);
1826 if (m->m_flags & M_VLANTAG) {
1827 m = mxge_vlan_tag_insert(m);
1828 if (__predict_false(m == NULL))
1829 goto drop;
1830 ip_off += ETHER_VLAN_ENCAP_LEN;
1831 }

--- 5 unchanged lines hidden (view full) ---

1837 BUS_DMA_NOWAIT);
1838 if (__predict_false(err == EFBIG)) {
1839 /* Too many segments in the chain. Try
1840 to defrag */
1841 m_tmp = m_defrag(m, M_NOWAIT);
1842 if (m_tmp == NULL) {
1843 goto drop;
1844 }
1942
1943 ip_off = sizeof (struct ether_header);
1944 if (m->m_flags & M_VLANTAG) {
1945 m = mxge_vlan_tag_insert(m);
1946 if (__predict_false(m == NULL))
1947 goto drop;
1948 ip_off += ETHER_VLAN_ENCAP_LEN;
1949 }

--- 5 unchanged lines hidden (view full) ---

1955 BUS_DMA_NOWAIT);
1956 if (__predict_false(err == EFBIG)) {
1957 /* Too many segments in the chain. Try
1958 to defrag */
1959 m_tmp = m_defrag(m, M_NOWAIT);
1960 if (m_tmp == NULL) {
1961 goto drop;
1962 }
1845 sc->tx_defrag++;
1963 ss->tx.defrag++;
1846 m = m_tmp;
1847 err = bus_dmamap_load_mbuf_sg(tx->dmat,
1848 tx->info[idx].map,
1849 m, tx->seg_list, &cnt,
1850 BUS_DMA_NOWAIT);
1851 }
1852 if (__predict_false(err != 0)) {
1853 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d"
1854 " packet len = %d\n", err, m->m_pkthdr.len);
1855 goto drop;
1856 }
1857 bus_dmamap_sync(tx->dmat, tx->info[idx].map,
1858 BUS_DMASYNC_PREWRITE);
1859 tx->info[idx].m = m;
1860
1861
1862 /* TSO is different enough, we handle it in another routine */
1863 if (m->m_pkthdr.csum_flags & (CSUM_TSO)) {
1964 m = m_tmp;
1965 err = bus_dmamap_load_mbuf_sg(tx->dmat,
1966 tx->info[idx].map,
1967 m, tx->seg_list, &cnt,
1968 BUS_DMA_NOWAIT);
1969 }
1970 if (__predict_false(err != 0)) {
1971 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d"
1972 " packet len = %d\n", err, m->m_pkthdr.len);
1973 goto drop;
1974 }
1975 bus_dmamap_sync(tx->dmat, tx->info[idx].map,
1976 BUS_DMASYNC_PREWRITE);
1977 tx->info[idx].m = m;
1978
1979
1980 /* TSO is different enough, we handle it in another routine */
1981 if (m->m_pkthdr.csum_flags & (CSUM_TSO)) {
1864 mxge_encap_tso(sc, m, cnt, ip_off);
1982 mxge_encap_tso(ss, m, cnt, ip_off);
1865 return;
1866 }
1867
1868 req = tx->req_list;
1869 cksum_offset = 0;
1870 pseudo_hdr_offset = 0;
1871 flags = MXGEFW_FLAGS_NO_TSO;
1872
1873 /* checksum offloading? */
1874 if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA)) {
1875 /* ensure ip header is in first mbuf, copy
1876 it to a scratch buffer if not */
1877 if (__predict_false(m->m_len < ip_off + sizeof (*ip))) {
1878 m_copydata(m, 0, ip_off + sizeof (*ip),
1983 return;
1984 }
1985
1986 req = tx->req_list;
1987 cksum_offset = 0;
1988 pseudo_hdr_offset = 0;
1989 flags = MXGEFW_FLAGS_NO_TSO;
1990
1991 /* checksum offloading? */
1992 if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA)) {
1993 /* ensure ip header is in first mbuf, copy
1994 it to a scratch buffer if not */
1995 if (__predict_false(m->m_len < ip_off + sizeof (*ip))) {
1996 m_copydata(m, 0, ip_off + sizeof (*ip),
1879 sc->scratch);
1880 ip = (struct ip *)(sc->scratch + ip_off);
1997 ss->scratch);
1998 ip = (struct ip *)(ss->scratch + ip_off);
1881 } else {
1882 ip = (struct ip *)(mtod(m, char *) + ip_off);
1883 }
1884 cksum_offset = ip_off + (ip->ip_hl << 2);
1885 pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data;
1886 pseudo_hdr_offset = htobe16(pseudo_hdr_offset);
1887 req->cksum_offset = cksum_offset;
1888 flags |= MXGEFW_FLAGS_CKSUM;

--- 69 unchanged lines hidden (view full) ---

1958 ifp->if_oerrors++;
1959 return;
1960}
1961
1962
1963
1964
1965static inline void
1999 } else {
2000 ip = (struct ip *)(mtod(m, char *) + ip_off);
2001 }
2002 cksum_offset = ip_off + (ip->ip_hl << 2);
2003 pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data;
2004 pseudo_hdr_offset = htobe16(pseudo_hdr_offset);
2005 req->cksum_offset = cksum_offset;
2006 flags |= MXGEFW_FLAGS_CKSUM;

--- 69 unchanged lines hidden (view full) ---

2076 ifp->if_oerrors++;
2077 return;
2078}
2079
2080
2081
2082
2083static inline void
1966mxge_start_locked(mxge_softc_t *sc)
2084mxge_start_locked(struct mxge_slice_state *ss)
1967{
2085{
2086 mxge_softc_t *sc;
1968 struct mbuf *m;
1969 struct ifnet *ifp;
2087 struct mbuf *m;
2088 struct ifnet *ifp;
1970 mxge_tx_buf_t *tx;
2089 mxge_tx_ring_t *tx;
1971
2090
2091 sc = ss->sc;
1972 ifp = sc->ifp;
2092 ifp = sc->ifp;
1973 tx = &sc->tx;
2093 tx = &ss->tx;
1974 while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) {
1975 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
1976 if (m == NULL) {
1977 return;
1978 }
1979 /* let BPF see it */
1980 BPF_MTAP(ifp, m);
1981
1982 /* give it to the nic */
2094 while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) {
2095 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
2096 if (m == NULL) {
2097 return;
2098 }
2099 /* let BPF see it */
2100 BPF_MTAP(ifp, m);
2101
2102 /* give it to the nic */
1983 mxge_encap(sc, m);
2103 mxge_encap(ss, m);
1984 }
1985 /* ran out of transmit slots */
1986 if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
1987 sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1988 tx->stall++;
1989 }
1990}
1991
1992static void
1993mxge_start(struct ifnet *ifp)
1994{
1995 mxge_softc_t *sc = ifp->if_softc;
2104 }
2105 /* ran out of transmit slots */
2106 if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
2107 sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2108 tx->stall++;
2109 }
2110}
2111
2112static void
2113mxge_start(struct ifnet *ifp)
2114{
2115 mxge_softc_t *sc = ifp->if_softc;
2116 struct mxge_slice_state *ss;
1996
2117
1997
1998 mtx_lock(&sc->tx_mtx);
1999 mxge_start_locked(sc);
2000 mtx_unlock(&sc->tx_mtx);
2118 /* only use the first slice for now */
2119 ss = &sc->ss[0];
2120 mtx_lock(&ss->tx.mtx);
2121 mxge_start_locked(ss);
2122 mtx_unlock(&ss->tx.mtx);
2001}
2002
2003/*
2004 * copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy
2005 * at most 32 bytes at a time, so as to avoid involving the software
2006 * pio handler in the nic. We re-write the first segment's low
2007 * DMA address to mark it valid only after we write the entire chunk
2008 * in a burst

--- 11 unchanged lines hidden (view full) ---

2020 mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
2021 mb();
2022 src->addr_low = low;
2023 dst->addr_low = low;
2024 mb();
2025}
2026
2027static int
2123}
2124
2125/*
2126 * copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy
2127 * at most 32 bytes at a time, so as to avoid involving the software
2128 * pio handler in the nic. We re-write the first segment's low
2129 * DMA address to mark it valid only after we write the entire chunk
2130 * in a burst

--- 11 unchanged lines hidden (view full) ---

2142 mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
2143 mb();
2144 src->addr_low = low;
2145 dst->addr_low = low;
2146 mb();
2147}
2148
2149static int
2028mxge_get_buf_small(mxge_softc_t *sc, bus_dmamap_t map, int idx)
2150mxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2029{
2030 bus_dma_segment_t seg;
2031 struct mbuf *m;
2151{
2152 bus_dma_segment_t seg;
2153 struct mbuf *m;
2032 mxge_rx_buf_t *rx = &sc->rx_small;
2154 mxge_rx_ring_t *rx = &ss->rx_small;
2033 int cnt, err;
2034
2035 m = m_gethdr(M_DONTWAIT, MT_DATA);
2036 if (m == NULL) {
2037 rx->alloc_fail++;
2038 err = ENOBUFS;
2039 goto done;
2040 }

--- 12 unchanged lines hidden (view full) ---

2053
2054done:
2055 if ((idx & 7) == 7)
2056 mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]);
2057 return err;
2058}
2059
2060static int
2155 int cnt, err;
2156
2157 m = m_gethdr(M_DONTWAIT, MT_DATA);
2158 if (m == NULL) {
2159 rx->alloc_fail++;
2160 err = ENOBUFS;
2161 goto done;
2162 }

--- 12 unchanged lines hidden (view full) ---

2175
2176done:
2177 if ((idx & 7) == 7)
2178 mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]);
2179 return err;
2180}
2181
2182static int
2061mxge_get_buf_big(mxge_softc_t *sc, bus_dmamap_t map, int idx)
2183mxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2062{
2063 bus_dma_segment_t seg[3];
2064 struct mbuf *m;
2184{
2185 bus_dma_segment_t seg[3];
2186 struct mbuf *m;
2065 mxge_rx_buf_t *rx = &sc->rx_big;
2187 mxge_rx_ring_t *rx = &ss->rx_big;
2066 int cnt, err, i;
2067
2068 if (rx->cl_size == MCLBYTES)
2069 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
2070 else
2071 m = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, rx->cl_size);
2072 if (m == NULL) {
2073 rx->alloc_fail++;

--- 101 unchanged lines hidden (view full) ---

2175 */
2176 bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN,
2177 ETHER_HDR_LEN - ETHER_TYPE_LEN);
2178 m_adj(m, ETHER_VLAN_ENCAP_LEN);
2179}
2180
2181
2182static inline void
2188 int cnt, err, i;
2189
2190 if (rx->cl_size == MCLBYTES)
2191 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
2192 else
2193 m = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, rx->cl_size);
2194 if (m == NULL) {
2195 rx->alloc_fail++;

--- 101 unchanged lines hidden (view full) ---

2297 */
2298 bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN,
2299 ETHER_HDR_LEN - ETHER_TYPE_LEN);
2300 m_adj(m, ETHER_VLAN_ENCAP_LEN);
2301}
2302
2303
2304static inline void
2183mxge_rx_done_big(mxge_softc_t *sc, uint32_t len, uint32_t csum)
2305mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, uint32_t csum)
2184{
2306{
2307 mxge_softc_t *sc;
2185 struct ifnet *ifp;
2186 struct mbuf *m;
2187 struct ether_header *eh;
2308 struct ifnet *ifp;
2309 struct mbuf *m;
2310 struct ether_header *eh;
2188 mxge_rx_buf_t *rx;
2311 mxge_rx_ring_t *rx;
2189 bus_dmamap_t old_map;
2190 int idx;
2191 uint16_t tcpudp_csum;
2192
2312 bus_dmamap_t old_map;
2313 int idx;
2314 uint16_t tcpudp_csum;
2315
2316 sc = ss->sc;
2193 ifp = sc->ifp;
2317 ifp = sc->ifp;
2194 rx = &sc->rx_big;
2318 rx = &ss->rx_big;
2195 idx = rx->cnt & rx->mask;
2196 rx->cnt += rx->nbufs;
2197 /* save a pointer to the received mbuf */
2198 m = rx->info[idx].m;
2199 /* try to replace the received mbuf */
2319 idx = rx->cnt & rx->mask;
2320 rx->cnt += rx->nbufs;
2321 /* save a pointer to the received mbuf */
2322 m = rx->info[idx].m;
2323 /* try to replace the received mbuf */
2200 if (mxge_get_buf_big(sc, rx->extra_map, idx)) {
2324 if (mxge_get_buf_big(ss, rx->extra_map, idx)) {
2201 /* drop the frame -- the old mbuf is re-cycled */
2202 ifp->if_ierrors++;
2203 return;
2204 }
2205
2206 /* unmap the received buffer */
2207 old_map = rx->info[idx].map;
2208 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);

--- 4 unchanged lines hidden (view full) ---

2213 rx->extra_map = old_map;
2214
2215 /* mcp implicitly skips 1st 2 bytes so that packet is properly
2216 * aligned */
2217 m->m_data += MXGEFW_PAD;
2218
2219 m->m_pkthdr.rcvif = ifp;
2220 m->m_len = m->m_pkthdr.len = len;
2325 /* drop the frame -- the old mbuf is re-cycled */
2326 ifp->if_ierrors++;
2327 return;
2328 }
2329
2330 /* unmap the received buffer */
2331 old_map = rx->info[idx].map;
2332 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);

--- 4 unchanged lines hidden (view full) ---

2337 rx->extra_map = old_map;
2338
2339 /* mcp implicitly skips 1st 2 bytes so that packet is properly
2340 * aligned */
2341 m->m_data += MXGEFW_PAD;
2342
2343 m->m_pkthdr.rcvif = ifp;
2344 m->m_len = m->m_pkthdr.len = len;
2221 ifp->if_ipackets++;
2345 ss->ipackets++;
2222 eh = mtod(m, struct ether_header *);
2223 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2224 mxge_vlan_tag_remove(m, &csum);
2225 }
2226 /* if the checksum is valid, mark it in the mbuf header */
2227 if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) {
2346 eh = mtod(m, struct ether_header *);
2347 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2348 mxge_vlan_tag_remove(m, &csum);
2349 }
2350 /* if the checksum is valid, mark it in the mbuf header */
2351 if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) {
2228 if (sc->lro_cnt && (0 == mxge_lro_rx(sc, m, csum)))
2352 if (sc->lro_cnt && (0 == mxge_lro_rx(ss, m, csum)))
2229 return;
2230 /* otherwise, it was a UDP frame, or a TCP frame which
2231 we could not do LRO on. Tell the stack that the
2232 checksum is good */
2233 m->m_pkthdr.csum_data = 0xffff;
2234 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID;
2235 }
2236 /* pass the frame up the stack */
2237 (*ifp->if_input)(ifp, m);
2238}
2239
2240static inline void
2353 return;
2354 /* otherwise, it was a UDP frame, or a TCP frame which
2355 we could not do LRO on. Tell the stack that the
2356 checksum is good */
2357 m->m_pkthdr.csum_data = 0xffff;
2358 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID;
2359 }
2360 /* pass the frame up the stack */
2361 (*ifp->if_input)(ifp, m);
2362}
2363
2364static inline void
2241mxge_rx_done_small(mxge_softc_t *sc, uint32_t len, uint32_t csum)
2365mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, uint32_t csum)
2242{
2366{
2367 mxge_softc_t *sc;
2243 struct ifnet *ifp;
2244 struct ether_header *eh;
2245 struct mbuf *m;
2368 struct ifnet *ifp;
2369 struct ether_header *eh;
2370 struct mbuf *m;
2246 mxge_rx_buf_t *rx;
2371 mxge_rx_ring_t *rx;
2247 bus_dmamap_t old_map;
2248 int idx;
2249 uint16_t tcpudp_csum;
2250
2372 bus_dmamap_t old_map;
2373 int idx;
2374 uint16_t tcpudp_csum;
2375
2376 sc = ss->sc;
2251 ifp = sc->ifp;
2377 ifp = sc->ifp;
2252 rx = &sc->rx_small;
2378 rx = &ss->rx_small;
2253 idx = rx->cnt & rx->mask;
2254 rx->cnt++;
2255 /* save a pointer to the received mbuf */
2256 m = rx->info[idx].m;
2257 /* try to replace the received mbuf */
2379 idx = rx->cnt & rx->mask;
2380 rx->cnt++;
2381 /* save a pointer to the received mbuf */
2382 m = rx->info[idx].m;
2383 /* try to replace the received mbuf */
2258 if (mxge_get_buf_small(sc, rx->extra_map, idx)) {
2384 if (mxge_get_buf_small(ss, rx->extra_map, idx)) {
2259 /* drop the frame -- the old mbuf is re-cycled */
2260 ifp->if_ierrors++;
2261 return;
2262 }
2263
2264 /* unmap the received buffer */
2265 old_map = rx->info[idx].map;
2266 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);

--- 4 unchanged lines hidden (view full) ---

2271 rx->extra_map = old_map;
2272
2273 /* mcp implicitly skips 1st 2 bytes so that packet is properly
2274 * aligned */
2275 m->m_data += MXGEFW_PAD;
2276
2277 m->m_pkthdr.rcvif = ifp;
2278 m->m_len = m->m_pkthdr.len = len;
2385 /* drop the frame -- the old mbuf is re-cycled */
2386 ifp->if_ierrors++;
2387 return;
2388 }
2389
2390 /* unmap the received buffer */
2391 old_map = rx->info[idx].map;
2392 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);

--- 4 unchanged lines hidden (view full) ---

2397 rx->extra_map = old_map;
2398
2399 /* mcp implicitly skips 1st 2 bytes so that packet is properly
2400 * aligned */
2401 m->m_data += MXGEFW_PAD;
2402
2403 m->m_pkthdr.rcvif = ifp;
2404 m->m_len = m->m_pkthdr.len = len;
2279 ifp->if_ipackets++;
2405 ss->ipackets++;
2280 eh = mtod(m, struct ether_header *);
2281 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2282 mxge_vlan_tag_remove(m, &csum);
2283 }
2284 /* if the checksum is valid, mark it in the mbuf header */
2285 if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) {
2406 eh = mtod(m, struct ether_header *);
2407 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2408 mxge_vlan_tag_remove(m, &csum);
2409 }
2410 /* if the checksum is valid, mark it in the mbuf header */
2411 if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) {
2286 if (sc->lro_cnt && (0 == mxge_lro_rx(sc, m, csum)))
2412 if (sc->lro_cnt && (0 == mxge_lro_rx(ss, m, csum)))
2287 return;
2288 /* otherwise, it was a UDP frame, or a TCP frame which
2289 we could not do LRO on. Tell the stack that the
2290 checksum is good */
2291 m->m_pkthdr.csum_data = 0xffff;
2292 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID;
2293 }
2413 return;
2414 /* otherwise, it was a UDP frame, or a TCP frame which
2415 we could not do LRO on. Tell the stack that the
2416 checksum is good */
2417 m->m_pkthdr.csum_data = 0xffff;
2418 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID;
2419 }
2294
2295 /* pass the frame up the stack */
2296 (*ifp->if_input)(ifp, m);
2297}
2298
2299static inline void
2420 /* pass the frame up the stack */
2421 (*ifp->if_input)(ifp, m);
2422}
2423
2424static inline void
2300mxge_clean_rx_done(mxge_softc_t *sc)
2425mxge_clean_rx_done(struct mxge_slice_state *ss)
2301{
2426{
2302 mxge_rx_done_t *rx_done = &sc->rx_done;
2427 mxge_rx_done_t *rx_done = &ss->rx_done;
2303 struct lro_entry *lro;
2304 int limit = 0;
2305 uint16_t length;
2306 uint16_t checksum;
2307
2308
2309 while (rx_done->entry[rx_done->idx].length != 0) {
2310 length = ntohs(rx_done->entry[rx_done->idx].length);
2311 rx_done->entry[rx_done->idx].length = 0;
2312 checksum = rx_done->entry[rx_done->idx].checksum;
2313 if (length <= (MHLEN - MXGEFW_PAD))
2428 struct lro_entry *lro;
2429 int limit = 0;
2430 uint16_t length;
2431 uint16_t checksum;
2432
2433
2434 while (rx_done->entry[rx_done->idx].length != 0) {
2435 length = ntohs(rx_done->entry[rx_done->idx].length);
2436 rx_done->entry[rx_done->idx].length = 0;
2437 checksum = rx_done->entry[rx_done->idx].checksum;
2438 if (length <= (MHLEN - MXGEFW_PAD))
2314 mxge_rx_done_small(sc, length, checksum);
2439 mxge_rx_done_small(ss, length, checksum);
2315 else
2440 else
2316 mxge_rx_done_big(sc, length, checksum);
2441 mxge_rx_done_big(ss, length, checksum);
2317 rx_done->cnt++;
2318 rx_done->idx = rx_done->cnt & rx_done->mask;
2319
2320 /* limit potential for livelock */
2321 if (__predict_false(++limit > rx_done->mask / 2))
2322 break;
2323 }
2442 rx_done->cnt++;
2443 rx_done->idx = rx_done->cnt & rx_done->mask;
2444
2445 /* limit potential for livelock */
2446 if (__predict_false(++limit > rx_done->mask / 2))
2447 break;
2448 }
2324 while(!SLIST_EMPTY(&sc->lro_active)) {
2325 lro = SLIST_FIRST(&sc->lro_active);
2326 SLIST_REMOVE_HEAD(&sc->lro_active, next);
2327 mxge_lro_flush(sc, lro);
2449 while (!SLIST_EMPTY(&ss->lro_active)) {
2450 lro = SLIST_FIRST(&ss->lro_active);
2451 SLIST_REMOVE_HEAD(&ss->lro_active, next);
2452 mxge_lro_flush(ss, lro);
2328 }
2329}
2330
2331
2332static inline void
2453 }
2454}
2455
2456
2457static inline void
2333mxge_tx_done(mxge_softc_t *sc, uint32_t mcp_idx)
2458mxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx)
2334{
2335 struct ifnet *ifp;
2459{
2460 struct ifnet *ifp;
2336 mxge_tx_buf_t *tx;
2461 mxge_tx_ring_t *tx;
2337 struct mbuf *m;
2338 bus_dmamap_t map;
2339 int idx;
2340
2462 struct mbuf *m;
2463 bus_dmamap_t map;
2464 int idx;
2465
2341 tx = &sc->tx;
2342 ifp = sc->ifp;
2466 tx = &ss->tx;
2467 ifp = ss->sc->ifp;
2343 while (tx->pkt_done != mcp_idx) {
2344 idx = tx->done & tx->mask;
2345 tx->done++;
2346 m = tx->info[idx].m;
2347 /* mbuf and DMA map only attached to the first
2348 segment per-mbuf */
2349 if (m != NULL) {
2350 ifp->if_opackets++;

--- 8 unchanged lines hidden (view full) ---

2359 }
2360 }
2361
2362 /* If we have space, clear IFF_OACTIVE to tell the stack that
2363 its OK to send packets */
2364
2365 if (ifp->if_drv_flags & IFF_DRV_OACTIVE &&
2366 tx->req - tx->done < (tx->mask + 1)/4) {
2468 while (tx->pkt_done != mcp_idx) {
2469 idx = tx->done & tx->mask;
2470 tx->done++;
2471 m = tx->info[idx].m;
2472 /* mbuf and DMA map only attached to the first
2473 segment per-mbuf */
2474 if (m != NULL) {
2475 ifp->if_opackets++;

--- 8 unchanged lines hidden (view full) ---

2484 }
2485 }
2486
2487 /* If we have space, clear IFF_OACTIVE to tell the stack that
2488 its OK to send packets */
2489
2490 if (ifp->if_drv_flags & IFF_DRV_OACTIVE &&
2491 tx->req - tx->done < (tx->mask + 1)/4) {
2367 mtx_lock(&sc->tx_mtx);
2492 mtx_lock(&ss->tx.mtx);
2368 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2493 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2369 sc->tx.wake++;
2370 mxge_start_locked(sc);
2371 mtx_unlock(&sc->tx_mtx);
2494 ss->tx.wake++;
2495 mxge_start_locked(ss);
2496 mtx_unlock(&ss->tx.mtx);
2372 }
2373}
2374
2375static struct mxge_media_type mxge_media_types[] =
2376{
2377 {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"},
2378 {IFM_10G_SR, (1 << 7), "10GBASE-SR"},
2379 {IFM_10G_LR, (1 << 6), "10GBASE-LR"},

--- 126 unchanged lines hidden (view full) ---

2506 device_printf(sc->dev, "XFP media 0x%x unknown\n", cmd.data0);
2507
2508 return;
2509}
2510
2511static void
2512mxge_intr(void *arg)
2513{
2497 }
2498}
2499
2500static struct mxge_media_type mxge_media_types[] =
2501{
2502 {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"},
2503 {IFM_10G_SR, (1 << 7), "10GBASE-SR"},
2504 {IFM_10G_LR, (1 << 6), "10GBASE-LR"},

--- 126 unchanged lines hidden (view full) ---

2631 device_printf(sc->dev, "XFP media 0x%x unknown\n", cmd.data0);
2632
2633 return;
2634}
2635
2636static void
2637mxge_intr(void *arg)
2638{
2514 mxge_softc_t *sc = arg;
2515 mcp_irq_data_t *stats = sc->fw_stats;
2516 mxge_tx_buf_t *tx = &sc->tx;
2517 mxge_rx_done_t *rx_done = &sc->rx_done;
2639 struct mxge_slice_state *ss = arg;
2640 mxge_softc_t *sc = ss->sc;
2641 mcp_irq_data_t *stats = ss->fw_stats;
2642 mxge_tx_ring_t *tx = &ss->tx;
2643 mxge_rx_done_t *rx_done = &ss->rx_done;
2518 uint32_t send_done_count;
2519 uint8_t valid;
2520
2521
2644 uint32_t send_done_count;
2645 uint8_t valid;
2646
2647
2648 /* an interrupt on a non-zero slice is implicitly valid
2649 since MSI-X irqs are not shared */
2650 if (ss != sc->ss) {
2651 mxge_clean_rx_done(ss);
2652 *ss->irq_claim = be32toh(3);
2653 return;
2654 }
2655
2522 /* make sure the DMA has finished */
2523 if (!stats->valid) {
2524 return;
2525 }
2526 valid = stats->valid;
2527
2528 if (!sc->msi_enabled) {
2529 /* lower legacy IRQ */

--- 6 unchanged lines hidden (view full) ---

2536 }
2537
2538 /* loop while waiting for legacy irq deassertion */
2539 do {
2540 /* check for transmit completes and receives */
2541 send_done_count = be32toh(stats->send_done_count);
2542 while ((send_done_count != tx->pkt_done) ||
2543 (rx_done->entry[rx_done->idx].length != 0)) {
2656 /* make sure the DMA has finished */
2657 if (!stats->valid) {
2658 return;
2659 }
2660 valid = stats->valid;
2661
2662 if (!sc->msi_enabled) {
2663 /* lower legacy IRQ */

--- 6 unchanged lines hidden (view full) ---

2670 }
2671
2672 /* loop while waiting for legacy irq deassertion */
2673 do {
2674 /* check for transmit completes and receives */
2675 send_done_count = be32toh(stats->send_done_count);
2676 while ((send_done_count != tx->pkt_done) ||
2677 (rx_done->entry[rx_done->idx].length != 0)) {
2544 mxge_tx_done(sc, (int)send_done_count);
2545 mxge_clean_rx_done(sc);
2678 mxge_tx_done(ss, (int)send_done_count);
2679 mxge_clean_rx_done(ss);
2546 send_done_count = be32toh(stats->send_done_count);
2547 }
2548 } while (*((volatile uint8_t *) &stats->valid));
2549
2550 if (__predict_false(stats->stats_updated)) {
2551 if (sc->link_state != stats->link_up) {
2552 sc->link_state = stats->link_up;
2553 if (sc->link_state) {
2554 if_link_state_change(sc->ifp, LINK_STATE_UP);
2555 if (mxge_verbose)
2556 device_printf(sc->dev, "link up\n");
2557 } else {
2558 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2559 if (mxge_verbose)
2560 device_printf(sc->dev, "link down\n");
2561 }
2562 sc->need_media_probe = 1;
2563 }
2564 if (sc->rdma_tags_available !=
2680 send_done_count = be32toh(stats->send_done_count);
2681 }
2682 } while (*((volatile uint8_t *) &stats->valid));
2683
2684 if (__predict_false(stats->stats_updated)) {
2685 if (sc->link_state != stats->link_up) {
2686 sc->link_state = stats->link_up;
2687 if (sc->link_state) {
2688 if_link_state_change(sc->ifp, LINK_STATE_UP);
2689 if (mxge_verbose)
2690 device_printf(sc->dev, "link up\n");
2691 } else {
2692 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2693 if (mxge_verbose)
2694 device_printf(sc->dev, "link down\n");
2695 }
2696 sc->need_media_probe = 1;
2697 }
2698 if (sc->rdma_tags_available !=
2565 be32toh(sc->fw_stats->rdma_tags_available)) {
2699 be32toh(stats->rdma_tags_available)) {
2566 sc->rdma_tags_available =
2700 sc->rdma_tags_available =
2567 be32toh(sc->fw_stats->rdma_tags_available);
2701 be32toh(stats->rdma_tags_available);
2568 device_printf(sc->dev, "RDMA timed out! %d tags "
2569 "left\n", sc->rdma_tags_available);
2570 }
2571
2572 if (stats->link_down) {
2573 sc->down_cnt += stats->link_down;
2574 sc->link_state = 0;
2575 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2576 }
2577 }
2578
2579 /* check to see if we have rx token to pass back */
2580 if (valid & 0x1)
2702 device_printf(sc->dev, "RDMA timed out! %d tags "
2703 "left\n", sc->rdma_tags_available);
2704 }
2705
2706 if (stats->link_down) {
2707 sc->down_cnt += stats->link_down;
2708 sc->link_state = 0;
2709 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2710 }
2711 }
2712
2713 /* check to see if we have rx token to pass back */
2714 if (valid & 0x1)
2581 *sc->irq_claim = be32toh(3);
2582 *(sc->irq_claim + 1) = be32toh(3);
2715 *ss->irq_claim = be32toh(3);
2716 *(ss->irq_claim + 1) = be32toh(3);
2583}
2584
2585static void
2586mxge_init(void *arg)
2587{
2588}
2589
2590
2591
2592static void
2717}
2718
2719static void
2720mxge_init(void *arg)
2721{
2722}
2723
2724
2725
2726static void
2593mxge_free_mbufs(mxge_softc_t *sc)
2727mxge_free_slice_mbufs(struct mxge_slice_state *ss)
2594{
2728{
2729 struct lro_entry *lro_entry;
2595 int i;
2596
2730 int i;
2731
2597 for (i = 0; i <= sc->rx_big.mask; i++) {
2598 if (sc->rx_big.info[i].m == NULL)
2732 while (!SLIST_EMPTY(&ss->lro_free)) {
2733 lro_entry = SLIST_FIRST(&ss->lro_free);
2734 SLIST_REMOVE_HEAD(&ss->lro_free, next);
2735 free(lro_entry, M_DEVBUF);
2736 }
2737
2738 for (i = 0; i <= ss->rx_big.mask; i++) {
2739 if (ss->rx_big.info[i].m == NULL)
2599 continue;
2740 continue;
2600 bus_dmamap_unload(sc->rx_big.dmat,
2601 sc->rx_big.info[i].map);
2602 m_freem(sc->rx_big.info[i].m);
2603 sc->rx_big.info[i].m = NULL;
2741 bus_dmamap_unload(ss->rx_big.dmat,
2742 ss->rx_big.info[i].map);
2743 m_freem(ss->rx_big.info[i].m);
2744 ss->rx_big.info[i].m = NULL;
2604 }
2605
2745 }
2746
2606 for (i = 0; i <= sc->rx_small.mask; i++) {
2607 if (sc->rx_small.info[i].m == NULL)
2747 for (i = 0; i <= ss->rx_small.mask; i++) {
2748 if (ss->rx_small.info[i].m == NULL)
2608 continue;
2749 continue;
2609 bus_dmamap_unload(sc->rx_small.dmat,
2610 sc->rx_small.info[i].map);
2611 m_freem(sc->rx_small.info[i].m);
2612 sc->rx_small.info[i].m = NULL;
2750 bus_dmamap_unload(ss->rx_small.dmat,
2751 ss->rx_small.info[i].map);
2752 m_freem(ss->rx_small.info[i].m);
2753 ss->rx_small.info[i].m = NULL;
2613 }
2614
2754 }
2755
2615 for (i = 0; i <= sc->tx.mask; i++) {
2616 sc->tx.info[i].flag = 0;
2617 if (sc->tx.info[i].m == NULL)
2756 /* transmit ring used only on the first slice */
2757 if (ss->tx.info == NULL)
2758 return;
2759
2760 for (i = 0; i <= ss->tx.mask; i++) {
2761 ss->tx.info[i].flag = 0;
2762 if (ss->tx.info[i].m == NULL)
2618 continue;
2763 continue;
2619 bus_dmamap_unload(sc->tx.dmat,
2620 sc->tx.info[i].map);
2621 m_freem(sc->tx.info[i].m);
2622 sc->tx.info[i].m = NULL;
2764 bus_dmamap_unload(ss->tx.dmat,
2765 ss->tx.info[i].map);
2766 m_freem(ss->tx.info[i].m);
2767 ss->tx.info[i].m = NULL;
2623 }
2624}
2625
2626static void
2768 }
2769}
2770
2771static void
2627mxge_free_rings(mxge_softc_t *sc)
2772mxge_free_mbufs(mxge_softc_t *sc)
2628{
2773{
2774 int slice;
2775
2776 for (slice = 0; slice < sc->num_slices; slice++)
2777 mxge_free_slice_mbufs(&sc->ss[slice]);
2778}
2779
2780static void
2781mxge_free_slice_rings(struct mxge_slice_state *ss)
2782{
2629 int i;
2630
2783 int i;
2784
2631 if (sc->rx_done.entry != NULL)
2632 mxge_dma_free(&sc->rx_done.dma);
2633 sc->rx_done.entry = NULL;
2634 if (sc->tx.req_bytes != NULL)
2635 free(sc->tx.req_bytes, M_DEVBUF);
2636 if (sc->tx.seg_list != NULL)
2637 free(sc->tx.seg_list, M_DEVBUF);
2638 if (sc->rx_small.shadow != NULL)
2639 free(sc->rx_small.shadow, M_DEVBUF);
2640 if (sc->rx_big.shadow != NULL)
2641 free(sc->rx_big.shadow, M_DEVBUF);
2642 if (sc->tx.info != NULL) {
2643 if (sc->tx.dmat != NULL) {
2644 for (i = 0; i <= sc->tx.mask; i++) {
2645 bus_dmamap_destroy(sc->tx.dmat,
2646 sc->tx.info[i].map);
2785
2786 if (ss->rx_done.entry != NULL)
2787 mxge_dma_free(&ss->rx_done.dma);
2788 ss->rx_done.entry = NULL;
2789
2790 if (ss->tx.req_bytes != NULL)
2791 free(ss->tx.req_bytes, M_DEVBUF);
2792 ss->tx.req_bytes = NULL;
2793
2794 if (ss->tx.seg_list != NULL)
2795 free(ss->tx.seg_list, M_DEVBUF);
2796 ss->tx.seg_list = NULL;
2797
2798 if (ss->rx_small.shadow != NULL)
2799 free(ss->rx_small.shadow, M_DEVBUF);
2800 ss->rx_small.shadow = NULL;
2801
2802 if (ss->rx_big.shadow != NULL)
2803 free(ss->rx_big.shadow, M_DEVBUF);
2804 ss->rx_big.shadow = NULL;
2805
2806 if (ss->tx.info != NULL) {
2807 if (ss->tx.dmat != NULL) {
2808 for (i = 0; i <= ss->tx.mask; i++) {
2809 bus_dmamap_destroy(ss->tx.dmat,
2810 ss->tx.info[i].map);
2647 }
2811 }
2648 bus_dma_tag_destroy(sc->tx.dmat);
2812 bus_dma_tag_destroy(ss->tx.dmat);
2649 }
2813 }
2650 free(sc->tx.info, M_DEVBUF);
2814 free(ss->tx.info, M_DEVBUF);
2651 }
2815 }
2652 if (sc->rx_small.info != NULL) {
2653 if (sc->rx_small.dmat != NULL) {
2654 for (i = 0; i <= sc->rx_small.mask; i++) {
2655 bus_dmamap_destroy(sc->rx_small.dmat,
2656 sc->rx_small.info[i].map);
2816 ss->tx.info = NULL;
2817
2818 if (ss->rx_small.info != NULL) {
2819 if (ss->rx_small.dmat != NULL) {
2820 for (i = 0; i <= ss->rx_small.mask; i++) {
2821 bus_dmamap_destroy(ss->rx_small.dmat,
2822 ss->rx_small.info[i].map);
2657 }
2823 }
2658 bus_dmamap_destroy(sc->rx_small.dmat,
2659 sc->rx_small.extra_map);
2660 bus_dma_tag_destroy(sc->rx_small.dmat);
2824 bus_dmamap_destroy(ss->rx_small.dmat,
2825 ss->rx_small.extra_map);
2826 bus_dma_tag_destroy(ss->rx_small.dmat);
2661 }
2827 }
2662 free(sc->rx_small.info, M_DEVBUF);
2828 free(ss->rx_small.info, M_DEVBUF);
2663 }
2829 }
2664 if (sc->rx_big.info != NULL) {
2665 if (sc->rx_big.dmat != NULL) {
2666 for (i = 0; i <= sc->rx_big.mask; i++) {
2667 bus_dmamap_destroy(sc->rx_big.dmat,
2668 sc->rx_big.info[i].map);
2830 ss->rx_small.info = NULL;
2831
2832 if (ss->rx_big.info != NULL) {
2833 if (ss->rx_big.dmat != NULL) {
2834 for (i = 0; i <= ss->rx_big.mask; i++) {
2835 bus_dmamap_destroy(ss->rx_big.dmat,
2836 ss->rx_big.info[i].map);
2669 }
2837 }
2670 bus_dmamap_destroy(sc->rx_big.dmat,
2671 sc->rx_big.extra_map);
2672 bus_dma_tag_destroy(sc->rx_big.dmat);
2838 bus_dmamap_destroy(ss->rx_big.dmat,
2839 ss->rx_big.extra_map);
2840 bus_dma_tag_destroy(ss->rx_big.dmat);
2673 }
2841 }
2674 free(sc->rx_big.info, M_DEVBUF);
2842 free(ss->rx_big.info, M_DEVBUF);
2675 }
2843 }
2844 ss->rx_big.info = NULL;
2676}
2677
2845}
2846
2678static int
2679mxge_alloc_rings(mxge_softc_t *sc)
2847static void
2848mxge_free_rings(mxge_softc_t *sc)
2680{
2849{
2681 mxge_cmd_t cmd;
2682 int tx_ring_size, rx_ring_size;
2683 int tx_ring_entries, rx_ring_entries;
2684 int i, err;
2685 unsigned long bytes;
2686
2687 /* get ring sizes */
2688 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
2689 tx_ring_size = cmd.data0;
2690 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
2691 if (err != 0) {
2692 device_printf(sc->dev, "Cannot determine ring sizes\n");
2693 goto abort_with_nothing;
2694 }
2850 int slice;
2695
2851
2696 rx_ring_size = cmd.data0;
2852 for (slice = 0; slice < sc->num_slices; slice++)
2853 mxge_free_slice_rings(&sc->ss[slice]);
2854}
2697
2855
2698 tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t);
2699 rx_ring_entries = rx_ring_size / sizeof (mcp_dma_addr_t);
2700 IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1);
2701 sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen;
2702 IFQ_SET_READY(&sc->ifp->if_snd);
2856static int
2857mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries,
2858 int tx_ring_entries)
2859{
2860 mxge_softc_t *sc = ss->sc;
2861 size_t bytes;
2862 int err, i;
2703
2863
2704 sc->tx.mask = tx_ring_entries - 1;
2705 sc->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4);
2706 sc->rx_small.mask = sc->rx_big.mask = rx_ring_entries - 1;
2707 sc->rx_done.mask = (2 * rx_ring_entries) - 1;
2708
2709 err = ENOMEM;
2710
2864 err = ENOMEM;
2865
2711 /* allocate interrupt queues */
2712 bytes = (sc->rx_done.mask + 1) * sizeof (*sc->rx_done.entry);
2713 err = mxge_dma_alloc(sc, &sc->rx_done.dma, bytes, 4096);
2714 if (err != 0)
2715 goto abort_with_nothing;
2716 sc->rx_done.entry = sc->rx_done.dma.addr;
2717 bzero(sc->rx_done.entry, bytes);
2866 /* allocate per-slice receive resources */
2718
2867
2719 /* allocate the tx request copy block */
2720 bytes = 8 +
2721 sizeof (*sc->tx.req_list) * (sc->tx.max_desc + 4);
2722 sc->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK);
2723 if (sc->tx.req_bytes == NULL)
2724 goto abort_with_alloc;
2725 /* ensure req_list entries are aligned to 8 bytes */
2726 sc->tx.req_list = (mcp_kreq_ether_send_t *)
2727 ((unsigned long)(sc->tx.req_bytes + 7) & ~7UL);
2868 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
2869 ss->rx_done.mask = (2 * rx_ring_entries) - 1;
2728
2870
2729 /* allocate the tx busdma segment list */
2730 bytes = sizeof (*sc->tx.seg_list) * sc->tx.max_desc;
2731 sc->tx.seg_list = (bus_dma_segment_t *)
2732 malloc(bytes, M_DEVBUF, M_WAITOK);
2733 if (sc->tx.seg_list == NULL)
2734 goto abort_with_alloc;
2735
2736 /* allocate the rx shadow rings */
2871 /* allocate the rx shadow rings */
2737 bytes = rx_ring_entries * sizeof (*sc->rx_small.shadow);
2738 sc->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2739 if (sc->rx_small.shadow == NULL)
2740 goto abort_with_alloc;
2872 bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
2873 ss->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2874 if (ss->rx_small.shadow == NULL)
2875 return err;;
2741
2876
2742 bytes = rx_ring_entries * sizeof (*sc->rx_big.shadow);
2743 sc->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2744 if (sc->rx_big.shadow == NULL)
2745 goto abort_with_alloc;
2877 bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
2878 ss->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2879 if (ss->rx_big.shadow == NULL)
2880 return err;;
2746
2881
2747 /* allocate the host info rings */
2748 bytes = tx_ring_entries * sizeof (*sc->tx.info);
2749 sc->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2750 if (sc->tx.info == NULL)
2751 goto abort_with_alloc;
2752
2753 bytes = rx_ring_entries * sizeof (*sc->rx_small.info);
2754 sc->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2755 if (sc->rx_small.info == NULL)
2756 goto abort_with_alloc;
2882 /* allocate the rx host info rings */
2883 bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
2884 ss->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2885 if (ss->rx_small.info == NULL)
2886 return err;;
2757
2887
2758 bytes = rx_ring_entries * sizeof (*sc->rx_big.info);
2759 sc->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2760 if (sc->rx_big.info == NULL)
2761 goto abort_with_alloc;
2888 bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
2889 ss->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2890 if (ss->rx_big.info == NULL)
2891 return err;;
2762
2892
2763 /* allocate the busdma resources */
2893 /* allocate the rx busdma resources */
2764 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
2765 1, /* alignment */
2894 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
2895 1, /* alignment */
2766 sc->tx.boundary, /* boundary */
2767 BUS_SPACE_MAXADDR, /* low */
2768 BUS_SPACE_MAXADDR, /* high */
2769 NULL, NULL, /* filter */
2770 65536 + 256, /* maxsize */
2771 sc->tx.max_desc - 2, /* num segs */
2772 sc->tx.boundary, /* maxsegsize */
2773 BUS_DMA_ALLOCNOW, /* flags */
2774 NULL, NULL, /* lock */
2775 &sc->tx.dmat); /* tag */
2776
2777 if (err != 0) {
2778 device_printf(sc->dev, "Err %d allocating tx dmat\n",
2779 err);
2780 goto abort_with_alloc;
2781 }
2782
2783 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
2784 1, /* alignment */
2785 4096, /* boundary */
2786 BUS_SPACE_MAXADDR, /* low */
2787 BUS_SPACE_MAXADDR, /* high */
2788 NULL, NULL, /* filter */
2789 MHLEN, /* maxsize */
2790 1, /* num segs */
2791 MHLEN, /* maxsegsize */
2792 BUS_DMA_ALLOCNOW, /* flags */
2793 NULL, NULL, /* lock */
2896 4096, /* boundary */
2897 BUS_SPACE_MAXADDR, /* low */
2898 BUS_SPACE_MAXADDR, /* high */
2899 NULL, NULL, /* filter */
2900 MHLEN, /* maxsize */
2901 1, /* num segs */
2902 MHLEN, /* maxsegsize */
2903 BUS_DMA_ALLOCNOW, /* flags */
2904 NULL, NULL, /* lock */
2794 &sc->rx_small.dmat); /* tag */
2905 &ss->rx_small.dmat); /* tag */
2795 if (err != 0) {
2796 device_printf(sc->dev, "Err %d allocating rx_small dmat\n",
2797 err);
2906 if (err != 0) {
2907 device_printf(sc->dev, "Err %d allocating rx_small dmat\n",
2908 err);
2798 goto abort_with_alloc;
2909 return err;;
2799 }
2800
2801 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
2802 1, /* alignment */
2803 4096, /* boundary */
2804 BUS_SPACE_MAXADDR, /* low */
2805 BUS_SPACE_MAXADDR, /* high */
2806 NULL, NULL, /* filter */
2807 3*4096, /* maxsize */
2808 3, /* num segs */
2809 4096, /* maxsegsize */
2810 BUS_DMA_ALLOCNOW, /* flags */
2811 NULL, NULL, /* lock */
2910 }
2911
2912 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
2913 1, /* alignment */
2914 4096, /* boundary */
2915 BUS_SPACE_MAXADDR, /* low */
2916 BUS_SPACE_MAXADDR, /* high */
2917 NULL, NULL, /* filter */
2918 3*4096, /* maxsize */
2919 3, /* num segs */
2920 4096, /* maxsegsize */
2921 BUS_DMA_ALLOCNOW, /* flags */
2922 NULL, NULL, /* lock */
2812 &sc->rx_big.dmat); /* tag */
2923 &ss->rx_big.dmat); /* tag */
2813 if (err != 0) {
2814 device_printf(sc->dev, "Err %d allocating rx_big dmat\n",
2815 err);
2924 if (err != 0) {
2925 device_printf(sc->dev, "Err %d allocating rx_big dmat\n",
2926 err);
2816 goto abort_with_alloc;
2927 return err;;
2817 }
2928 }
2818
2819 /* now use these tags to setup dmamaps for each slot
2820 in each ring */
2821 for (i = 0; i <= sc->tx.mask; i++) {
2822 err = bus_dmamap_create(sc->tx.dmat, 0,
2823 &sc->tx.info[i].map);
2929 for (i = 0; i <= ss->rx_small.mask; i++) {
2930 err = bus_dmamap_create(ss->rx_small.dmat, 0,
2931 &ss->rx_small.info[i].map);
2824 if (err != 0) {
2932 if (err != 0) {
2825 device_printf(sc->dev, "Err %d tx dmamap\n",
2826 err);
2827 goto abort_with_alloc;
2828 }
2829 }
2830 for (i = 0; i <= sc->rx_small.mask; i++) {
2831 err = bus_dmamap_create(sc->rx_small.dmat, 0,
2832 &sc->rx_small.info[i].map);
2833 if (err != 0) {
2834 device_printf(sc->dev, "Err %d rx_small dmamap\n",
2835 err);
2933 device_printf(sc->dev, "Err %d rx_small dmamap\n",
2934 err);
2836 goto abort_with_alloc;
2935 return err;;
2837 }
2838 }
2936 }
2937 }
2839 err = bus_dmamap_create(sc->rx_small.dmat, 0,
2840 &sc->rx_small.extra_map);
2938 err = bus_dmamap_create(ss->rx_small.dmat, 0,
2939 &ss->rx_small.extra_map);
2841 if (err != 0) {
2842 device_printf(sc->dev, "Err %d extra rx_small dmamap\n",
2843 err);
2940 if (err != 0) {
2941 device_printf(sc->dev, "Err %d extra rx_small dmamap\n",
2942 err);
2844 goto abort_with_alloc;
2943 return err;;
2845 }
2846
2944 }
2945
2847 for (i = 0; i <= sc->rx_big.mask; i++) {
2848 err = bus_dmamap_create(sc->rx_big.dmat, 0,
2849 &sc->rx_big.info[i].map);
2946 for (i = 0; i <= ss->rx_big.mask; i++) {
2947 err = bus_dmamap_create(ss->rx_big.dmat, 0,
2948 &ss->rx_big.info[i].map);
2850 if (err != 0) {
2851 device_printf(sc->dev, "Err %d rx_big dmamap\n",
2949 if (err != 0) {
2950 device_printf(sc->dev, "Err %d rx_big dmamap\n",
2852 err);
2853 goto abort_with_alloc;
2951 err);
2952 return err;;
2854 }
2855 }
2953 }
2954 }
2856 err = bus_dmamap_create(sc->rx_big.dmat, 0,
2857 &sc->rx_big.extra_map);
2955 err = bus_dmamap_create(ss->rx_big.dmat, 0,
2956 &ss->rx_big.extra_map);
2858 if (err != 0) {
2859 device_printf(sc->dev, "Err %d extra rx_big dmamap\n",
2860 err);
2957 if (err != 0) {
2958 device_printf(sc->dev, "Err %d extra rx_big dmamap\n",
2959 err);
2861 goto abort_with_alloc;
2960 return err;;
2862 }
2961 }
2962
2963 /* now allocate TX resouces */
2964
2965 /* only use a single TX ring for now */
2966 if (ss != ss->sc->ss)
2967 return 0;
2968
2969 ss->tx.mask = tx_ring_entries - 1;
2970 ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4);
2971
2972
2973 /* allocate the tx request copy block */
2974 bytes = 8 +
2975 sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4);
2976 ss->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK);
2977 if (ss->tx.req_bytes == NULL)
2978 return err;;
2979 /* ensure req_list entries are aligned to 8 bytes */
2980 ss->tx.req_list = (mcp_kreq_ether_send_t *)
2981 ((unsigned long)(ss->tx.req_bytes + 7) & ~7UL);
2982
2983 /* allocate the tx busdma segment list */
2984 bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc;
2985 ss->tx.seg_list = (bus_dma_segment_t *)
2986 malloc(bytes, M_DEVBUF, M_WAITOK);
2987 if (ss->tx.seg_list == NULL)
2988 return err;;
2989
2990 /* allocate the tx host info ring */
2991 bytes = tx_ring_entries * sizeof (*ss->tx.info);
2992 ss->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2993 if (ss->tx.info == NULL)
2994 return err;;
2995
2996 /* allocate the tx busdma resources */
2997 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
2998 1, /* alignment */
2999 sc->tx_boundary, /* boundary */
3000 BUS_SPACE_MAXADDR, /* low */
3001 BUS_SPACE_MAXADDR, /* high */
3002 NULL, NULL, /* filter */
3003 65536 + 256, /* maxsize */
3004 ss->tx.max_desc - 2, /* num segs */
3005 sc->tx_boundary, /* maxsegsz */
3006 BUS_DMA_ALLOCNOW, /* flags */
3007 NULL, NULL, /* lock */
3008 &ss->tx.dmat); /* tag */
3009
3010 if (err != 0) {
3011 device_printf(sc->dev, "Err %d allocating tx dmat\n",
3012 err);
3013 return err;;
3014 }
3015
3016 /* now use these tags to setup dmamaps for each slot
3017 in the ring */
3018 for (i = 0; i <= ss->tx.mask; i++) {
3019 err = bus_dmamap_create(ss->tx.dmat, 0,
3020 &ss->tx.info[i].map);
3021 if (err != 0) {
3022 device_printf(sc->dev, "Err %d tx dmamap\n",
3023 err);
3024 return err;;
3025 }
3026 }
2863 return 0;
2864
3027 return 0;
3028
2865abort_with_alloc:
2866 mxge_free_rings(sc);
3029}
2867
3030
2868abort_with_nothing:
3031static int
3032mxge_alloc_rings(mxge_softc_t *sc)
3033{
3034 mxge_cmd_t cmd;
3035 int tx_ring_size;
3036 int tx_ring_entries, rx_ring_entries;
3037 int err, slice;
3038
3039 /* get ring sizes */
3040 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
3041 tx_ring_size = cmd.data0;
3042 if (err != 0) {
3043 device_printf(sc->dev, "Cannot determine tx ring sizes\n");
3044 goto abort;
3045 }
3046
3047 tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t);
3048 rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t);
3049 IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1);
3050 sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen;
3051 IFQ_SET_READY(&sc->ifp->if_snd);
3052
3053 for (slice = 0; slice < sc->num_slices; slice++) {
3054 err = mxge_alloc_slice_rings(&sc->ss[slice],
3055 rx_ring_entries,
3056 tx_ring_entries);
3057 if (err != 0)
3058 goto abort;
3059 }
3060 return 0;
3061
3062abort:
3063 mxge_free_rings(sc);
2869 return err;
3064 return err;
3065
2870}
2871
3066}
3067
3068
2872static void
2873mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs)
2874{
2875 int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD;
2876
2877 if (bufsize < MCLBYTES) {
2878 /* easy, everything fits in a single buffer */
2879 *big_buf_size = MCLBYTES;

--- 13 unchanged lines hidden (view full) ---

2893 *cl_size = MJUM9BYTES;
2894 *big_buf_size = 4096;
2895 *nbufs = mtu / 4096 + 1;
2896 /* needs to be a power of two, so round up */
2897 if (*nbufs == 3)
2898 *nbufs = 4;
2899}
2900
3069static void
3070mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs)
3071{
3072 int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD;
3073
3074 if (bufsize < MCLBYTES) {
3075 /* easy, everything fits in a single buffer */
3076 *big_buf_size = MCLBYTES;

--- 13 unchanged lines hidden (view full) ---

3090 *cl_size = MJUM9BYTES;
3091 *big_buf_size = 4096;
3092 *nbufs = mtu / 4096 + 1;
3093 /* needs to be a power of two, so round up */
3094 if (*nbufs == 3)
3095 *nbufs = 4;
3096}
3097
2901static int
2902mxge_open(mxge_softc_t *sc)
3098static int
3099mxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size)
2903{
3100{
3101 mxge_softc_t *sc;
2904 mxge_cmd_t cmd;
3102 mxge_cmd_t cmd;
2905 int i, err, big_bytes;
2906 bus_dmamap_t map;
3103 bus_dmamap_t map;
2907 bus_addr_t bus;
2908 struct lro_entry *lro_entry;
3104 struct lro_entry *lro_entry;
3105 int err, i, slice;
2909
3106
2910 SLIST_INIT(&sc->lro_free);
2911 SLIST_INIT(&sc->lro_active);
2912
3107
3108 sc = ss->sc;
3109 slice = ss - sc->ss;
3110
3111 SLIST_INIT(&ss->lro_free);
3112 SLIST_INIT(&ss->lro_active);
3113
2913 for (i = 0; i < sc->lro_cnt; i++) {
2914 lro_entry = (struct lro_entry *)
3114 for (i = 0; i < sc->lro_cnt; i++) {
3115 lro_entry = (struct lro_entry *)
2915 malloc(sizeof (*lro_entry), M_DEVBUF, M_NOWAIT | M_ZERO);
3116 malloc(sizeof (*lro_entry), M_DEVBUF,
3117 M_NOWAIT | M_ZERO);
2916 if (lro_entry == NULL) {
2917 sc->lro_cnt = i;
2918 break;
2919 }
3118 if (lro_entry == NULL) {
3119 sc->lro_cnt = i;
3120 break;
3121 }
2920 SLIST_INSERT_HEAD(&sc->lro_free, lro_entry, next);
3122 SLIST_INSERT_HEAD(&ss->lro_free, lro_entry, next);
2921 }
3123 }
2922
2923 /* Copy the MAC address in case it was overridden */
2924 bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN);
2925
2926 err = mxge_reset(sc, 1);
2927 if (err != 0) {
2928 device_printf(sc->dev, "failed to reset\n");
2929 return EIO;
2930 }
2931
2932 mxge_choose_params(sc->ifp->if_mtu, &big_bytes,
2933 &sc->rx_big.cl_size, &sc->rx_big.nbufs);
2934
2935 cmd.data0 = sc->rx_big.nbufs;
2936 err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
2937 &cmd);
2938 /* error is only meaningful if we're trying to set
2939 MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */
2940 if (err && sc->rx_big.nbufs > 1) {
2941 device_printf(sc->dev,
2942 "Failed to set alway-use-n to %d\n",
2943 sc->rx_big.nbufs);
2944 return EIO;
2945 }
2946 /* get the lanai pointers to the send and receive rings */
2947
3124 /* get the lanai pointers to the send and receive rings */
3125
2948 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
2949 sc->tx.lanai =
2950 (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0);
3126 err = 0;
3127 /* We currently only send from the first slice */
3128 if (slice == 0) {
3129 cmd.data0 = slice;
3130 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
3131 ss->tx.lanai =
3132 (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0);
3133 }
3134 cmd.data0 = slice;
2951 err |= mxge_send_cmd(sc,
3135 err |= mxge_send_cmd(sc,
2952 MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
2953 sc->rx_small.lanai =
3136 MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
3137 ss->rx_small.lanai =
2954 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
3138 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
3139 cmd.data0 = slice;
2955 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
3140 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
2956 sc->rx_big.lanai =
3141 ss->rx_big.lanai =
2957 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
2958
2959 if (err != 0) {
2960 device_printf(sc->dev,
2961 "failed to get ring sizes or locations\n");
2962 return EIO;
2963 }
2964
2965 /* stock receive rings */
3142 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
3143
3144 if (err != 0) {
3145 device_printf(sc->dev,
3146 "failed to get ring sizes or locations\n");
3147 return EIO;
3148 }
3149
3150 /* stock receive rings */
2966 for (i = 0; i <= sc->rx_small.mask; i++) {
2967 map = sc->rx_small.info[i].map;
2968 err = mxge_get_buf_small(sc, map, i);
3151 for (i = 0; i <= ss->rx_small.mask; i++) {
3152 map = ss->rx_small.info[i].map;
3153 err = mxge_get_buf_small(ss, map, i);
2969 if (err) {
2970 device_printf(sc->dev, "alloced %d/%d smalls\n",
3154 if (err) {
3155 device_printf(sc->dev, "alloced %d/%d smalls\n",
2971 i, sc->rx_small.mask + 1);
2972 goto abort;
3156 i, ss->rx_small.mask + 1);
3157 return ENOMEM;
2973 }
2974 }
3158 }
3159 }
2975 for (i = 0; i <= sc->rx_big.mask; i++) {
2976 sc->rx_big.shadow[i].addr_low = 0xffffffff;
2977 sc->rx_big.shadow[i].addr_high = 0xffffffff;
3160 for (i = 0; i <= ss->rx_big.mask; i++) {
3161 ss->rx_big.shadow[i].addr_low = 0xffffffff;
3162 ss->rx_big.shadow[i].addr_high = 0xffffffff;
2978 }
3163 }
2979 for (i = 0; i <= sc->rx_big.mask; i += sc->rx_big.nbufs) {
2980 map = sc->rx_big.info[i].map;
2981 err = mxge_get_buf_big(sc, map, i);
3164 ss->rx_big.nbufs = nbufs;
3165 ss->rx_big.cl_size = cl_size;
3166 for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) {
3167 map = ss->rx_big.info[i].map;
3168 err = mxge_get_buf_big(ss, map, i);
2982 if (err) {
2983 device_printf(sc->dev, "alloced %d/%d bigs\n",
3169 if (err) {
3170 device_printf(sc->dev, "alloced %d/%d bigs\n",
2984 i, sc->rx_big.mask + 1);
2985 goto abort;
3171 i, ss->rx_big.mask + 1);
3172 return ENOMEM;
2986 }
2987 }
3173 }
3174 }
3175 return 0;
3176}
2988
3177
3178static int
3179mxge_open(mxge_softc_t *sc)
3180{
3181 mxge_cmd_t cmd;
3182 int err, big_bytes, nbufs, slice, cl_size, i;
3183 bus_addr_t bus;
3184 volatile uint8_t *itable;
3185
3186 /* Copy the MAC address in case it was overridden */
3187 bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN);
3188
3189 err = mxge_reset(sc, 1);
3190 if (err != 0) {
3191 device_printf(sc->dev, "failed to reset\n");
3192 return EIO;
3193 }
3194
3195 if (sc->num_slices > 1) {
3196 /* setup the indirection table */
3197 cmd.data0 = sc->num_slices;
3198 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
3199 &cmd);
3200
3201 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET,
3202 &cmd);
3203 if (err != 0) {
3204 device_printf(sc->dev,
3205 "failed to setup rss tables\n");
3206 return err;
3207 }
3208
3209 /* just enable an identity mapping */
3210 itable = sc->sram + cmd.data0;
3211 for (i = 0; i < sc->num_slices; i++)
3212 itable[i] = (uint8_t)i;
3213
3214 cmd.data0 = 1;
3215 cmd.data1 = mxge_rss_hash_type;
3216 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd);
3217 if (err != 0) {
3218 device_printf(sc->dev, "failed to enable slices\n");
3219 return err;
3220 }
3221 }
3222
3223
3224 mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs);
3225
3226 cmd.data0 = nbufs;
3227 err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
3228 &cmd);
3229 /* error is only meaningful if we're trying to set
3230 MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */
3231 if (err && nbufs > 1) {
3232 device_printf(sc->dev,
3233 "Failed to set alway-use-n to %d\n",
3234 nbufs);
3235 return EIO;
3236 }
2989 /* Give the firmware the mtu and the big and small buffer
2990 sizes. The firmware wants the big buf size to be a power
2991 of two. Luckily, FreeBSD's clusters are powers of two */
2992 cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2993 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd);
2994 cmd.data0 = MHLEN - MXGEFW_PAD;
2995 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE,
2996 &cmd);
2997 cmd.data0 = big_bytes;
2998 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
2999
3000 if (err != 0) {
3001 device_printf(sc->dev, "failed to setup params\n");
3002 goto abort;
3003 }
3004
3005 /* Now give him the pointer to the stats block */
3237 /* Give the firmware the mtu and the big and small buffer
3238 sizes. The firmware wants the big buf size to be a power
3239 of two. Luckily, FreeBSD's clusters are powers of two */
3240 cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3241 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd);
3242 cmd.data0 = MHLEN - MXGEFW_PAD;
3243 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE,
3244 &cmd);
3245 cmd.data0 = big_bytes;
3246 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
3247
3248 if (err != 0) {
3249 device_printf(sc->dev, "failed to setup params\n");
3250 goto abort;
3251 }
3252
3253 /* Now give him the pointer to the stats block */
3006 cmd.data0 = MXGE_LOWPART_TO_U32(sc->fw_stats_dma.bus_addr);
3007 cmd.data1 = MXGE_HIGHPART_TO_U32(sc->fw_stats_dma.bus_addr);
3254 cmd.data0 = MXGE_LOWPART_TO_U32(sc->ss->fw_stats_dma.bus_addr);
3255 cmd.data1 = MXGE_HIGHPART_TO_U32(sc->ss->fw_stats_dma.bus_addr);
3008 cmd.data2 = sizeof(struct mcp_irq_data);
3009 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
3010
3011 if (err != 0) {
3256 cmd.data2 = sizeof(struct mcp_irq_data);
3257 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
3258
3259 if (err != 0) {
3012 bus = sc->fw_stats_dma.bus_addr;
3260 bus = sc->ss->fw_stats_dma.bus_addr;
3013 bus += offsetof(struct mcp_irq_data, send_done_count);
3014 cmd.data0 = MXGE_LOWPART_TO_U32(bus);
3015 cmd.data1 = MXGE_HIGHPART_TO_U32(bus);
3016 err = mxge_send_cmd(sc,
3017 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
3018 &cmd);
3019 /* Firmware cannot support multicast without STATS_DMA_V2 */
3020 sc->fw_multicast_support = 0;
3021 } else {
3022 sc->fw_multicast_support = 1;
3023 }
3024
3025 if (err != 0) {
3026 device_printf(sc->dev, "failed to setup params\n");
3027 goto abort;
3028 }
3029
3261 bus += offsetof(struct mcp_irq_data, send_done_count);
3262 cmd.data0 = MXGE_LOWPART_TO_U32(bus);
3263 cmd.data1 = MXGE_HIGHPART_TO_U32(bus);
3264 err = mxge_send_cmd(sc,
3265 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
3266 &cmd);
3267 /* Firmware cannot support multicast without STATS_DMA_V2 */
3268 sc->fw_multicast_support = 0;
3269 } else {
3270 sc->fw_multicast_support = 1;
3271 }
3272
3273 if (err != 0) {
3274 device_printf(sc->dev, "failed to setup params\n");
3275 goto abort;
3276 }
3277
3278 for (slice = 0; slice < sc->num_slices; slice++) {
3279 err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size);
3280 if (err != 0) {
3281 device_printf(sc->dev, "couldn't open slice %d\n",
3282 slice);
3283 goto abort;
3284 }
3285 }
3286
3030 /* Finally, start the firmware running */
3031 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd);
3032 if (err) {
3033 device_printf(sc->dev, "Couldn't bring up link\n");
3034 goto abort;
3035 }
3036 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
3037 sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;

--- 5 unchanged lines hidden (view full) ---

3043 mxge_free_mbufs(sc);
3044
3045 return err;
3046}
3047
3048static int
3049mxge_close(mxge_softc_t *sc)
3050{
3287 /* Finally, start the firmware running */
3288 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd);
3289 if (err) {
3290 device_printf(sc->dev, "Couldn't bring up link\n");
3291 goto abort;
3292 }
3293 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
3294 sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;

--- 5 unchanged lines hidden (view full) ---

3300 mxge_free_mbufs(sc);
3301
3302 return err;
3303}
3304
3305static int
3306mxge_close(mxge_softc_t *sc)
3307{
3051 struct lro_entry *lro_entry;
3052 mxge_cmd_t cmd;
3053 int err, old_down_cnt;
3054
3055 sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3056 old_down_cnt = sc->down_cnt;
3057 mb();
3058 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
3059 if (err) {
3060 device_printf(sc->dev, "Couldn't bring down link\n");
3061 }
3062 if (old_down_cnt == sc->down_cnt) {
3063 /* wait for down irq */
3064 DELAY(10 * sc->intr_coal_delay);
3065 }
3308 mxge_cmd_t cmd;
3309 int err, old_down_cnt;
3310
3311 sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3312 old_down_cnt = sc->down_cnt;
3313 mb();
3314 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
3315 if (err) {
3316 device_printf(sc->dev, "Couldn't bring down link\n");
3317 }
3318 if (old_down_cnt == sc->down_cnt) {
3319 /* wait for down irq */
3320 DELAY(10 * sc->intr_coal_delay);
3321 }
3322 mb();
3066 if (old_down_cnt == sc->down_cnt) {
3067 device_printf(sc->dev, "never got down irq\n");
3068 }
3069
3070 mxge_free_mbufs(sc);
3071
3323 if (old_down_cnt == sc->down_cnt) {
3324 device_printf(sc->dev, "never got down irq\n");
3325 }
3326
3327 mxge_free_mbufs(sc);
3328
3072 while (!SLIST_EMPTY(&sc->lro_free)) {
3073 lro_entry = SLIST_FIRST(&sc->lro_free);
3074 SLIST_REMOVE_HEAD(&sc->lro_free, next);
3075 }
3076 return 0;
3077}
3078
3079static void
3080mxge_setup_cfg_space(mxge_softc_t *sc)
3081{
3082 device_t dev = sc->dev;
3083 int reg;

--- 77 unchanged lines hidden (view full) ---

3161 /* XXXX waiting for pci_cfg_restore() to be exported */
3162 goto abort; /* just abort for now */
3163
3164 /* and redo any changes we made to our config space */
3165 mxge_setup_cfg_space(sc);
3166 } else {
3167 device_printf(sc->dev, "NIC did not reboot, ring state:\n");
3168 device_printf(sc->dev, "tx.req=%d tx.done=%d\n",
3329 return 0;
3330}
3331
3332static void
3333mxge_setup_cfg_space(mxge_softc_t *sc)
3334{
3335 device_t dev = sc->dev;
3336 int reg;

--- 77 unchanged lines hidden (view full) ---

3414 /* XXXX waiting for pci_cfg_restore() to be exported */
3415 goto abort; /* just abort for now */
3416
3417 /* and redo any changes we made to our config space */
3418 mxge_setup_cfg_space(sc);
3419 } else {
3420 device_printf(sc->dev, "NIC did not reboot, ring state:\n");
3421 device_printf(sc->dev, "tx.req=%d tx.done=%d\n",
3169 sc->tx.req, sc->tx.done);
3422 sc->ss->tx.req, sc->ss->tx.done);
3170 device_printf(sc->dev, "pkt_done=%d fw=%d\n",
3423 device_printf(sc->dev, "pkt_done=%d fw=%d\n",
3171 sc->tx.pkt_done,
3172 be32toh(sc->fw_stats->send_done_count));
3424 sc->ss->tx.pkt_done,
3425 be32toh(sc->ss->fw_stats->send_done_count));
3173 }
3174
3175 if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) {
3176 mxge_close(sc);
3177 err = mxge_open(sc);
3178 }
3179
3180abort:

--- 4 unchanged lines hidden (view full) ---

3185 if (err != 0) {
3186 callout_stop(&sc->co_hdl);
3187 }
3188}
3189
3190static void
3191mxge_watchdog(mxge_softc_t *sc)
3192{
3426 }
3427
3428 if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) {
3429 mxge_close(sc);
3430 err = mxge_open(sc);
3431 }
3432
3433abort:

--- 4 unchanged lines hidden (view full) ---

3438 if (err != 0) {
3439 callout_stop(&sc->co_hdl);
3440 }
3441}
3442
3443static void
3444mxge_watchdog(mxge_softc_t *sc)
3445{
3193 mxge_tx_buf_t *tx = &sc->tx;
3194 uint32_t rx_pause = be32toh(sc->fw_stats->dropped_pause);
3446 mxge_tx_ring_t *tx = &sc->ss->tx;
3447 uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause);
3195
3196 /* see if we have outstanding transmits, which
3197 have been pending for more than mxge_ticks */
3198 if (tx->req != tx->done &&
3199 tx->watchdog_req != tx->watchdog_done &&
3200 tx->done == tx->watchdog_done) {
3201 /* check for pause blocking before resetting */
3202 if (tx->watchdog_rx_pause == rx_pause)

--- 7 unchanged lines hidden (view full) ---

3210 tx->watchdog_done = tx->done;
3211 tx->watchdog_rx_pause = rx_pause;
3212
3213 if (sc->need_media_probe)
3214 mxge_media_probe(sc);
3215}
3216
3217static void
3448
3449 /* see if we have outstanding transmits, which
3450 have been pending for more than mxge_ticks */
3451 if (tx->req != tx->done &&
3452 tx->watchdog_req != tx->watchdog_done &&
3453 tx->done == tx->watchdog_done) {
3454 /* check for pause blocking before resetting */
3455 if (tx->watchdog_rx_pause == rx_pause)

--- 7 unchanged lines hidden (view full) ---

3463 tx->watchdog_done = tx->done;
3464 tx->watchdog_rx_pause = rx_pause;
3465
3466 if (sc->need_media_probe)
3467 mxge_media_probe(sc);
3468}
3469
3470static void
3471mxge_update_stats(mxge_softc_t *sc)
3472{
3473 struct mxge_slice_state *ss;
3474 u_long ipackets = 0;
3475 int slice;
3476
3477 for(slice = 0; slice < sc->num_slices; slice++) {
3478 ss = &sc->ss[slice];
3479 ipackets += ss->ipackets;
3480 }
3481 sc->ifp->if_ipackets = ipackets;
3482
3483}
3484static void
3218mxge_tick(void *arg)
3219{
3220 mxge_softc_t *sc = arg;
3221
3222
3223 /* Synchronize with possible callout reset/stop. */
3224 if (callout_pending(&sc->co_hdl) ||
3225 !callout_active(&sc->co_hdl)) {
3226 mtx_unlock(&sc->driver_mtx);
3227 return;
3228 }
3229
3485mxge_tick(void *arg)
3486{
3487 mxge_softc_t *sc = arg;
3488
3489
3490 /* Synchronize with possible callout reset/stop. */
3491 if (callout_pending(&sc->co_hdl) ||
3492 !callout_active(&sc->co_hdl)) {
3493 mtx_unlock(&sc->driver_mtx);
3494 return;
3495 }
3496
3497 /* aggregate stats from different slices */
3498 mxge_update_stats(sc);
3499
3230 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
3500 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
3231 mxge_watchdog(sc);
3501 if (!sc->watchdog_countdown) {
3502 mxge_watchdog(sc);
3503 sc->watchdog_countdown = 4;
3504 }
3505 sc->watchdog_countdown--;
3232}
3233
3234static int
3235mxge_media_change(struct ifnet *ifp)
3236{
3237 return EINVAL;
3238}
3239

--- 69 unchanged lines hidden (view full) ---

3309 /* take care of promis can allmulti
3310 flag chages */
3311 mxge_change_promisc(sc,
3312 ifp->if_flags & IFF_PROMISC);
3313 mxge_set_multicast_list(sc);
3314 }
3315 } else {
3316 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3506}
3507
3508static int
3509mxge_media_change(struct ifnet *ifp)
3510{
3511 return EINVAL;
3512}
3513

--- 69 unchanged lines hidden (view full) ---

3583 /* take care of promis can allmulti
3584 flag chages */
3585 mxge_change_promisc(sc,
3586 ifp->if_flags & IFF_PROMISC);
3587 mxge_set_multicast_list(sc);
3588 }
3589 } else {
3590 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3317 mxge_close(sc);
3318 callout_stop(&sc->co_hdl);
3591 callout_stop(&sc->co_hdl);
3592 mxge_close(sc);
3319 }
3320 }
3321 mtx_unlock(&sc->driver_mtx);
3322 break;
3323
3324 case SIOCADDMULTI:
3325 case SIOCDELMULTI:
3326 mtx_lock(&sc->driver_mtx);

--- 57 unchanged lines hidden (view full) ---

3384 err = ENOTTY;
3385 }
3386 return err;
3387}
3388
3389static void
3390mxge_fetch_tunables(mxge_softc_t *sc)
3391{
3593 }
3594 }
3595 mtx_unlock(&sc->driver_mtx);
3596 break;
3597
3598 case SIOCADDMULTI:
3599 case SIOCDELMULTI:
3600 mtx_lock(&sc->driver_mtx);

--- 57 unchanged lines hidden (view full) ---

3658 err = ENOTTY;
3659 }
3660 return err;
3661}
3662
3663static void
3664mxge_fetch_tunables(mxge_softc_t *sc)
3665{
3392
3666
3667 TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices);
3393 TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled",
3394 &mxge_flow_control);
3395 TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay",
3396 &mxge_intr_coal_delay);
3397 TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable",
3398 &mxge_nvidia_ecrc_enable);
3399 TUNABLE_INT_FETCH("hw.mxge.force_firmware",
3400 &mxge_force_firmware);
3401 TUNABLE_INT_FETCH("hw.mxge.deassert_wait",
3402 &mxge_deassert_wait);
3403 TUNABLE_INT_FETCH("hw.mxge.verbose",
3404 &mxge_verbose);
3405 TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks);
3406 TUNABLE_INT_FETCH("hw.mxge.lro_cnt", &sc->lro_cnt);
3668 TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled",
3669 &mxge_flow_control);
3670 TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay",
3671 &mxge_intr_coal_delay);
3672 TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable",
3673 &mxge_nvidia_ecrc_enable);
3674 TUNABLE_INT_FETCH("hw.mxge.force_firmware",
3675 &mxge_force_firmware);
3676 TUNABLE_INT_FETCH("hw.mxge.deassert_wait",
3677 &mxge_deassert_wait);
3678 TUNABLE_INT_FETCH("hw.mxge.verbose",
3679 &mxge_verbose);
3680 TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks);
3681 TUNABLE_INT_FETCH("hw.mxge.lro_cnt", &sc->lro_cnt);
3682 TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc);
3683 TUNABLE_INT_FETCH("hw.mxge.rss_hash_type", &mxge_rss_hash_type);
3407 if (sc->lro_cnt != 0)
3408 mxge_lro_cnt = sc->lro_cnt;
3409
3410 if (bootverbose)
3411 mxge_verbose = 1;
3412 if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000)
3413 mxge_intr_coal_delay = 30;
3414 if (mxge_ticks == 0)
3684 if (sc->lro_cnt != 0)
3685 mxge_lro_cnt = sc->lro_cnt;
3686
3687 if (bootverbose)
3688 mxge_verbose = 1;
3689 if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000)
3690 mxge_intr_coal_delay = 30;
3691 if (mxge_ticks == 0)
3415 mxge_ticks = hz;
3692 mxge_ticks = hz / 2;
3416 sc->pause = mxge_flow_control;
3693 sc->pause = mxge_flow_control;
3694 if (mxge_rss_hash_type < MXGEFW_RSS_HASH_TYPE_IPV4
3695 || mxge_rss_hash_type > MXGEFW_RSS_HASH_TYPE_SRC_PORT) {
3696 mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_PORT;
3697 }
3698}
3417
3699
3700
3701static void
3702mxge_free_slices(mxge_softc_t *sc)
3703{
3704 struct mxge_slice_state *ss;
3705 int i;
3706
3707
3708 if (sc->ss == NULL)
3709 return;
3710
3711 for (i = 0; i < sc->num_slices; i++) {
3712 ss = &sc->ss[i];
3713 if (ss->fw_stats != NULL) {
3714 mxge_dma_free(&ss->fw_stats_dma);
3715 ss->fw_stats = NULL;
3716 mtx_destroy(&ss->tx.mtx);
3717 }
3718 if (ss->rx_done.entry != NULL) {
3719 mxge_dma_free(&ss->rx_done.dma);
3720 ss->rx_done.entry = NULL;
3721 }
3722 }
3723 free(sc->ss, M_DEVBUF);
3724 sc->ss = NULL;
3418}
3419
3725}
3726
3727static int
3728mxge_alloc_slices(mxge_softc_t *sc)
3729{
3730 mxge_cmd_t cmd;
3731 struct mxge_slice_state *ss;
3732 size_t bytes;
3733 int err, i, max_intr_slots;
3734
3735 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
3736 if (err != 0) {
3737 device_printf(sc->dev, "Cannot determine rx ring size\n");
3738 return err;
3739 }
3740 sc->rx_ring_size = cmd.data0;
3741 max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t));
3742
3743 bytes = sizeof (*sc->ss) * sc->num_slices;
3744 sc->ss = malloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO);
3745 if (sc->ss == NULL)
3746 return (ENOMEM);
3747 for (i = 0; i < sc->num_slices; i++) {
3748 ss = &sc->ss[i];
3749
3750 ss->sc = sc;
3751
3752 /* allocate per-slice rx interrupt queues */
3753
3754 bytes = max_intr_slots * sizeof (*ss->rx_done.entry);
3755 err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096);
3756 if (err != 0)
3757 goto abort;
3758 ss->rx_done.entry = ss->rx_done.dma.addr;
3759 bzero(ss->rx_done.entry, bytes);
3760
3761 /*
3762 * allocate the per-slice firmware stats; stats
3763 * (including tx) are used used only on the first
3764 * slice for now
3765 */
3766 if (i > 0)
3767 continue;
3768
3769 bytes = sizeof (*ss->fw_stats);
3770 err = mxge_dma_alloc(sc, &ss->fw_stats_dma,
3771 sizeof (*ss->fw_stats), 64);
3772 if (err != 0)
3773 goto abort;
3774 ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr;
3775 snprintf(ss->tx.mtx_name, sizeof(ss->tx.mtx_name),
3776 "%s:tx(%d)", device_get_nameunit(sc->dev), i);
3777 mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF);
3778 }
3779
3780 return (0);
3781
3782abort:
3783 mxge_free_slices(sc);
3784 return (ENOMEM);
3785}
3786
3787static void
3788mxge_slice_probe(mxge_softc_t *sc)
3789{
3790 mxge_cmd_t cmd;
3791 char *old_fw;
3792 int msix_cnt, status, max_intr_slots;
3793
3794 sc->num_slices = 1;
3795 /*
3796 * don't enable multiple slices if they are not enabled,
3797 * or if this is not an SMP system
3798 */
3799
3800 if (mxge_max_slices == 0 || mxge_max_slices == 1 || mp_ncpus < 2)
3801 return;
3802
3803 /* see how many MSI-X interrupts are available */
3804 msix_cnt = pci_msix_count(sc->dev);
3805 if (msix_cnt < 2)
3806 return;
3807
3808 /* now load the slice aware firmware see what it supports */
3809 old_fw = sc->fw_name;
3810 if (old_fw == mxge_fw_aligned)
3811 sc->fw_name = mxge_fw_rss_aligned;
3812 else
3813 sc->fw_name = mxge_fw_rss_unaligned;
3814 status = mxge_load_firmware(sc, 0);
3815 if (status != 0) {
3816 device_printf(sc->dev, "Falling back to a single slice\n");
3817 return;
3818 }
3819
3820 /* try to send a reset command to the card to see if it
3821 is alive */
3822 memset(&cmd, 0, sizeof (cmd));
3823 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
3824 if (status != 0) {
3825 device_printf(sc->dev, "failed reset\n");
3826 goto abort_with_fw;
3827 }
3828
3829 /* get rx ring size */
3830 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
3831 if (status != 0) {
3832 device_printf(sc->dev, "Cannot determine rx ring size\n");
3833 goto abort_with_fw;
3834 }
3835 max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t));
3836
3837 /* tell it the size of the interrupt queues */
3838 cmd.data0 = max_intr_slots * sizeof (struct mcp_slot);
3839 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
3840 if (status != 0) {
3841 device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n");
3842 goto abort_with_fw;
3843 }
3844
3845 /* ask the maximum number of slices it supports */
3846 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd);
3847 if (status != 0) {
3848 device_printf(sc->dev,
3849 "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n");
3850 goto abort_with_fw;
3851 }
3852 sc->num_slices = cmd.data0;
3853 if (sc->num_slices > msix_cnt)
3854 sc->num_slices = msix_cnt;
3855
3856 if (mxge_max_slices == -1) {
3857 /* cap to number of CPUs in system */
3858 if (sc->num_slices > mp_ncpus)
3859 sc->num_slices = mp_ncpus;
3860 } else {
3861 if (sc->num_slices > mxge_max_slices)
3862 sc->num_slices = mxge_max_slices;
3863 }
3864 /* make sure it is a power of two */
3865 while (sc->num_slices & (sc->num_slices - 1))
3866 sc->num_slices--;
3867
3868 if (mxge_verbose)
3869 device_printf(sc->dev, "using %d slices\n",
3870 sc->num_slices);
3871
3872 return;
3873
3874abort_with_fw:
3875 sc->fw_name = old_fw;
3876 (void) mxge_load_firmware(sc, 0);
3877}
3878
3879static int
3880mxge_add_msix_irqs(mxge_softc_t *sc)
3881{
3882 size_t bytes;
3883 int count, err, i, rid;
3884
3885 rid = PCIR_BAR(2);
3886 sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3887 &rid, RF_ACTIVE);
3888
3889 if (sc->msix_table_res == NULL) {
3890 device_printf(sc->dev, "couldn't alloc MSIX table res\n");
3891 return ENXIO;
3892 }
3893
3894 count = sc->num_slices;
3895 err = pci_alloc_msix(sc->dev, &count);
3896 if (err != 0) {
3897 device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d"
3898 "err = %d \n", sc->num_slices, err);
3899 goto abort_with_msix_table;
3900 }
3901 if (count < sc->num_slices) {
3902 device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n",
3903 count, sc->num_slices);
3904 device_printf(sc->dev,
3905 "Try setting hw.mxge.max_slices to %d\n",
3906 count);
3907 err = ENOSPC;
3908 goto abort_with_msix;
3909 }
3910 bytes = sizeof (*sc->msix_irq_res) * sc->num_slices;
3911 sc->msix_irq_res = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
3912 if (sc->msix_irq_res == NULL) {
3913 err = ENOMEM;
3914 goto abort_with_msix;
3915 }
3916
3917 for (i = 0; i < sc->num_slices; i++) {
3918 rid = i + 1;
3919 sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev,
3920 SYS_RES_IRQ,
3921 &rid, RF_ACTIVE);
3922 if (sc->msix_irq_res[i] == NULL) {
3923 device_printf(sc->dev, "couldn't allocate IRQ res"
3924 " for message %d\n", i);
3925 err = ENXIO;
3926 goto abort_with_res;
3927 }
3928 }
3929
3930 bytes = sizeof (*sc->msix_ih) * sc->num_slices;
3931 sc->msix_ih = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
3932
3933 for (i = 0; i < sc->num_slices; i++) {
3934 err = bus_setup_intr(sc->dev, sc->msix_irq_res[i],
3935 INTR_TYPE_NET | INTR_MPSAFE,
3936 NULL, mxge_intr, &sc->ss[i],
3937 &sc->msix_ih[i]);
3938 if (err != 0) {
3939 device_printf(sc->dev, "couldn't setup intr for "
3940 "message %d\n", i);
3941 goto abort_with_intr;
3942 }
3943 }
3944
3945 if (mxge_verbose) {
3946 device_printf(sc->dev, "using %d msix IRQs:",
3947 sc->num_slices);
3948 for (i = 0; i < sc->num_slices; i++)
3949 printf(" %ld", rman_get_start(sc->msix_irq_res[i]));
3950 printf("\n");
3951 }
3952 return (0);
3953
3954abort_with_intr:
3955 for (i = 0; i < sc->num_slices; i++) {
3956 if (sc->msix_ih[i] != NULL) {
3957 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
3958 sc->msix_ih[i]);
3959 sc->msix_ih[i] = NULL;
3960 }
3961 }
3962 free(sc->msix_ih, M_DEVBUF);
3963
3964
3965abort_with_res:
3966 for (i = 0; i < sc->num_slices; i++) {
3967 rid = i + 1;
3968 if (sc->msix_irq_res[i] != NULL)
3969 bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
3970 sc->msix_irq_res[i]);
3971 sc->msix_irq_res[i] = NULL;
3972 }
3973 free(sc->msix_irq_res, M_DEVBUF);
3974
3975
3976abort_with_msix:
3977 pci_release_msi(sc->dev);
3978
3979abort_with_msix_table:
3980 bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
3981 sc->msix_table_res);
3982
3983 return err;
3984}
3985
3986static int
3987mxge_add_single_irq(mxge_softc_t *sc)
3988{
3989 int count, err, rid;
3990
3991 count = pci_msi_count(sc->dev);
3992 if (count == 1 && pci_alloc_msi(sc->dev, &count) == 0) {
3993 rid = 1;
3994 sc->msi_enabled = 1;
3995 } else {
3996 rid = 0;
3997 }
3998 sc->irq_res = bus_alloc_resource(sc->dev, SYS_RES_IRQ, &rid, 0, ~0,
3999 1, RF_SHAREABLE | RF_ACTIVE);
4000 if (sc->irq_res == NULL) {
4001 device_printf(sc->dev, "could not alloc interrupt\n");
4002 return ENXIO;
4003 }
4004 if (mxge_verbose)
4005 device_printf(sc->dev, "using %s irq %ld\n",
4006 sc->msi_enabled ? "MSI" : "INTx",
4007 rman_get_start(sc->irq_res));
4008 err = bus_setup_intr(sc->dev, sc->irq_res,
4009 INTR_TYPE_NET | INTR_MPSAFE,
4010 NULL, mxge_intr, &sc->ss[0], &sc->ih);
4011 if (err != 0) {
4012 bus_release_resource(sc->dev, SYS_RES_IRQ,
4013 sc->msi_enabled ? 1 : 0, sc->irq_res);
4014 if (sc->msi_enabled)
4015 pci_release_msi(sc->dev);
4016 }
4017 return err;
4018}
4019
4020static void
4021mxge_rem_msix_irqs(mxge_softc_t *sc)
4022{
4023 int i, rid;
4024
4025 for (i = 0; i < sc->num_slices; i++) {
4026 if (sc->msix_ih[i] != NULL) {
4027 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
4028 sc->msix_ih[i]);
4029 sc->msix_ih[i] = NULL;
4030 }
4031 }
4032 free(sc->msix_ih, M_DEVBUF);
4033
4034 for (i = 0; i < sc->num_slices; i++) {
4035 rid = i + 1;
4036 if (sc->msix_irq_res[i] != NULL)
4037 bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
4038 sc->msix_irq_res[i]);
4039 sc->msix_irq_res[i] = NULL;
4040 }
4041 free(sc->msix_irq_res, M_DEVBUF);
4042
4043 bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
4044 sc->msix_table_res);
4045
4046 pci_release_msi(sc->dev);
4047 return;
4048}
4049
4050static void
4051mxge_rem_single_irq(mxge_softc_t *sc)
4052{
4053 bus_teardown_intr(sc->dev, sc->irq_res, sc->ih);
4054 bus_release_resource(sc->dev, SYS_RES_IRQ,
4055 sc->msi_enabled ? 1 : 0, sc->irq_res);
4056 if (sc->msi_enabled)
4057 pci_release_msi(sc->dev);
4058}
4059
4060static void
4061mxge_rem_irq(mxge_softc_t *sc)
4062{
4063 if (sc->num_slices > 1)
4064 mxge_rem_msix_irqs(sc);
4065 else
4066 mxge_rem_single_irq(sc);
4067}
4068
4069static int
4070mxge_add_irq(mxge_softc_t *sc)
4071{
4072 int err;
4073
4074 if (sc->num_slices > 1)
4075 err = mxge_add_msix_irqs(sc);
4076 else
4077 err = mxge_add_single_irq(sc);
4078
4079 if (0 && err == 0 && sc->num_slices > 1) {
4080 mxge_rem_msix_irqs(sc);
4081 err = mxge_add_msix_irqs(sc);
4082 }
4083 return err;
4084}
4085
4086
3420static int
3421mxge_attach(device_t dev)
3422{
3423 mxge_softc_t *sc = device_get_softc(dev);
3424 struct ifnet *ifp;
4087static int
4088mxge_attach(device_t dev)
4089{
4090 mxge_softc_t *sc = device_get_softc(dev);
4091 struct ifnet *ifp;
3425 int count, rid, err;
4092 int err, rid;
3426
3427 sc->dev = dev;
3428 mxge_fetch_tunables(sc);
3429
3430 err = bus_dma_tag_create(NULL, /* parent */
3431 1, /* alignment */
4093
4094 sc->dev = dev;
4095 mxge_fetch_tunables(sc);
4096
4097 err = bus_dma_tag_create(NULL, /* parent */
4098 1, /* alignment */
3432 4096, /* boundary */
4099 0, /* boundary */
3433 BUS_SPACE_MAXADDR, /* low */
3434 BUS_SPACE_MAXADDR, /* high */
3435 NULL, NULL, /* filter */
3436 65536 + 256, /* maxsize */
3437 MXGE_MAX_SEND_DESC, /* num segs */
4100 BUS_SPACE_MAXADDR, /* low */
4101 BUS_SPACE_MAXADDR, /* high */
4102 NULL, NULL, /* filter */
4103 65536 + 256, /* maxsize */
4104 MXGE_MAX_SEND_DESC, /* num segs */
3438 4096, /* maxsegsize */
4105 65536, /* maxsegsize */
3439 0, /* flags */
3440 NULL, NULL, /* lock */
3441 &sc->parent_dmat); /* tag */
3442
3443 if (err != 0) {
3444 device_printf(sc->dev, "Err %d allocating parent dmat\n",
3445 err);
3446 goto abort_with_nothing;
3447 }
3448
3449 ifp = sc->ifp = if_alloc(IFT_ETHER);
3450 if (ifp == NULL) {
3451 device_printf(dev, "can not if_alloc()\n");
3452 err = ENOSPC;
3453 goto abort_with_parent_dmat;
3454 }
4106 0, /* flags */
4107 NULL, NULL, /* lock */
4108 &sc->parent_dmat); /* tag */
4109
4110 if (err != 0) {
4111 device_printf(sc->dev, "Err %d allocating parent dmat\n",
4112 err);
4113 goto abort_with_nothing;
4114 }
4115
4116 ifp = sc->ifp = if_alloc(IFT_ETHER);
4117 if (ifp == NULL) {
4118 device_printf(dev, "can not if_alloc()\n");
4119 err = ENOSPC;
4120 goto abort_with_parent_dmat;
4121 }
4122 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
4123
3455 snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd",
3456 device_get_nameunit(dev));
3457 mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF);
4124 snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd",
4125 device_get_nameunit(dev));
4126 mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF);
3458 snprintf(sc->tx_mtx_name, sizeof(sc->tx_mtx_name), "%s:tx",
3459 device_get_nameunit(dev));
3460 mtx_init(&sc->tx_mtx, sc->tx_mtx_name, NULL, MTX_DEF);
3461 snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name),
3462 "%s:drv", device_get_nameunit(dev));
3463 mtx_init(&sc->driver_mtx, sc->driver_mtx_name,
3464 MTX_NETWORK_LOCK, MTX_DEF);
3465
3466 callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0);
3467
3468 mxge_setup_cfg_space(sc);

--- 36 unchanged lines hidden (view full) ---

3505 sizeof (mxge_cmd_t), 64);
3506 if (err != 0)
3507 goto abort_with_mem_res;
3508 sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr;
3509 err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64);
3510 if (err != 0)
3511 goto abort_with_cmd_dma;
3512
4127 snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name),
4128 "%s:drv", device_get_nameunit(dev));
4129 mtx_init(&sc->driver_mtx, sc->driver_mtx_name,
4130 MTX_NETWORK_LOCK, MTX_DEF);
4131
4132 callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0);
4133
4134 mxge_setup_cfg_space(sc);

--- 36 unchanged lines hidden (view full) ---

4171 sizeof (mxge_cmd_t), 64);
4172 if (err != 0)
4173 goto abort_with_mem_res;
4174 sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr;
4175 err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64);
4176 if (err != 0)
4177 goto abort_with_cmd_dma;
4178
3513 err = mxge_dma_alloc(sc, &sc->fw_stats_dma,
3514 sizeof (*sc->fw_stats), 64);
3515 if (err != 0)
3516 goto abort_with_zeropad_dma;
3517 sc->fw_stats = (mcp_irq_data_t *)sc->fw_stats_dma.addr;
3518
3519 err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096);
3520 if (err != 0)
4179 err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096);
4180 if (err != 0)
3521 goto abort_with_fw_stats;
4181 goto abort_with_zeropad_dma;
3522
4182
3523 /* Add our ithread */
3524 count = pci_msi_count(dev);
3525 if (count == 1 && pci_alloc_msi(dev, &count) == 0) {
3526 rid = 1;
3527 sc->msi_enabled = 1;
3528 } else {
3529 rid = 0;
3530 }
3531 sc->irq_res = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 0, ~0,
3532 1, RF_SHAREABLE | RF_ACTIVE);
3533 if (sc->irq_res == NULL) {
3534 device_printf(dev, "could not alloc interrupt\n");
3535 goto abort_with_dmabench;
3536 }
3537 if (mxge_verbose)
3538 device_printf(dev, "using %s irq %ld\n",
3539 sc->msi_enabled ? "MSI" : "INTx",
3540 rman_get_start(sc->irq_res));
3541 /* select & load the firmware */
3542 err = mxge_select_firmware(sc);
3543 if (err != 0)
4183 /* select & load the firmware */
4184 err = mxge_select_firmware(sc);
4185 if (err != 0)
3544 goto abort_with_irq_res;
4186 goto abort_with_dmabench;
3545 sc->intr_coal_delay = mxge_intr_coal_delay;
4187 sc->intr_coal_delay = mxge_intr_coal_delay;
4188
4189 mxge_slice_probe(sc);
4190 err = mxge_alloc_slices(sc);
4191 if (err != 0)
4192 goto abort_with_dmabench;
4193
3546 err = mxge_reset(sc, 0);
3547 if (err != 0)
4194 err = mxge_reset(sc, 0);
4195 if (err != 0)
3548 goto abort_with_irq_res;
4196 goto abort_with_slices;
3549
3550 err = mxge_alloc_rings(sc);
3551 if (err != 0) {
3552 device_printf(sc->dev, "failed to allocate rings\n");
4197
4198 err = mxge_alloc_rings(sc);
4199 if (err != 0) {
4200 device_printf(sc->dev, "failed to allocate rings\n");
3553 goto abort_with_irq_res;
4201 goto abort_with_dmabench;
3554 }
3555
4202 }
4203
3556 err = bus_setup_intr(sc->dev, sc->irq_res,
3557 INTR_TYPE_NET | INTR_MPSAFE,
3558 NULL, mxge_intr, sc, &sc->ih);
4204 err = mxge_add_irq(sc);
3559 if (err != 0) {
4205 if (err != 0) {
4206 device_printf(sc->dev, "failed to add irq\n");
3560 goto abort_with_rings;
3561 }
4207 goto abort_with_rings;
4208 }
3562 /* hook into the network stack */
3563 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
4209
3564 ifp->if_baudrate = 100000000;
3565 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 |
3566 IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
3567 IFCAP_VLAN_HWCSUM | IFCAP_LRO;
3568
3569 sc->max_mtu = mxge_max_mtu(sc);
3570 if (sc->max_mtu >= 9000)
3571 ifp->if_capabilities |= IFCAP_JUMBO_MTU;

--- 21 unchanged lines hidden (view full) ---

3593 if (ifp->if_capabilities & IFCAP_JUMBO_MTU)
3594 ifp->if_mtu = 9000;
3595
3596 mxge_add_sysctls(sc);
3597 return 0;
3598
3599abort_with_rings:
3600 mxge_free_rings(sc);
4210 ifp->if_baudrate = 100000000;
4211 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 |
4212 IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
4213 IFCAP_VLAN_HWCSUM | IFCAP_LRO;
4214
4215 sc->max_mtu = mxge_max_mtu(sc);
4216 if (sc->max_mtu >= 9000)
4217 ifp->if_capabilities |= IFCAP_JUMBO_MTU;

--- 21 unchanged lines hidden (view full) ---

4239 if (ifp->if_capabilities & IFCAP_JUMBO_MTU)
4240 ifp->if_mtu = 9000;
4241
4242 mxge_add_sysctls(sc);
4243 return 0;
4244
4245abort_with_rings:
4246 mxge_free_rings(sc);
3601abort_with_irq_res:
3602 bus_release_resource(dev, SYS_RES_IRQ,
3603 sc->msi_enabled ? 1 : 0, sc->irq_res);
3604 if (sc->msi_enabled)
3605 pci_release_msi(dev);
4247abort_with_slices:
4248 mxge_free_slices(sc);
3606abort_with_dmabench:
3607 mxge_dma_free(&sc->dmabench_dma);
4249abort_with_dmabench:
4250 mxge_dma_free(&sc->dmabench_dma);
3608abort_with_fw_stats:
3609 mxge_dma_free(&sc->fw_stats_dma);
3610abort_with_zeropad_dma:
3611 mxge_dma_free(&sc->zeropad_dma);
3612abort_with_cmd_dma:
3613 mxge_dma_free(&sc->cmd_dma);
3614abort_with_mem_res:
3615 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
3616abort_with_lock:
3617 pci_disable_busmaster(dev);
3618 mtx_destroy(&sc->cmd_mtx);
4251abort_with_zeropad_dma:
4252 mxge_dma_free(&sc->zeropad_dma);
4253abort_with_cmd_dma:
4254 mxge_dma_free(&sc->cmd_dma);
4255abort_with_mem_res:
4256 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
4257abort_with_lock:
4258 pci_disable_busmaster(dev);
4259 mtx_destroy(&sc->cmd_mtx);
3619 mtx_destroy(&sc->tx_mtx);
3620 mtx_destroy(&sc->driver_mtx);
3621 if_free(ifp);
3622abort_with_parent_dmat:
3623 bus_dma_tag_destroy(sc->parent_dmat);
3624
3625abort_with_nothing:
3626 return err;
3627}

--- 4 unchanged lines hidden (view full) ---

3632 mxge_softc_t *sc = device_get_softc(dev);
3633
3634 if (sc->ifp->if_vlantrunk != NULL) {
3635 device_printf(sc->dev,
3636 "Detach vlans before removing module\n");
3637 return EBUSY;
3638 }
3639 mtx_lock(&sc->driver_mtx);
4260 mtx_destroy(&sc->driver_mtx);
4261 if_free(ifp);
4262abort_with_parent_dmat:
4263 bus_dma_tag_destroy(sc->parent_dmat);
4264
4265abort_with_nothing:
4266 return err;
4267}

--- 4 unchanged lines hidden (view full) ---

4272 mxge_softc_t *sc = device_get_softc(dev);
4273
4274 if (sc->ifp->if_vlantrunk != NULL) {
4275 device_printf(sc->dev,
4276 "Detach vlans before removing module\n");
4277 return EBUSY;
4278 }
4279 mtx_lock(&sc->driver_mtx);
4280 callout_stop(&sc->co_hdl);
3640 if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING)
3641 mxge_close(sc);
4281 if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING)
4282 mxge_close(sc);
3642 callout_stop(&sc->co_hdl);
3643 mtx_unlock(&sc->driver_mtx);
3644 ether_ifdetach(sc->ifp);
3645 ifmedia_removeall(&sc->media);
3646 mxge_dummy_rdma(sc, 0);
4283 mtx_unlock(&sc->driver_mtx);
4284 ether_ifdetach(sc->ifp);
4285 ifmedia_removeall(&sc->media);
4286 mxge_dummy_rdma(sc, 0);
3647 bus_teardown_intr(sc->dev, sc->irq_res, sc->ih);
4287 mxge_rem_sysctls(sc);
4288 mxge_rem_irq(sc);
3648 mxge_free_rings(sc);
4289 mxge_free_rings(sc);
3649 bus_release_resource(dev, SYS_RES_IRQ,
3650 sc->msi_enabled ? 1 : 0, sc->irq_res);
3651 if (sc->msi_enabled)
3652 pci_release_msi(dev);
3653
3654 sc->rx_done.entry = NULL;
3655 mxge_dma_free(&sc->fw_stats_dma);
4290 mxge_free_slices(sc);
3656 mxge_dma_free(&sc->dmabench_dma);
3657 mxge_dma_free(&sc->zeropad_dma);
3658 mxge_dma_free(&sc->cmd_dma);
3659 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
3660 pci_disable_busmaster(dev);
3661 mtx_destroy(&sc->cmd_mtx);
4291 mxge_dma_free(&sc->dmabench_dma);
4292 mxge_dma_free(&sc->zeropad_dma);
4293 mxge_dma_free(&sc->cmd_dma);
4294 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
4295 pci_disable_busmaster(dev);
4296 mtx_destroy(&sc->cmd_mtx);
3662 mtx_destroy(&sc->tx_mtx);
3663 mtx_destroy(&sc->driver_mtx);
3664 if_free(sc->ifp);
3665 bus_dma_tag_destroy(sc->parent_dmat);
3666 return 0;
3667}
3668
3669static int
3670mxge_shutdown(device_t dev)

--- 12 unchanged lines hidden --- 		4297 mtx_destroy(&sc->driver_mtx);
4298 if_free(sc->ifp);
4299 bus_dma_tag_destroy(sc->parent_dmat);
4300 return 0;
4301}
4302
4303static int
4304mxge_shutdown(device_t dev)

--- 12 unchanged lines hidden ---