1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
122. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_main.c 171335 2007-07-10 06:01:45Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/bus.h>
37#include <sys/module.h>
38#include <sys/pciio.h>
39#include <sys/conf.h>
40#include <machine/bus.h>
41#include <machine/resource.h>
42#include <sys/bus_dma.h>
43#include <sys/rman.h>
44#include <sys/ioccom.h>
45#include <sys/mbuf.h>
46#include <sys/linker.h>
47#include <sys/firmware.h>
48#include <sys/socket.h>
49#include <sys/sockio.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <sys/queue.h>
53#include <sys/taskqueue.h>
54
55#include <net/bpf.h>
56#include <net/ethernet.h>
57#include <net/if.h>
58#include <net/if_arp.h>
59#include <net/if_dl.h>
60#include <net/if_media.h>
61#include <net/if_types.h>
62
63#include <netinet/in_systm.h>
64#include <netinet/in.h>
65#include <netinet/if_ether.h>
66#include <netinet/ip.h>
67#include <netinet/ip.h>
68#include <netinet/tcp.h>
69#include <netinet/udp.h>
70
71#include <dev/pci/pcireg.h>
72#include <dev/pci/pcivar.h>
73#include <dev/pci/pci_private.h>
74
75#ifdef CONFIG_DEFINED
76#include <cxgb_include.h>
77#else
78#include <dev/cxgb/cxgb_include.h>
79#endif
80
81#ifdef PRIV_SUPPORTED
82#include <sys/priv.h>
83#endif
84
85static int cxgb_setup_msix(adapter_t *, int);
86static void cxgb_teardown_msix(adapter_t *);
87static void cxgb_init(void *);
88static void cxgb_init_locked(struct port_info *);
89static void cxgb_stop_locked(struct port_info *);
90static void cxgb_set_rxmode(struct port_info *);
91static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
92static void cxgb_start(struct ifnet *);
93static void cxgb_start_proc(void *, int ncount);
94static int cxgb_media_change(struct ifnet *);
95static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
96static int setup_sge_qsets(adapter_t *);
97static void cxgb_async_intr(void *);
98static void cxgb_ext_intr_handler(void *, int);
99static void cxgb_tick_handler(void *, int);
100static void cxgb_down_locked(struct adapter *sc);
101static void cxgb_tick(void *);
102static void setup_rss(adapter_t *sc);
103
104/* Attachment glue for the PCI controller end of the device. Each port of
105 * the device is attached separately, as defined later.
106 */
107static int cxgb_controller_probe(device_t);
108static int cxgb_controller_attach(device_t);
109static int cxgb_controller_detach(device_t);
110static void cxgb_free(struct adapter *);
111static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
112 unsigned int end);
113static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf);
114static int cxgb_get_regs_len(void);
115static int offload_open(struct port_info *pi);
116#ifdef notyet
117static int offload_close(struct toedev *tdev);
118#endif
119
120
121static device_method_t cxgb_controller_methods[] = {
122 DEVMETHOD(device_probe, cxgb_controller_probe),
123 DEVMETHOD(device_attach, cxgb_controller_attach),
124 DEVMETHOD(device_detach, cxgb_controller_detach),
125
126 /* bus interface */
127 DEVMETHOD(bus_print_child, bus_generic_print_child),
128 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
129
130 { 0, 0 }
131};
132
133static driver_t cxgb_controller_driver = {
134 "cxgbc",
135 cxgb_controller_methods,
136 sizeof(struct adapter)
137};
138
139static devclass_t cxgb_controller_devclass;
140DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
141
142/*
143 * Attachment glue for the ports. Attachment is done directly to the
144 * controller device.
145 */
146static int cxgb_port_probe(device_t);
147static int cxgb_port_attach(device_t);
148static int cxgb_port_detach(device_t);
149
150static device_method_t cxgb_port_methods[] = {
151 DEVMETHOD(device_probe, cxgb_port_probe),
152 DEVMETHOD(device_attach, cxgb_port_attach),
153 DEVMETHOD(device_detach, cxgb_port_detach),
154 { 0, 0 }
155};
156
157static driver_t cxgb_port_driver = {
158 "cxgb",
159 cxgb_port_methods,
160 0
161};
162
163static d_ioctl_t cxgb_extension_ioctl;
164static d_open_t cxgb_extension_open;
165static d_close_t cxgb_extension_close;
166
167static struct cdevsw cxgb_cdevsw = {
168 .d_version = D_VERSION,
169 .d_flags = 0,
170 .d_open = cxgb_extension_open,
171 .d_close = cxgb_extension_close,
172 .d_ioctl = cxgb_extension_ioctl,
173 .d_name = "cxgb",
174};
175
176static devclass_t cxgb_port_devclass;
177DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
178
179#define SGE_MSIX_COUNT (SGE_QSETS + 1)
180
181extern int collapse_mbufs;
182/*
183 * The driver uses the best interrupt scheme available on a platform in the
184 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
185 * of these schemes the driver may consider as follows:
186 *
187 * msi = 2: choose from among all three options
188 * msi = 1 : only consider MSI and pin interrupts
189 * msi = 0: force pin interrupts
190 */
191static int msi_allowed = 2;
192
193TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
194SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
195SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
196 "MSI-X, MSI, INTx selector");
197
198/*
199 * The driver enables offload as a default.
200 * To disable it, use ofld_disable = 1.
201 */
202static int ofld_disable = 0;
203TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
204SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
205 "disable ULP offload");
206
207/*
208 * The driver uses an auto-queue algorithm by default.
209 * To disable it and force a single queue-set per port, use singleq = 1.
210 */
211static int singleq = 1;
212TUNABLE_INT("hw.cxgb.singleq", &singleq);
213SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
214 "use a single queue-set per port");
215
216enum {
217 MAX_TXQ_ENTRIES = 16384,
218 MAX_CTRL_TXQ_ENTRIES = 1024,
219 MAX_RSPQ_ENTRIES = 16384,
220 MAX_RX_BUFFERS = 16384,
221 MAX_RX_JUMBO_BUFFERS = 16384,
222 MIN_TXQ_ENTRIES = 4,
223 MIN_CTRL_TXQ_ENTRIES = 4,
224 MIN_RSPQ_ENTRIES = 32,
225 MIN_FL_ENTRIES = 32
226};
227
228#define PORT_MASK ((1 << MAX_NPORTS) - 1)
229
230/* Table for probing the cards. The desc field isn't actually used */
231struct cxgb_ident {
232 uint16_t vendor;
233 uint16_t device;
234 int index;
235 char *desc;
236} cxgb_identifiers[] = {
237 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
238 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
239 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
240 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
241 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
242 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
243 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
244 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
245 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
246 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
247 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
248 {0, 0, 0, NULL}
249};
250
251static struct cxgb_ident *
252cxgb_get_ident(device_t dev)
253{
254 struct cxgb_ident *id;
255
256 for (id = cxgb_identifiers; id->desc != NULL; id++) {
257 if ((id->vendor == pci_get_vendor(dev)) &&
258 (id->device == pci_get_device(dev))) {
259 return (id);
260 }
261 }
262 return (NULL);
263}
264
265static const struct adapter_info *
266cxgb_get_adapter_info(device_t dev)
267{
268 struct cxgb_ident *id;
269 const struct adapter_info *ai;
270
271 id = cxgb_get_ident(dev);
272 if (id == NULL)
273 return (NULL);
274
275 ai = t3_get_adapter_info(id->index);
276
277 return (ai);
278}
279
280static int
281cxgb_controller_probe(device_t dev)
282{
283 const struct adapter_info *ai;
284 char *ports, buf[80];
285 int nports;
286
287 ai = cxgb_get_adapter_info(dev);
288 if (ai == NULL)
289 return (ENXIO);
290
291 nports = ai->nports0 + ai->nports1;
292 if (nports == 1)
293 ports = "port";
294 else
295 ports = "ports";
296
297 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports);
298 device_set_desc_copy(dev, buf);
299 return (BUS_PROBE_DEFAULT);
300}
301
302static int
303upgrade_fw(adapter_t *sc)
304{
305 char buf[32];
306#ifdef FIRMWARE_LATEST
307 const struct firmware *fw;
308#else
309 struct firmware *fw;
310#endif
311 int status;
312
313 snprintf(&buf[0], sizeof(buf), "t3fw%d%d%d", FW_VERSION_MAJOR,
314 FW_VERSION_MINOR, FW_VERSION_MICRO);
315
316 fw = firmware_get(buf);
317
318 if (fw == NULL) {
319 device_printf(sc->dev, "Could not find firmware image %s\n", buf);
320 return (ENOENT);
321 }
322 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
323
324 firmware_put(fw, FIRMWARE_UNLOAD);
325
326 return (status);
327}
328
329static int
330cxgb_controller_attach(device_t dev)
331{
332 device_t child;
333 const struct adapter_info *ai;
334 struct adapter *sc;
335 int i, reg, msi_needed, error = 0;
336 uint32_t vers;
337 int port_qsets = 1;
338
339 sc = device_get_softc(dev);
340 sc->dev = dev;
341 sc->msi_count = 0;
342
343 /* find the PCIe link width and set max read request to 4KB*/
344 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
345 uint16_t lnk, pectl;
346 lnk = pci_read_config(dev, reg + 0x12, 2);
347 sc->link_width = (lnk >> 4) & 0x3f;
348
349 pectl = pci_read_config(dev, reg + 0x8, 2);
350 pectl = (pectl & ~0x7000) | (5 << 12);
351 pci_write_config(dev, reg + 0x8, pectl, 2);
352 }
353 if (sc->link_width != 0 && sc->link_width <= 4) {
354 device_printf(sc->dev,
355 "PCIe x%d Link, expect reduced performance\n",
356 sc->link_width);
357 }
358
359 pci_enable_busmaster(dev);
360 /*
361 * Allocate the registers and make them available to the driver.
362 * The registers that we care about for NIC mode are in BAR 0
363 */
364 sc->regs_rid = PCIR_BAR(0);
365 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
366 &sc->regs_rid, RF_ACTIVE)) == NULL) {
367 device_printf(dev, "Cannot allocate BAR\n");
368 return (ENXIO);
369 }
370
371 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
372 device_get_unit(dev));
373 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
374
375 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
376 device_get_unit(dev));
377 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
378 device_get_unit(dev));
379 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
380 device_get_unit(dev));
381
382 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_DEF);
383 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
384 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
385
386 sc->bt = rman_get_bustag(sc->regs_res);
387 sc->bh = rman_get_bushandle(sc->regs_res);
388 sc->mmio_len = rman_get_size(sc->regs_res);
389
390 ai = cxgb_get_adapter_info(dev);
391 if (t3_prep_adapter(sc, ai, 1) < 0) {
392 printf("prep adapter failed\n");
393 error = ENODEV;
394 goto out;
395 }
396 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
397 * enough messages for the queue sets. If that fails, try falling
398 * back to MSI. If that fails, then try falling back to the legacy
399 * interrupt pin model.
400 */
401#ifdef MSI_SUPPORTED
402
403 sc->msix_regs_rid = 0x20;
404 if ((msi_allowed >= 2) &&
405 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
406 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
407
408 msi_needed = sc->msi_count = SGE_MSIX_COUNT;
409
410 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
411 (sc->msi_count != msi_needed)) {
412 device_printf(dev, "msix allocation failed - msi_count = %d"
413 " msi_needed=%d will try msi err=%d\n", sc->msi_count,
414 msi_needed, error);
415 sc->msi_count = 0;
416 pci_release_msi(dev);
417 bus_release_resource(dev, SYS_RES_MEMORY,
418 sc->msix_regs_rid, sc->msix_regs_res);
419 sc->msix_regs_res = NULL;
420 } else {
421 sc->flags |= USING_MSIX;
422 sc->cxgb_intr = t3_intr_msix;
423 }
424 }
425
426 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
427 sc->msi_count = 1;
428 if (pci_alloc_msi(dev, &sc->msi_count)) {
429 device_printf(dev, "alloc msi failed - will try INTx\n");
430 sc->msi_count = 0;
431 pci_release_msi(dev);
432 } else {
433 sc->flags |= USING_MSI;
434 sc->irq_rid = 1;
435 sc->cxgb_intr = t3_intr_msi;
436 }
437 }
438#endif
439 if (sc->msi_count == 0) {
440 device_printf(dev, "using line interrupts\n");
441 sc->irq_rid = 0;
442 sc->cxgb_intr = t3b_intr;
443 }
444
445
446 /* Create a private taskqueue thread for handling driver events */
447#ifdef TASKQUEUE_CURRENT
448 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
449 taskqueue_thread_enqueue, &sc->tq);
450#else
451 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
452 taskqueue_thread_enqueue, &sc->tq);
453#endif
454 if (sc->tq == NULL) {
455 device_printf(dev, "failed to allocate controller task queue\n");
456 goto out;
457 }
458
459 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
460 device_get_nameunit(dev));
461 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
462 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
463
464
465 /* Create a periodic callout for checking adapter status */
466 callout_init(&sc->cxgb_tick_ch, TRUE);
467
468 if (t3_check_fw_version(sc) != 0) {
469 /*
470 * Warn user that a firmware update will be attempted in init.
471 */
472 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
473 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
474 sc->flags &= ~FW_UPTODATE;
475 } else {
476 sc->flags |= FW_UPTODATE;
477 }
478
479 if ((sc->flags & USING_MSIX) && !singleq)
480 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
481
482 /*
483 * Create a child device for each MAC. The ethernet attachment
484 * will be done in these children.
485 */
486 for (i = 0; i < (sc)->params.nports; i++) {
487 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
488 device_printf(dev, "failed to add child port\n");
489 error = EINVAL;
490 goto out;
491 }
492 sc->portdev[i] = child;
493 sc->port[i].adapter = sc;
494 sc->port[i].nqsets = port_qsets;
495 sc->port[i].first_qset = i*port_qsets;
496 sc->port[i].port = i;
497 device_set_softc(child, &sc->port[i]);
498 }
499 if ((error = bus_generic_attach(dev)) != 0)
500 goto out;
501
502 /*
503 * XXX need to poll for link status
504 */
505 sc->params.stats_update_period = 1;
506
507 /* initialize sge private state */
508 t3_sge_init_adapter(sc);
509
510 t3_led_ready(sc);
511
512 cxgb_offload_init();
513 if (is_offload(sc)) {
514 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
515 cxgb_adapter_ofld(sc);
516 }
517 error = t3_get_fw_version(sc, &vers);
518 if (error)
519 goto out;
520
521 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
522 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
523 G_FW_VERSION_MICRO(vers));
524
525 t3_add_sysctls(sc);
526out:
527 if (error)
528 cxgb_free(sc);
529
530 return (error);
531}
532
533static int
534cxgb_controller_detach(device_t dev)
535{
536 struct adapter *sc;
537
538 sc = device_get_softc(dev);
539
540 cxgb_free(sc);
541
542 return (0);
543}
544
545static void
546cxgb_free(struct adapter *sc)
547{
548 int i;
549
550 ADAPTER_LOCK(sc);
551 /*
552 * drops the lock
553 */
554 cxgb_down_locked(sc);
555
556#ifdef MSI_SUPPORTED
557 if (sc->flags & (USING_MSI | USING_MSIX)) {
558 device_printf(sc->dev, "releasing msi message(s)\n");
559 pci_release_msi(sc->dev);
560 } else {
561 device_printf(sc->dev, "no msi message to release\n");
562 }
563#endif
564 if (sc->msix_regs_res != NULL) {
565 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
566 sc->msix_regs_res);
567 }
568
569 t3_sge_deinit_sw(sc);
570
571 if (sc->tq != NULL) {
572 taskqueue_drain(sc->tq, &sc->ext_intr_task);
573 taskqueue_drain(sc->tq, &sc->tick_task);
574 taskqueue_free(sc->tq);
575 }
576
577 tsleep(&sc, 0, "cxgb unload", hz);
578
579 for (i = 0; i < (sc)->params.nports; ++i) {
580 if (sc->portdev[i] != NULL)
581 device_delete_child(sc->dev, sc->portdev[i]);
582 }
583
584 bus_generic_detach(sc->dev);
585#ifdef notyet
586 if (is_offload(sc)) {
587 cxgb_adapter_unofld(sc);
588 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
589 offload_close(&sc->tdev);
590 }
591#endif
592 t3_free_sge_resources(sc);
593 t3_sge_free(sc);
594
595 cxgb_offload_exit();
596
597 if (sc->regs_res != NULL)
598 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
599 sc->regs_res);
600
601 MTX_DESTROY(&sc->mdio_lock);
602 MTX_DESTROY(&sc->sge.reg_lock);
603 MTX_DESTROY(&sc->elmer_lock);
604 ADAPTER_LOCK_DEINIT(sc);
605
606 return;
607}
608
609/**
610 * setup_sge_qsets - configure SGE Tx/Rx/response queues
611 * @sc: the controller softc
612 *
613 * Determines how many sets of SGE queues to use and initializes them.
614 * We support multiple queue sets per port if we have MSI-X, otherwise
615 * just one queue set per port.
616 */
617static int
618setup_sge_qsets(adapter_t *sc)
619{
620 int i, j, err, irq_idx, qset_idx;
621 u_int ntxq = SGE_TXQ_PER_SET;
622
623 if ((err = t3_sge_alloc(sc)) != 0) {
624 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
625 return (err);
626 }
627
628 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
629 irq_idx = -1;
630 else
631 irq_idx = 0;
632
633 for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) {
634 struct port_info *pi = &sc->port[i];
635
636 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
637 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
638 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
639 &sc->params.sge.qset[qset_idx], ntxq, pi);
640 if (err) {
641 t3_free_sge_resources(sc);
642 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err);
643 return (err);
644 }
645 }
646 }
647
648 return (0);
649}
650
651static void
652cxgb_teardown_msix(adapter_t *sc)
653{
654 int i, nqsets;
655
656 for (nqsets = i = 0; i < (sc)->params.nports; i++)
657 nqsets += sc->port[i].nqsets;
658
659 for (i = 0; i < nqsets; i++) {
660 if (sc->msix_intr_tag[i] != NULL) {
661 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
662 sc->msix_intr_tag[i]);
663 sc->msix_intr_tag[i] = NULL;
664 }
665 if (sc->msix_irq_res[i] != NULL) {
666 bus_release_resource(sc->dev, SYS_RES_IRQ,
667 sc->msix_irq_rid[i], sc->msix_irq_res[i]);
668 sc->msix_irq_res[i] = NULL;
669 }
670 }
671}
672
673static int
674cxgb_setup_msix(adapter_t *sc, int msix_count)
675{
676 int i, j, k, nqsets, rid;
677
678 /* The first message indicates link changes and error conditions */
679 sc->irq_rid = 1;
680 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
681 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
682 device_printf(sc->dev, "Cannot allocate msix interrupt\n");
683 return (EINVAL);
684 }
685
686 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
687#ifdef INTR_FILTERS
688 NULL,
689#endif
690 cxgb_async_intr, sc, &sc->intr_tag)) {
691 device_printf(sc->dev, "Cannot set up interrupt\n");
692 return (EINVAL);
693 }
694 for (i = k = 0; i < (sc)->params.nports; i++) {
695 nqsets = sc->port[i].nqsets;
696 for (j = 0; j < nqsets; j++, k++) {
697 struct sge_qset *qs = &sc->sge.qs[k];
698
699 rid = k + 2;
700 if (cxgb_debug)
701 printf("rid=%d ", rid);
702 if ((sc->msix_irq_res[k] = bus_alloc_resource_any(
703 sc->dev, SYS_RES_IRQ, &rid,
704 RF_SHAREABLE | RF_ACTIVE)) == NULL) {
705 device_printf(sc->dev, "Cannot allocate "
706 "interrupt for message %d\n", rid);
707 return (EINVAL);
708 }
709 sc->msix_irq_rid[k] = rid;
710 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
711 INTR_MPSAFE|INTR_TYPE_NET,
712#ifdef INTR_FILTERS
713 NULL,
714#endif
715 t3_intr_msix, qs, &sc->msix_intr_tag[k])) {
716 device_printf(sc->dev, "Cannot set up "
717 "interrupt for message %d\n", rid);
718 return (EINVAL);
719 }
720 }
721 }
722
723
724 return (0);
725}
726
727static int
728cxgb_port_probe(device_t dev)
729{
730 struct port_info *p;
731 char buf[80];
732
733 p = device_get_softc(dev);
734
735 snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc);
736 device_set_desc_copy(dev, buf);
737 return (0);
738}
739
740
741static int
742cxgb_makedev(struct port_info *pi)
743{
744
745 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
746 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
747
748 if (pi->port_cdev == NULL)
749 return (ENOMEM);
750
751 pi->port_cdev->si_drv1 = (void *)pi;
752
753 return (0);
754}
755
756
757#ifdef TSO_SUPPORTED
758#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU)
759/* Don't enable TSO6 yet */
760#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU)
761#else
762#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
763/* Don't enable TSO6 yet */
764#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
765#define IFCAP_TSO4 0x0
766#define CSUM_TSO 0x0
767#endif
768
769
770static int
771cxgb_port_attach(device_t dev)
772{
773 struct port_info *p;
774 struct ifnet *ifp;
775 int err, media_flags;
776
777 p = device_get_softc(dev);
778
779 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
780 device_get_unit(device_get_parent(dev)), p->port);
781 PORT_LOCK_INIT(p, p->lockbuf);
782
783 /* Allocate an ifnet object and set it up */
784 ifp = p->ifp = if_alloc(IFT_ETHER);
785 if (ifp == NULL) {
786 device_printf(dev, "Cannot allocate ifnet\n");
787 return (ENOMEM);
788 }
789
790 /*
791 * Note that there is currently no watchdog timer.
792 */
793 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
794 ifp->if_init = cxgb_init;
795 ifp->if_softc = p;
796 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
797 ifp->if_ioctl = cxgb_ioctl;
798 ifp->if_start = cxgb_start;
799 ifp->if_timer = 0; /* Disable ifnet watchdog */
800 ifp->if_watchdog = NULL;
801
802 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE;
803 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
804 IFQ_SET_READY(&ifp->if_snd);
805
806 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
807 ifp->if_capabilities |= CXGB_CAP;
808 ifp->if_capenable |= CXGB_CAP_ENABLE;
809 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
810
811 ether_ifattach(ifp, p->hw_addr);
812#ifdef DEFAULT_JUMBO
813 ifp->if_mtu = 9000;
814#endif
815 if ((err = cxgb_makedev(p)) != 0) {
816 printf("makedev failed %d\n", err);
817 return (err);
818 }
819 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
820 cxgb_media_status);
821
822 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) {
823 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
824 } else if (!strcmp(p->port_type->desc, "10GBASE-SR")) {
825 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
826 } else if (!strcmp(p->port_type->desc, "10GBASE-XR")) {
827 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
828 } else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) {
829 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
830 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
831 0, NULL);
832 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX,
833 0, NULL);
834 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
835 0, NULL);
836 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
837 0, NULL);
838 media_flags = 0;
839 } else {
840 printf("unsupported media type %s\n", p->port_type->desc);
841 return (ENXIO);
842 }
843 if (media_flags) {
844 ifmedia_add(&p->media, media_flags, 0, NULL);
845 ifmedia_set(&p->media, media_flags);
846 } else {
847 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
848 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO);
849 }
850
851
852 snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port);
853#ifdef TASKQUEUE_CURRENT
854 /* Create a port for handling TX without starvation */
855 p->tq = taskqueue_create(p->taskqbuf, M_NOWAIT,
856 taskqueue_thread_enqueue, &p->tq);
857#else
858 /* Create a port for handling TX without starvation */
859 p->tq = taskqueue_create_fast(buf, M_NOWAIT,
860 taskqueue_thread_enqueue, &p->tq);
861#endif
862
863 if (p->tq == NULL) {
864 device_printf(dev, "failed to allocate port task queue\n");
865 return (ENOMEM);
866 }
867 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq",
868 device_get_nameunit(dev));
869 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp);
870
871 t3_sge_init_port(p);
872
873 return (0);
874}
875
876static int
877cxgb_port_detach(device_t dev)
878{
879 struct port_info *p;
880
881 p = device_get_softc(dev);
882
883 PORT_LOCK(p);
884 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING)
885 cxgb_stop_locked(p);
886 PORT_UNLOCK(p);
887
888 if (p->tq != NULL) {
889 taskqueue_drain(p->tq, &p->start_task);
890 taskqueue_free(p->tq);
891 p->tq = NULL;
892 }
893
894 PORT_LOCK_DEINIT(p);
895 ether_ifdetach(p->ifp);
896 if_free(p->ifp);
897
898 if (p->port_cdev != NULL)
899 destroy_dev(p->port_cdev);
900
901 return (0);
902}
903
904void
905t3_fatal_err(struct adapter *sc)
906{
907 u_int fw_status[4];
908
909 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
910 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
911 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
912 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
913}
914
915int
916t3_os_find_pci_capability(adapter_t *sc, int cap)
917{
918 device_t dev;
919 struct pci_devinfo *dinfo;
920 pcicfgregs *cfg;
921 uint32_t status;
922 uint8_t ptr;
923
924 dev = sc->dev;
925 dinfo = device_get_ivars(dev);
926 cfg = &dinfo->cfg;
927
928 status = pci_read_config(dev, PCIR_STATUS, 2);
929 if (!(status & PCIM_STATUS_CAPPRESENT))
930 return (0);
931
932 switch (cfg->hdrtype & PCIM_HDRTYPE) {
933 case 0:
934 case 1:
935 ptr = PCIR_CAP_PTR;
936 break;
937 case 2:
938 ptr = PCIR_CAP_PTR_2;
939 break;
940 default:
941 return (0);
942 break;
943 }
944 ptr = pci_read_config(dev, ptr, 1);
945
946 while (ptr != 0) {
947 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
948 return (ptr);
949 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
950 }
951
952 return (0);
953}
954
955int
956t3_os_pci_save_state(struct adapter *sc)
957{
958 device_t dev;
959 struct pci_devinfo *dinfo;
960
961 dev = sc->dev;
962 dinfo = device_get_ivars(dev);
963
964 pci_cfg_save(dev, dinfo, 0);
965 return (0);
966}
967
968int
969t3_os_pci_restore_state(struct adapter *sc)
970{
971 device_t dev;
972 struct pci_devinfo *dinfo;
973
974 dev = sc->dev;
975 dinfo = device_get_ivars(dev);
976
977 pci_cfg_restore(dev, dinfo);
978 return (0);
979}
980
981/**
982 * t3_os_link_changed - handle link status changes
983 * @adapter: the adapter associated with the link change
984 * @port_id: the port index whose limk status has changed
985 * @link_stat: the new status of the link
986 * @speed: the new speed setting
987 * @duplex: the new duplex setting
988 * @fc: the new flow-control setting
989 *
990 * This is the OS-dependent handler for link status changes. The OS
991 * neutral handler takes care of most of the processing for these events,
992 * then calls this handler for any OS-specific processing.
993 */
994void
995t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
996 int duplex, int fc)
997{
998 struct port_info *pi = &adapter->port[port_id];
999 struct cmac *mac = &adapter->port[port_id].mac;
1000
1001 if ((pi->ifp->if_flags & IFF_UP) == 0)
1002 return;
1003
1004 if (link_status) {
1005 t3_mac_enable(mac, MAC_DIRECTION_RX);
1006 if_link_state_change(pi->ifp, LINK_STATE_UP);
1007 } else {
1008 if_link_state_change(pi->ifp, LINK_STATE_DOWN);
1009 pi->phy.ops->power_down(&pi->phy, 1);
1010 t3_mac_disable(mac, MAC_DIRECTION_RX);
1011 t3_link_start(&pi->phy, mac, &pi->link_config);
1012 }
1013}
1014
1015
1016/*
1017 * Interrupt-context handler for external (PHY) interrupts.
1018 */
1019void
1020t3_os_ext_intr_handler(adapter_t *sc)
1021{
1022 if (cxgb_debug)
1023 printf("t3_os_ext_intr_handler\n");
1024 /*
1025 * Schedule a task to handle external interrupts as they may be slow
1026 * and we use a mutex to protect MDIO registers. We disable PHY
1027 * interrupts in the meantime and let the task reenable them when
1028 * it's done.
1029 */
1030 ADAPTER_LOCK(sc);
1031 if (sc->slow_intr_mask) {
1032 sc->slow_intr_mask &= ~F_T3DBG;
1033 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1034 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1035 }
1036 ADAPTER_UNLOCK(sc);
1037}
1038
1039void
1040t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1041{
1042
1043 /*
1044 * The ifnet might not be allocated before this gets called,
1045 * as this is called early on in attach by t3_prep_adapter
1046 * save the address off in the port structure
1047 */
1048 if (cxgb_debug)
1049 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1050 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1051}
1052
1053/**
1054 * link_start - enable a port
1055 * @p: the port to enable
1056 *
1057 * Performs the MAC and PHY actions needed to enable a port.
1058 */
1059static void
1060cxgb_link_start(struct port_info *p)
1061{
1062 struct ifnet *ifp;
1063 struct t3_rx_mode rm;
1064 struct cmac *mac = &p->mac;
1065
1066 ifp = p->ifp;
1067
1068 t3_init_rx_mode(&rm, p);
1069 t3_mac_reset(mac);
1070 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1071 t3_mac_set_address(mac, 0, p->hw_addr);
1072 t3_mac_set_rx_mode(mac, &rm);
1073 t3_link_start(&p->phy, mac, &p->link_config);
1074 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1075}
1076
1077/**
1078 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1079 * @adap: the adapter
1080 *
1081 * Sets up RSS to distribute packets to multiple receive queues. We
1082 * configure the RSS CPU lookup table to distribute to the number of HW
1083 * receive queues, and the response queue lookup table to narrow that
1084 * down to the response queues actually configured for each port.
1085 * We always configure the RSS mapping for two ports since the mapping
1086 * table has plenty of entries.
1087 */
1088static void
1089setup_rss(adapter_t *adap)
1090{
1091 int i;
1092 u_int nq0 = adap->port[0].nqsets;
1093 u_int nq1 = max((u_int)adap->port[1].nqsets, 1U);
1094 uint8_t cpus[SGE_QSETS + 1];
1095 uint16_t rspq_map[RSS_TABLE_SIZE];
1096
1097 for (i = 0; i < SGE_QSETS; ++i)
1098 cpus[i] = i;
1099 cpus[SGE_QSETS] = 0xff;
1100
1101 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1102 rspq_map[i] = i % nq0;
1103 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
1104 }
1105
1106 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1107 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
1108 V_RRCPLCPUSIZE(6), cpus, rspq_map);
1109}
1110
1111/*
1112 * Sends an mbuf to an offload queue driver
1113 * after dealing with any active network taps.
1114 */
1115static inline int
1116offload_tx(struct toedev *tdev, struct mbuf *m)
1117{
1118 int ret;
1119
1120 critical_enter();
1121 ret = t3_offload_tx(tdev, m);
1122 critical_exit();
1123 return (ret);
1124}
1125
1126static int
1127write_smt_entry(struct adapter *adapter, int idx)
1128{
1129 struct port_info *pi = &adapter->port[idx];
1130 struct cpl_smt_write_req *req;
1131 struct mbuf *m;
1132
1133 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1134 return (ENOMEM);
1135
1136 req = mtod(m, struct cpl_smt_write_req *);
1137 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1138 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1139 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
1140 req->iff = idx;
1141 memset(req->src_mac1, 0, sizeof(req->src_mac1));
1142 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1143
1144 m_set_priority(m, 1);
1145
1146 offload_tx(&adapter->tdev, m);
1147
1148 return (0);
1149}
1150
1151static int
1152init_smt(struct adapter *adapter)
1153{
1154 int i;
1155
1156 for_each_port(adapter, i)
1157 write_smt_entry(adapter, i);
1158 return 0;
1159}
1160
1161static void
1162init_port_mtus(adapter_t *adapter)
1163{
1164 unsigned int mtus = adapter->port[0].ifp->if_mtu;
1165
1166 if (adapter->port[1].ifp)
1167 mtus |= adapter->port[1].ifp->if_mtu << 16;
1168 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1169}
1170
1171static void
1172send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1173 int hi, int port)
1174{
1175 struct mbuf *m;
1176 struct mngt_pktsched_wr *req;
1177
1178 m = m_gethdr(M_NOWAIT, MT_DATA);
1179 if (m) {
1180 req = mtod(m, struct mngt_pktsched_wr *);
1181 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1182 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1183 req->sched = sched;
1184 req->idx = qidx;
1185 req->min = lo;
1186 req->max = hi;
1187 req->binding = port;
1188 m->m_len = m->m_pkthdr.len = sizeof(*req);
1189 t3_mgmt_tx(adap, m);
1190 }
1191}
1192
1193static void
1194bind_qsets(adapter_t *sc)
1195{
1196 int i, j;
1197
1198 for (i = 0; i < (sc)->params.nports; ++i) {
1199 const struct port_info *pi = adap2pinfo(sc, i);
1200
1201 for (j = 0; j < pi->nqsets; ++j)
1202 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1203 -1, i);
1204 }
1205}
1206
1207/**
1208 * cxgb_up - enable the adapter
1209 * @adap: adapter being enabled
1210 *
1211 * Called when the first port is enabled, this function performs the
1212 * actions necessary to make an adapter operational, such as completing
1213 * the initialization of HW modules, and enabling interrupts.
1214 *
1215 */
1216static int
1217cxgb_up(struct adapter *sc)
1218{
1219 int err = 0;
1220
1221 if ((sc->flags & FULL_INIT_DONE) == 0) {
1222
1223 if ((sc->flags & FW_UPTODATE) == 0)
1224 err = upgrade_fw(sc);
1225
1226 if (err)
1227 goto out;
1228
1229 err = t3_init_hw(sc, 0);
1230 if (err)
1231 goto out;
1232
1233 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1234
1235 err = setup_sge_qsets(sc);
1236 if (err)
1237 goto out;
1238
1239 setup_rss(sc);
1240 sc->flags |= FULL_INIT_DONE;
1241 }
1242
1243 t3_intr_clear(sc);
1244
1245 /* If it's MSI or INTx, allocate a single interrupt for everything */
1246 if ((sc->flags & USING_MSIX) == 0) {
1247 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
1248 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
1249 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid);
1250 err = EINVAL;
1251 goto out;
1252 }
1253 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
1254
1255 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
1256#ifdef INTR_FILTERS
1257 NULL,
1258#endif
1259 sc->cxgb_intr, sc, &sc->intr_tag)) {
1260 device_printf(sc->dev, "Cannot set up interrupt\n");
1261 err = EINVAL;
1262 goto irq_err;
1263 }
1264 } else {
1265 cxgb_setup_msix(sc, sc->msi_count);
1266 }
1267
1268 t3_sge_start(sc);
1269 t3_intr_enable(sc);
1270
1271 if ((sc->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
1272 bind_qsets(sc);
1273 sc->flags |= QUEUES_BOUND;
1274out:
1275 return (err);
1276irq_err:
1277 CH_ERR(sc, "request_irq failed, err %d\n", err);
1278 goto out;
1279}
1280
1281
1282/*
1283 * Release resources when all the ports and offloading have been stopped.
1284 */
1285static void
1286cxgb_down_locked(struct adapter *sc)
1287{
1288 int i;
1289
1290 t3_sge_stop(sc);
1291 t3_intr_disable(sc);
1292
1293 if (sc->intr_tag != NULL) {
1294 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
1295 sc->intr_tag = NULL;
1296 }
1297 if (sc->irq_res != NULL) {
1298 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
1299 sc->irq_rid, sc->irq_res);
1300 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
1301 sc->irq_res);
1302 sc->irq_res = NULL;
1303 }
1304
1305 if (sc->flags & USING_MSIX)
1306 cxgb_teardown_msix(sc);
1307 ADAPTER_UNLOCK(sc);
1308
1309 callout_drain(&sc->cxgb_tick_ch);
1310 callout_drain(&sc->sge_timer_ch);
1311
1312 if (sc->tq != NULL)
1313 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1314 for (i = 0; i < sc->params.nports; i++)
1315 if (sc->port[i].tq != NULL)
1316 taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task);
1317
1318}
1319
1320static int
1321offload_open(struct port_info *pi)
1322{
1323 struct adapter *adapter = pi->adapter;
1324 struct toedev *tdev = TOEDEV(pi->ifp);
1325 int adap_up = adapter->open_device_map & PORT_MASK;
1326 int err = 0;
1327
1328 if (atomic_cmpset_int(&adapter->open_device_map,
1329 (adapter->open_device_map & ~OFFLOAD_DEVMAP_BIT),
1330 (adapter->open_device_map | OFFLOAD_DEVMAP_BIT)) == 0)
1331 return (0);
1332
1333 ADAPTER_LOCK(pi->adapter);
1334 if (!adap_up)
1335 err = cxgb_up(adapter);
1336 ADAPTER_UNLOCK(pi->adapter);
1337 if (err < 0)
1338 return (err);
1339
1340 t3_tp_set_offload_mode(adapter, 1);
1341 tdev->lldev = adapter->port[0].ifp;
1342 err = cxgb_offload_activate(adapter);
1343 if (err)
1344 goto out;
1345
1346 init_port_mtus(adapter);
1347 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1348 adapter->params.b_wnd,
1349 adapter->params.rev == 0 ?
1350 adapter->port[0].ifp->if_mtu : 0xffff);
1351 init_smt(adapter);
1352
1353 /* Call back all registered clients */
1354 cxgb_add_clients(tdev);
1355
1356out:
1357 /* restore them in case the offload module has changed them */
1358 if (err) {
1359 t3_tp_set_offload_mode(adapter, 0);
1360 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1361 cxgb_set_dummy_ops(tdev);
1362 }
1363 return (err);
1364}
1365#ifdef notyet
1366static int
1367offload_close(struct toedev *tdev)
1368{
1369 struct adapter *adapter = tdev2adap(tdev);
1370
1371 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1372 return (0);
1373
1374 /* Call back all registered clients */
1375 cxgb_remove_clients(tdev);
1376 tdev->lldev = NULL;
1377 cxgb_set_dummy_ops(tdev);
1378 t3_tp_set_offload_mode(adapter, 0);
1379 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1380
1381 if (!adapter->open_device_map)
1382 cxgb_down(adapter);
1383
1384 cxgb_offload_deactivate(adapter);
1385 return (0);
1386}
1387#endif
1388
1389static void
1390cxgb_init(void *arg)
1391{
1392 struct port_info *p = arg;
1393
1394 PORT_LOCK(p);
1395 cxgb_init_locked(p);
1396 PORT_UNLOCK(p);
1397}
1398
1399static void
1400cxgb_init_locked(struct port_info *p)
1401{
1402 struct ifnet *ifp;
1403 adapter_t *sc = p->adapter;
1404 int err;
1405
1406 PORT_LOCK_ASSERT_OWNED(p);
1407 ifp = p->ifp;
1408
1409 ADAPTER_LOCK(p->adapter);
1410 if ((sc->open_device_map == 0) && ((err = cxgb_up(sc)) < 0)) {
1411 ADAPTER_UNLOCK(p->adapter);
1412 cxgb_stop_locked(p);
1413 return;
1414 }
1415 if (p->adapter->open_device_map == 0) {
1416 t3_intr_clear(sc);
1417 t3_sge_init_adapter(sc);
1418 }
1419 setbit(&p->adapter->open_device_map, p->port);
1420 ADAPTER_UNLOCK(p->adapter);
1421
1422 if (is_offload(sc) && !ofld_disable) {
1423 err = offload_open(p);
1424 if (err)
1425 log(LOG_WARNING,
1426 "Could not initialize offload capabilities\n");
1427 }
1428 cxgb_link_start(p);
1429 t3_link_changed(sc, p->port);
1430 ifp->if_baudrate = p->link_config.speed * 1000000;
1431
1432 t3_port_intr_enable(sc, p->port);
1433
1434 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1435 cxgb_tick, sc);
1436
1437 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1438 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1439}
1440
1441static void
1442cxgb_set_rxmode(struct port_info *p)
1443{
1444 struct t3_rx_mode rm;
1445 struct cmac *mac = &p->mac;
1446
1447 PORT_LOCK_ASSERT_OWNED(p);
1448
1449 t3_init_rx_mode(&rm, p);
1450 t3_mac_set_rx_mode(mac, &rm);
1451}
1452
1453static void
1454cxgb_stop_locked(struct port_info *p)
1455{
1456 struct ifnet *ifp;
1457
1458 PORT_LOCK_ASSERT_OWNED(p);
1459 ADAPTER_LOCK_ASSERT_NOTOWNED(p->adapter);
1460
1461 ifp = p->ifp;
1462
1463 t3_port_intr_disable(p->adapter, p->port);
1464 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1465 p->phy.ops->power_down(&p->phy, 1);
1466 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1467
1468 ADAPTER_LOCK(p->adapter);
1469 clrbit(&p->adapter->open_device_map, p->port);
1470
1471
1472 if (p->adapter->open_device_map == 0) {
1473 cxgb_down_locked(p->adapter);
1474 } else
1475 ADAPTER_UNLOCK(p->adapter);
1476
1477}
1478
1479static int
1480cxgb_set_mtu(struct port_info *p, int mtu)
1481{
1482 struct ifnet *ifp = p->ifp;
1483 int error = 0;
1484
1485 if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO))
1486 error = EINVAL;
1487 else if (ifp->if_mtu != mtu) {
1488 PORT_LOCK(p);
1489 ifp->if_mtu = mtu;
1490 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1491 callout_stop(&p->adapter->cxgb_tick_ch);
1492 cxgb_stop_locked(p);
1493 cxgb_init_locked(p);
1494 }
1495 PORT_UNLOCK(p);
1496 }
1497 return (error);
1498}
1499
1500static int
1501cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1502{
1503 struct port_info *p = ifp->if_softc;
1504 struct ifaddr *ifa = (struct ifaddr *)data;
1505 struct ifreq *ifr = (struct ifreq *)data;
1506 int flags, error = 0;
1507 uint32_t mask;
1508
1509 /*
1510 * XXX need to check that we aren't in the middle of an unload
1511 */
1512 switch (command) {
1513 case SIOCSIFMTU:
1514 error = cxgb_set_mtu(p, ifr->ifr_mtu);
1515 break;
1516 case SIOCSIFADDR:
1517 case SIOCGIFADDR:
1518 PORT_LOCK(p);
1519 if (ifa->ifa_addr->sa_family == AF_INET) {
1520 ifp->if_flags |= IFF_UP;
1521 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1522 cxgb_init_locked(p);
1523 arp_ifinit(ifp, ifa);
1524 } else
1525 error = ether_ioctl(ifp, command, data);
1526 PORT_UNLOCK(p);
1527 break;
1528 case SIOCSIFFLAGS:
1529 callout_drain(&p->adapter->cxgb_tick_ch);
1530 PORT_LOCK(p);
1531 if (ifp->if_flags & IFF_UP) {
1532 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1533 flags = p->if_flags;
1534 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1535 ((ifp->if_flags ^ flags) & IFF_ALLMULTI))
1536 cxgb_set_rxmode(p);
1537 } else
1538 cxgb_init_locked(p);
1539 p->if_flags = ifp->if_flags;
1540 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1541 cxgb_stop_locked(p);
1542
1543 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1544 adapter_t *sc = p->adapter;
1545 callout_reset(&sc->cxgb_tick_ch,
1546 sc->params.stats_update_period * hz,
1547 cxgb_tick, sc);
1548 }
1549 PORT_UNLOCK(p);
1550 break;
1551 case SIOCSIFMEDIA:
1552 case SIOCGIFMEDIA:
1553 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
1554 break;
1555 case SIOCSIFCAP:
1556 PORT_LOCK(p);
1557 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1558 if (mask & IFCAP_TXCSUM) {
1559 if (IFCAP_TXCSUM & ifp->if_capenable) {
1560 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
1561 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
1562 | CSUM_TSO);
1563 } else {
1564 ifp->if_capenable |= IFCAP_TXCSUM;
1565 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1566 }
1567 } else if (mask & IFCAP_RXCSUM) {
1568 if (IFCAP_RXCSUM & ifp->if_capenable) {
1569 ifp->if_capenable &= ~IFCAP_RXCSUM;
1570 } else {
1571 ifp->if_capenable |= IFCAP_RXCSUM;
1572 }
1573 }
1574 if (mask & IFCAP_TSO4) {
1575 if (IFCAP_TSO4 & ifp->if_capenable) {
1576 ifp->if_capenable &= ~IFCAP_TSO4;
1577 ifp->if_hwassist &= ~CSUM_TSO;
1578 } else if (IFCAP_TXCSUM & ifp->if_capenable) {
1579 ifp->if_capenable |= IFCAP_TSO4;
1580 ifp->if_hwassist |= CSUM_TSO;
1581 } else {
1582 if (cxgb_debug)
1583 printf("cxgb requires tx checksum offload"
1584 " be enabled to use TSO\n");
1585 error = EINVAL;
1586 }
1587 }
1588 PORT_UNLOCK(p);
1589 break;
1590 default:
1591 error = ether_ioctl(ifp, command, data);
1592 break;
1593 }
1594 return (error);
1595}
1596
1597static int
1598cxgb_start_tx(struct ifnet *ifp, uint32_t txmax)
1599{
1600 struct sge_qset *qs;
1601 struct sge_txq *txq;
1602 struct port_info *p = ifp->if_softc;
1603 struct mbuf *m0, *m = NULL;
1604 int err, in_use_init;
1605
1606 if (!p->link_config.link_ok)
1607 return (ENXIO);
1608
1609 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1610 return (ENOBUFS);
1611
1612 qs = &p->adapter->sge.qs[p->first_qset];
1613 txq = &qs->txq[TXQ_ETH];
1614 err = 0;
1615
1616 if (txq->flags & TXQ_TRANSMITTING)
1617 return (EINPROGRESS);
1618
1619 mtx_lock(&txq->lock);
1620 txq->flags |= TXQ_TRANSMITTING;
1621 in_use_init = txq->in_use;
1622 while ((txq->in_use - in_use_init < txmax) &&
1623 (txq->size > txq->in_use + TX_MAX_DESC)) {
1624 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
1625 if (m == NULL)
1626 break;
1627 /*
1628 * Convert chain to M_IOVEC
1629 */
1630 KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early"));
1631 m0 = m;
1632#ifdef INVARIANTS
1633 /*
1634 * Clean up after net stack sloppiness
1635 * before calling m_sanity
1636 */
1637 m0 = m->m_next;
1638 while (m0) {
1639 m0->m_flags &= ~M_PKTHDR;
1640 m0 = m0->m_next;
1641 }
1642 m_sanity(m0, 0);
1643 m0 = m;
1644#endif
1645 if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES &&
1646 m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) {
1647 if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) {
1648 m = m0;
1649 m_collapse(m, TX_MAX_SEGS, &m0);
1650 } else
1651 break;
1652 }
1653 m = m0;
1654 if ((err = t3_encap(p, &m)) != 0)
1655 break;
1656 BPF_MTAP(ifp, m);
1657 }
1658 txq->flags &= ~TXQ_TRANSMITTING;
1659 mtx_unlock(&txq->lock);
1660
1661 if (__predict_false(err)) {
1662 if (err == ENOMEM) {
1663 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1664 IFQ_LOCK(&ifp->if_snd);
1665 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1666 IFQ_UNLOCK(&ifp->if_snd);
1667 }
1668 }
1669 if (err == 0 && m == NULL)
1670 err = ENOBUFS;
1671 else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) &&
1672 (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
1673 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1674 err = ENOSPC;
1675 }
1676 return (err);
1677}
1678
1679static void
1680cxgb_start_proc(void *arg, int ncount)
1681{
1682 struct ifnet *ifp = arg;
1683 struct port_info *pi = ifp->if_softc;
1684 struct sge_qset *qs;
1685 struct sge_txq *txq;
1686 int error;
1687
1688 qs = &pi->adapter->sge.qs[pi->first_qset];
1689 txq = &qs->txq[TXQ_ETH];
1690
1691 do {
1692 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1693 taskqueue_enqueue(pi->adapter->tq, &txq->qreclaim_task);
1694
1695 error = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1696 } while (error == 0);
1697}
1698
1699static void
1700cxgb_start(struct ifnet *ifp)
1701{
1702 struct port_info *pi = ifp->if_softc;
1703 struct sge_qset *qs;
1704 struct sge_txq *txq;
1705 int err;
1706
1707 qs = &pi->adapter->sge.qs[pi->first_qset];
1708 txq = &qs->txq[TXQ_ETH];
1709
1710 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1711 taskqueue_enqueue(pi->adapter->tq,
1712 &pi->timer_reclaim_task);
1713
1714 err = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1715
1716 if (err == 0)
1717 taskqueue_enqueue(pi->tq, &pi->start_task);
1718}
1719
1720
1721static int
1722cxgb_media_change(struct ifnet *ifp)
1723{
1724 if_printf(ifp, "media change not supported\n");
1725 return (ENXIO);
1726}
1727
1728static void
1729cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1730{
1731 struct port_info *p = ifp->if_softc;
1732
1733 ifmr->ifm_status = IFM_AVALID;
1734 ifmr->ifm_active = IFM_ETHER;
1735
1736 if (!p->link_config.link_ok)
1737 return;
1738
1739 ifmr->ifm_status |= IFM_ACTIVE;
1740
1741 switch (p->link_config.speed) {
1742 case 10:
1743 ifmr->ifm_active |= IFM_10_T;
1744 break;
1745 case 100:
1746 ifmr->ifm_active |= IFM_100_TX;
1747 break;
1748 case 1000:
1749 ifmr->ifm_active |= IFM_1000_T;
1750 break;
1751 }
1752
1753 if (p->link_config.duplex)
1754 ifmr->ifm_active |= IFM_FDX;
1755 else
1756 ifmr->ifm_active |= IFM_HDX;
1757}
1758
1759static void
1760cxgb_async_intr(void *data)
1761{
1762 adapter_t *sc = data;
1763
1764 if (cxgb_debug)
1765 device_printf(sc->dev, "cxgb_async_intr\n");
1766 /*
1767 * May need to sleep - defer to taskqueue
1768 */
1769 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
1770}
1771
1772static void
1773cxgb_ext_intr_handler(void *arg, int count)
1774{
1775 adapter_t *sc = (adapter_t *)arg;
1776
1777 if (cxgb_debug)
1778 printf("cxgb_ext_intr_handler\n");
1779
1780 t3_phy_intr_handler(sc);
1781
1782 /* Now reenable external interrupts */
1783 ADAPTER_LOCK(sc);
1784 if (sc->slow_intr_mask) {
1785 sc->slow_intr_mask |= F_T3DBG;
1786 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
1787 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1788 }
1789 ADAPTER_UNLOCK(sc);
1790}
1791
1792static void
1793check_link_status(adapter_t *sc)
1794{
1795 int i;
1796
1797 for (i = 0; i < (sc)->params.nports; ++i) {
1798 struct port_info *p = &sc->port[i];
1799
1800 if (!(p->port_type->caps & SUPPORTED_IRQ))
1801 t3_link_changed(sc, i);
1802 p->ifp->if_baudrate = p->link_config.speed * 1000000;
1803 }
1804}
1805
1806static void
1807check_t3b2_mac(struct adapter *adapter)
1808{
1809 int i;
1810
1811 for_each_port(adapter, i) {
1812 struct port_info *p = &adapter->port[i];
1813 struct ifnet *ifp = p->ifp;
1814 int status;
1815
1816 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1817 continue;
1818
1819 status = 0;
1820 PORT_LOCK(p);
1821 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
1822 status = t3b2_mac_watchdog_task(&p->mac);
1823 if (status == 1)
1824 p->mac.stats.num_toggled++;
1825 else if (status == 2) {
1826 struct cmac *mac = &p->mac;
1827
1828 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN
1829 + ETHER_VLAN_ENCAP_LEN);
1830 t3_mac_set_address(mac, 0, p->hw_addr);
1831 cxgb_set_rxmode(p);
1832 t3_link_start(&p->phy, mac, &p->link_config);
1833 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1834 t3_port_intr_enable(adapter, p->port);
1835 p->mac.stats.num_resets++;
1836 }
1837 PORT_UNLOCK(p);
1838 }
1839}
1840
1841static void
1842cxgb_tick(void *arg)
1843{
1844 adapter_t *sc = (adapter_t *)arg;
1845
1846 taskqueue_enqueue(sc->tq, &sc->tick_task);
1847
1848 if (sc->open_device_map != 0)
1849 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1850 cxgb_tick, sc);
1851}
1852
1853static void
1854cxgb_tick_handler(void *arg, int count)
1855{
1856 adapter_t *sc = (adapter_t *)arg;
1857 const struct adapter_params *p = &sc->params;
1858
1859 ADAPTER_LOCK(sc);
1860 if (p->linkpoll_period)
1861 check_link_status(sc);
1862
1863 /*
1864 * adapter lock can currently only be acquire after the
1865 * port lock
1866 */
1867 ADAPTER_UNLOCK(sc);
1868
1869 if (p->rev == T3_REV_B2)
1870 check_t3b2_mac(sc);
1871}
1872
1873static int
1874in_range(int val, int lo, int hi)
1875{
1876 return val < 0 || (val <= hi && val >= lo);
1877}
1878
1879static int
1880cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td)
1881{
1882 return (0);
1883}
1884
1885static int
1886cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
1887{
1888 return (0);
1889}
1890
1891static int
1892cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
1893 int fflag, struct thread *td)
1894{
1895 int mmd, error = 0;
1896 struct port_info *pi = dev->si_drv1;
1897 adapter_t *sc = pi->adapter;
1898
1899#ifdef PRIV_SUPPORTED
1900 if (priv_check(td, PRIV_DRIVER)) {
1901 if (cxgb_debug)
1902 printf("user does not have access to privileged ioctls\n");
1903 return (EPERM);
1904 }
1905#else
1906 if (suser(td)) {
1907 if (cxgb_debug)
1908 printf("user does not have access to privileged ioctls\n");
1909 return (EPERM);
1910 }
1911#endif
1912
1913 switch (cmd) {
1914 case SIOCGMIIREG: {
1915 uint32_t val;
1916 struct cphy *phy = &pi->phy;
1917 struct mii_data *mid = (struct mii_data *)data;
1918
1919 if (!phy->mdio_read)
1920 return (EOPNOTSUPP);
1921 if (is_10G(sc)) {
1922 mmd = mid->phy_id >> 8;
1923 if (!mmd)
1924 mmd = MDIO_DEV_PCS;
1925 else if (mmd > MDIO_DEV_XGXS)
1926 return -EINVAL;
1927
1928 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
1929 mid->reg_num, &val);
1930 } else
1931 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
1932 mid->reg_num & 0x1f, &val);
1933 if (error == 0)
1934 mid->val_out = val;
1935 break;
1936 }
1937 case SIOCSMIIREG: {
1938 struct cphy *phy = &pi->phy;
1939 struct mii_data *mid = (struct mii_data *)data;
1940
1941 if (!phy->mdio_write)
1942 return (EOPNOTSUPP);
1943 if (is_10G(sc)) {
1944 mmd = mid->phy_id >> 8;
1945 if (!mmd)
1946 mmd = MDIO_DEV_PCS;
1947 else if (mmd > MDIO_DEV_XGXS)
1948 return (EINVAL);
1949
1950 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
1951 mmd, mid->reg_num, mid->val_in);
1952 } else
1953 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
1954 mid->reg_num & 0x1f,
1955 mid->val_in);
1956 break;
1957 }
1958 case CHELSIO_SETREG: {
1959 struct ch_reg *edata = (struct ch_reg *)data;
1960 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1961 return (EFAULT);
1962 t3_write_reg(sc, edata->addr, edata->val);
1963 break;
1964 }
1965 case CHELSIO_GETREG: {
1966 struct ch_reg *edata = (struct ch_reg *)data;
1967 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1968 return (EFAULT);
1969 edata->val = t3_read_reg(sc, edata->addr);
1970 break;
1971 }
1972 case CHELSIO_GET_SGE_CONTEXT: {
1973 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
1974 mtx_lock(&sc->sge.reg_lock);
1975 switch (ecntxt->cntxt_type) {
1976 case CNTXT_TYPE_EGRESS:
1977 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
1978 ecntxt->data);
1979 break;
1980 case CNTXT_TYPE_FL:
1981 error = t3_sge_read_fl(sc, ecntxt->cntxt_id,
1982 ecntxt->data);
1983 break;
1984 case CNTXT_TYPE_RSP:
1985 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id,
1986 ecntxt->data);
1987 break;
1988 case CNTXT_TYPE_CQ:
1989 error = t3_sge_read_cq(sc, ecntxt->cntxt_id,
1990 ecntxt->data);
1991 break;
1992 default:
1993 error = EINVAL;
1994 break;
1995 }
1996 mtx_unlock(&sc->sge.reg_lock);
1997 break;
1998 }
1999 case CHELSIO_GET_SGE_DESC: {
2000 struct ch_desc *edesc = (struct ch_desc *)data;
2001 int ret;
2002 if (edesc->queue_num >= SGE_QSETS * 6)
2003 return (EINVAL);
2004 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2005 edesc->queue_num % 6, edesc->idx, edesc->data);
2006 if (ret < 0)
2007 return (EINVAL);
2008 edesc->size = ret;
2009 break;
2010 }
2011 case CHELSIO_SET_QSET_PARAMS: {
2012 struct qset_params *q;
2013 struct ch_qset_params *t = (struct ch_qset_params *)data;
2014
2015 if (t->qset_idx >= SGE_QSETS)
2016 return -EINVAL;
2017 if (!in_range(t->intr_lat, 0, M_NEWTIMER) ||
2018 !in_range(t->cong_thres, 0, 255) ||
2019 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES,
2020 MAX_TXQ_ENTRIES) ||
2021 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES,
2022 MAX_TXQ_ENTRIES) ||
2023 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2024 MAX_CTRL_TXQ_ENTRIES) ||
2025 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) ||
2026 !in_range(t->fl_size[1], MIN_FL_ENTRIES,
2027 MAX_RX_JUMBO_BUFFERS) ||
2028 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
2029 return -EINVAL;
2030 if ((sc->flags & FULL_INIT_DONE) &&
2031 (t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
2032 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||
2033 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 ||
2034 t->polling >= 0 || t->cong_thres >= 0))
2035 return -EBUSY;
2036
2037 q = &sc->params.sge.qset[t->qset_idx];
2038
2039 if (t->rspq_size >= 0)
2040 q->rspq_size = t->rspq_size;
2041 if (t->fl_size[0] >= 0)
2042 q->fl_size = t->fl_size[0];
2043 if (t->fl_size[1] >= 0)
2044 q->jumbo_size = t->fl_size[1];
2045 if (t->txq_size[0] >= 0)
2046 q->txq_size[0] = t->txq_size[0];
2047 if (t->txq_size[1] >= 0)
2048 q->txq_size[1] = t->txq_size[1];
2049 if (t->txq_size[2] >= 0)
2050 q->txq_size[2] = t->txq_size[2];
2051 if (t->cong_thres >= 0)
2052 q->cong_thres = t->cong_thres;
2053 if (t->intr_lat >= 0) {
2054 struct sge_qset *qs = &sc->sge.qs[t->qset_idx];
2055
2056 q->coalesce_nsecs = t->intr_lat*1000;
2057 t3_update_qset_coalesce(qs, q);
2058 }
2059 break;
2060 }
2061 case CHELSIO_GET_QSET_PARAMS: {
2062 struct qset_params *q;
2063 struct ch_qset_params *t = (struct ch_qset_params *)data;
2064
2065 if (t->qset_idx >= SGE_QSETS)
2066 return (EINVAL);
2067
2068 q = &(sc)->params.sge.qset[t->qset_idx];
2069 t->rspq_size = q->rspq_size;
2070 t->txq_size[0] = q->txq_size[0];
2071 t->txq_size[1] = q->txq_size[1];
2072 t->txq_size[2] = q->txq_size[2];
2073 t->fl_size[0] = q->fl_size;
2074 t->fl_size[1] = q->jumbo_size;
2075 t->polling = q->polling;
2076 t->intr_lat = q->coalesce_nsecs / 1000;
2077 t->cong_thres = q->cong_thres;
2078 break;
2079 }
2080 case CHELSIO_SET_QSET_NUM: {
2081 struct ch_reg *edata = (struct ch_reg *)data;
2082 unsigned int port_idx = pi->port;
2083
2084 if (sc->flags & FULL_INIT_DONE)
2085 return (EBUSY);
2086 if (edata->val < 1 ||
2087 (edata->val > 1 && !(sc->flags & USING_MSIX)))
2088 return (EINVAL);
2089 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS)
2090 return (EINVAL);
2091 sc->port[port_idx].nqsets = edata->val;
2092 sc->port[0].first_qset = 0;
2093 /*
2094 * XXX hardcode ourselves to 2 ports just like LEEENUX
2095 */
2096 sc->port[1].first_qset = sc->port[0].nqsets;
2097 break;
2098 }
2099 case CHELSIO_GET_QSET_NUM: {
2100 struct ch_reg *edata = (struct ch_reg *)data;
2101 edata->val = pi->nqsets;
2102 break;
2103 }
2104#ifdef notyet
2105 case CHELSIO_LOAD_FW:
2106 case CHELSIO_GET_PM:
2107 case CHELSIO_SET_PM:
2108 return (EOPNOTSUPP);
2109 break;
2110#endif
2111 case CHELSIO_SETMTUTAB: {
2112 struct ch_mtus *m = (struct ch_mtus *)data;
2113 int i;
2114
2115 if (!is_offload(sc))
2116 return (EOPNOTSUPP);
2117 if (offload_running(sc))
2118 return (EBUSY);
2119 if (m->nmtus != NMTUS)
2120 return (EINVAL);
2121 if (m->mtus[0] < 81) /* accommodate SACK */
2122 return (EINVAL);
2123
2124 /*
2125 * MTUs must be in ascending order
2126 */
2127 for (i = 1; i < NMTUS; ++i)
2128 if (m->mtus[i] < m->mtus[i - 1])
2129 return (EINVAL);
2130
2131 memcpy(sc->params.mtus, m->mtus,
2132 sizeof(sc->params.mtus));
2133 break;
2134 }
2135 case CHELSIO_GETMTUTAB: {
2136 struct ch_mtus *m = (struct ch_mtus *)data;
2137
2138 if (!is_offload(sc))
2139 return (EOPNOTSUPP);
2140
2141 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2142 m->nmtus = NMTUS;
2143 break;
2144 }
2145 case CHELSIO_DEVUP:
2146 if (!is_offload(sc))
2147 return (EOPNOTSUPP);
2148 return offload_open(pi);
2149 break;
2150 case CHELSIO_GET_MEM: {
2151 struct ch_mem_range *t = (struct ch_mem_range *)data;
2152 struct mc7 *mem;
2153 uint8_t *useraddr;
2154 u64 buf[32];
2155
2156 if (!is_offload(sc))
2157 return (EOPNOTSUPP);
2158 if (!(sc->flags & FULL_INIT_DONE))
2159 return (EIO); /* need the memory controllers */
2160 if ((t->addr & 0x7) || (t->len & 0x7))
2161 return (EINVAL);
2162 if (t->mem_id == MEM_CM)
2163 mem = &sc->cm;
2164 else if (t->mem_id == MEM_PMRX)
2165 mem = &sc->pmrx;
2166 else if (t->mem_id == MEM_PMTX)
2167 mem = &sc->pmtx;
2168 else
2169 return (EINVAL);
2170
2171 /*
2172 * Version scheme:
2173 * bits 0..9: chip version
2174 * bits 10..15: chip revision
2175 */
2176 t->version = 3 | (sc->params.rev << 10);
2177
2178 /*
2179 * Read 256 bytes at a time as len can be large and we don't
2180 * want to use huge intermediate buffers.
2181 */
2182 useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */
2183 while (t->len) {
2184 unsigned int chunk = min(t->len, sizeof(buf));
2185
2186 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf);
2187 if (error)
2188 return (-error);
2189 if (copyout(buf, useraddr, chunk))
2190 return (EFAULT);
2191 useraddr += chunk;
2192 t->addr += chunk;
2193 t->len -= chunk;
2194 }
2195 break;
2196 }
2197 case CHELSIO_READ_TCAM_WORD: {
2198 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2199
2200 if (!is_offload(sc))
2201 return (EOPNOTSUPP);
2202 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2203 break;
2204 }
2205 case CHELSIO_SET_TRACE_FILTER: {
2206 struct ch_trace *t = (struct ch_trace *)data;
2207 const struct trace_params *tp;
2208
2209 tp = (const struct trace_params *)&t->sip;
2210 if (t->config_tx)
2211 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2212 t->trace_tx);
2213 if (t->config_rx)
2214 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2215 t->trace_rx);
2216 break;
2217 }
2218 case CHELSIO_SET_PKTSCHED: {
2219 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2220 if (sc->open_device_map == 0)
2221 return (EAGAIN);
2222 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2223 p->binding);
2224 break;
2225 }
2226 case CHELSIO_IFCONF_GETREGS: {
2227 struct ifconf_regs *regs = (struct ifconf_regs *)data;
2228 int reglen = cxgb_get_regs_len();
2229 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT);
2230 if (buf == NULL) {
2231 return (ENOMEM);
2232 } if (regs->len > reglen)
2233 regs->len = reglen;
2234 else if (regs->len < reglen) {
2235 error = E2BIG;
2236 goto done;
2237 }
2238 cxgb_get_regs(sc, regs, buf);
2239 error = copyout(buf, regs->data, reglen);
2240
2241 done:
2242 free(buf, M_DEVBUF);
2243
2244 break;
2245 }
2246 case CHELSIO_SET_HW_SCHED: {
2247 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2248 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2249
2250 if ((sc->flags & FULL_INIT_DONE) == 0)
2251 return (EAGAIN); /* need TP to be initialized */
2252 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2253 !in_range(t->channel, 0, 1) ||
2254 !in_range(t->kbps, 0, 10000000) ||
2255 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2256 !in_range(t->flow_ipg, 0,
2257 dack_ticks_to_usec(sc, 0x7ff)))
2258 return (EINVAL);
2259
2260 if (t->kbps >= 0) {
2261 error = t3_config_sched(sc, t->kbps, t->sched);
2262 if (error < 0)
2263 return (-error);
2264 }
2265 if (t->class_ipg >= 0)
2266 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2267 if (t->flow_ipg >= 0) {
2268 t->flow_ipg *= 1000; /* us -> ns */
2269 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2270 }
2271 if (t->mode >= 0) {
2272 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2273
2274 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2275 bit, t->mode ? bit : 0);
2276 }
2277 if (t->channel >= 0)
2278 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2279 1 << t->sched, t->channel << t->sched);
2280 break;
2281 }
2282 default:
2283 return (EOPNOTSUPP);
2284 break;
2285 }
2286
2287 return (error);
2288}
2289
2290static __inline void
2291reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
2292 unsigned int end)
2293{
2294 uint32_t *p = (uint32_t *)buf + start;
2295
2296 for ( ; start <= end; start += sizeof(uint32_t))
2297 *p++ = t3_read_reg(ap, start);
2298}
2299
2300#define T3_REGMAP_SIZE (3 * 1024)
2301static int
2302cxgb_get_regs_len(void)
2303{
2304 return T3_REGMAP_SIZE;
2305}
2306#undef T3_REGMAP_SIZE
2307
2308static void
2309cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf)
2310{
2311
2312 /*
2313 * Version scheme:
2314 * bits 0..9: chip version
2315 * bits 10..15: chip revision
2316 * bit 31: set for PCIe cards
2317 */
2318 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
2319
2320 /*
2321 * We skip the MAC statistics registers because they are clear-on-read.
2322 * Also reading multi-register stats would need to synchronize with the
2323 * periodic mac stats accumulation. Hard to justify the complexity.
2324 */
2325 memset(buf, 0, REGDUMP_SIZE);
2326 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
2327 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
2328 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
2329 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
2330 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
2331 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
2332 XGM_REG(A_XGM_SERDES_STAT3, 1));
2333 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
2334 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
2335}
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
122. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_main.c 171335 2007-07-10 06:01:45Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/bus.h>
37#include <sys/module.h>
38#include <sys/pciio.h>
39#include <sys/conf.h>
40#include <machine/bus.h>
41#include <machine/resource.h>
42#include <sys/bus_dma.h>
43#include <sys/rman.h>
44#include <sys/ioccom.h>
45#include <sys/mbuf.h>
46#include <sys/linker.h>
47#include <sys/firmware.h>
48#include <sys/socket.h>
49#include <sys/sockio.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <sys/queue.h>
53#include <sys/taskqueue.h>
54
55#include <net/bpf.h>
56#include <net/ethernet.h>
57#include <net/if.h>
58#include <net/if_arp.h>
59#include <net/if_dl.h>
60#include <net/if_media.h>
61#include <net/if_types.h>
62
63#include <netinet/in_systm.h>
64#include <netinet/in.h>
65#include <netinet/if_ether.h>
66#include <netinet/ip.h>
67#include <netinet/ip.h>
68#include <netinet/tcp.h>
69#include <netinet/udp.h>
70
71#include <dev/pci/pcireg.h>
72#include <dev/pci/pcivar.h>
73#include <dev/pci/pci_private.h>
74
75#ifdef CONFIG_DEFINED
76#include <cxgb_include.h>
77#else
78#include <dev/cxgb/cxgb_include.h>
79#endif
80
81#ifdef PRIV_SUPPORTED
82#include <sys/priv.h>
83#endif
84
85static int cxgb_setup_msix(adapter_t *, int);
86static void cxgb_teardown_msix(adapter_t *);
87static void cxgb_init(void *);
88static void cxgb_init_locked(struct port_info *);
89static void cxgb_stop_locked(struct port_info *);
90static void cxgb_set_rxmode(struct port_info *);
91static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
92static void cxgb_start(struct ifnet *);
93static void cxgb_start_proc(void *, int ncount);
94static int cxgb_media_change(struct ifnet *);
95static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
96static int setup_sge_qsets(adapter_t *);
97static void cxgb_async_intr(void *);
98static void cxgb_ext_intr_handler(void *, int);
99static void cxgb_tick_handler(void *, int);
100static void cxgb_down_locked(struct adapter *sc);
101static void cxgb_tick(void *);
102static void setup_rss(adapter_t *sc);
103
104/* Attachment glue for the PCI controller end of the device. Each port of
105 * the device is attached separately, as defined later.
106 */
107static int cxgb_controller_probe(device_t);
108static int cxgb_controller_attach(device_t);
109static int cxgb_controller_detach(device_t);
110static void cxgb_free(struct adapter *);
111static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
112 unsigned int end);
113static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf);
114static int cxgb_get_regs_len(void);
115static int offload_open(struct port_info *pi);
116#ifdef notyet
117static int offload_close(struct toedev *tdev);
118#endif
119
120
121static device_method_t cxgb_controller_methods[] = {
122 DEVMETHOD(device_probe, cxgb_controller_probe),
123 DEVMETHOD(device_attach, cxgb_controller_attach),
124 DEVMETHOD(device_detach, cxgb_controller_detach),
125
126 /* bus interface */
127 DEVMETHOD(bus_print_child, bus_generic_print_child),
128 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
129
130 { 0, 0 }
131};
132
133static driver_t cxgb_controller_driver = {
134 "cxgbc",
135 cxgb_controller_methods,
136 sizeof(struct adapter)
137};
138
139static devclass_t cxgb_controller_devclass;
140DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
141
142/*
143 * Attachment glue for the ports. Attachment is done directly to the
144 * controller device.
145 */
146static int cxgb_port_probe(device_t);
147static int cxgb_port_attach(device_t);
148static int cxgb_port_detach(device_t);
149
150static device_method_t cxgb_port_methods[] = {
151 DEVMETHOD(device_probe, cxgb_port_probe),
152 DEVMETHOD(device_attach, cxgb_port_attach),
153 DEVMETHOD(device_detach, cxgb_port_detach),
154 { 0, 0 }
155};
156
157static driver_t cxgb_port_driver = {
158 "cxgb",
159 cxgb_port_methods,
160 0
161};
162
163static d_ioctl_t cxgb_extension_ioctl;
164static d_open_t cxgb_extension_open;
165static d_close_t cxgb_extension_close;
166
167static struct cdevsw cxgb_cdevsw = {
168 .d_version = D_VERSION,
169 .d_flags = 0,
170 .d_open = cxgb_extension_open,
171 .d_close = cxgb_extension_close,
172 .d_ioctl = cxgb_extension_ioctl,
173 .d_name = "cxgb",
174};
175
176static devclass_t cxgb_port_devclass;
177DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
178
179#define SGE_MSIX_COUNT (SGE_QSETS + 1)
180
181extern int collapse_mbufs;
182/*
183 * The driver uses the best interrupt scheme available on a platform in the
184 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
185 * of these schemes the driver may consider as follows:
186 *
187 * msi = 2: choose from among all three options
188 * msi = 1 : only consider MSI and pin interrupts
189 * msi = 0: force pin interrupts
190 */
191static int msi_allowed = 2;
192
193TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
194SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
195SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
196 "MSI-X, MSI, INTx selector");
197
198/*
199 * The driver enables offload as a default.
200 * To disable it, use ofld_disable = 1.
201 */
202static int ofld_disable = 0;
203TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
204SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
205 "disable ULP offload");
206
207/*
208 * The driver uses an auto-queue algorithm by default.
209 * To disable it and force a single queue-set per port, use singleq = 1.
210 */
211static int singleq = 1;
212TUNABLE_INT("hw.cxgb.singleq", &singleq);
213SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
214 "use a single queue-set per port");
215
216enum {
217 MAX_TXQ_ENTRIES = 16384,
218 MAX_CTRL_TXQ_ENTRIES = 1024,
219 MAX_RSPQ_ENTRIES = 16384,
220 MAX_RX_BUFFERS = 16384,
221 MAX_RX_JUMBO_BUFFERS = 16384,
222 MIN_TXQ_ENTRIES = 4,
223 MIN_CTRL_TXQ_ENTRIES = 4,
224 MIN_RSPQ_ENTRIES = 32,
225 MIN_FL_ENTRIES = 32
226};
227
228#define PORT_MASK ((1 << MAX_NPORTS) - 1)
229
230/* Table for probing the cards. The desc field isn't actually used */
231struct cxgb_ident {
232 uint16_t vendor;
233 uint16_t device;
234 int index;
235 char *desc;
236} cxgb_identifiers[] = {
237 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
238 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
239 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
240 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
241 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
242 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
243 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
244 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
245 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
246 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
247 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
248 {0, 0, 0, NULL}
249};
250
251static struct cxgb_ident *
252cxgb_get_ident(device_t dev)
253{
254 struct cxgb_ident *id;
255
256 for (id = cxgb_identifiers; id->desc != NULL; id++) {
257 if ((id->vendor == pci_get_vendor(dev)) &&
258 (id->device == pci_get_device(dev))) {
259 return (id);
260 }
261 }
262 return (NULL);
263}
264
265static const struct adapter_info *
266cxgb_get_adapter_info(device_t dev)
267{
268 struct cxgb_ident *id;
269 const struct adapter_info *ai;
270
271 id = cxgb_get_ident(dev);
272 if (id == NULL)
273 return (NULL);
274
275 ai = t3_get_adapter_info(id->index);
276
277 return (ai);
278}
279
280static int
281cxgb_controller_probe(device_t dev)
282{
283 const struct adapter_info *ai;
284 char *ports, buf[80];
285 int nports;
286
287 ai = cxgb_get_adapter_info(dev);
288 if (ai == NULL)
289 return (ENXIO);
290
291 nports = ai->nports0 + ai->nports1;
292 if (nports == 1)
293 ports = "port";
294 else
295 ports = "ports";
296
297 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports);
298 device_set_desc_copy(dev, buf);
299 return (BUS_PROBE_DEFAULT);
300}
301
302static int
303upgrade_fw(adapter_t *sc)
304{
305 char buf[32];
306#ifdef FIRMWARE_LATEST
307 const struct firmware *fw;
308#else
309 struct firmware *fw;
310#endif
311 int status;
312
313 snprintf(&buf[0], sizeof(buf), "t3fw%d%d%d", FW_VERSION_MAJOR,
314 FW_VERSION_MINOR, FW_VERSION_MICRO);
315
316 fw = firmware_get(buf);
317
318 if (fw == NULL) {
319 device_printf(sc->dev, "Could not find firmware image %s\n", buf);
320 return (ENOENT);
321 }
322 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
323
324 firmware_put(fw, FIRMWARE_UNLOAD);
325
326 return (status);
327}
328
329static int
330cxgb_controller_attach(device_t dev)
331{
332 device_t child;
333 const struct adapter_info *ai;
334 struct adapter *sc;
335 int i, reg, msi_needed, error = 0;
336 uint32_t vers;
337 int port_qsets = 1;
338
339 sc = device_get_softc(dev);
340 sc->dev = dev;
341 sc->msi_count = 0;
342
343 /* find the PCIe link width and set max read request to 4KB*/
344 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
345 uint16_t lnk, pectl;
346 lnk = pci_read_config(dev, reg + 0x12, 2);
347 sc->link_width = (lnk >> 4) & 0x3f;
348
349 pectl = pci_read_config(dev, reg + 0x8, 2);
350 pectl = (pectl & ~0x7000) | (5 << 12);
351 pci_write_config(dev, reg + 0x8, pectl, 2);
352 }
353 if (sc->link_width != 0 && sc->link_width <= 4) {
354 device_printf(sc->dev,
355 "PCIe x%d Link, expect reduced performance\n",
356 sc->link_width);
357 }
358
359 pci_enable_busmaster(dev);
360 /*
361 * Allocate the registers and make them available to the driver.
362 * The registers that we care about for NIC mode are in BAR 0
363 */
364 sc->regs_rid = PCIR_BAR(0);
365 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
366 &sc->regs_rid, RF_ACTIVE)) == NULL) {
367 device_printf(dev, "Cannot allocate BAR\n");
368 return (ENXIO);
369 }
370
371 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
372 device_get_unit(dev));
373 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
374
375 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
376 device_get_unit(dev));
377 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
378 device_get_unit(dev));
379 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
380 device_get_unit(dev));
381
382 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_DEF);
383 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
384 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
385
386 sc->bt = rman_get_bustag(sc->regs_res);
387 sc->bh = rman_get_bushandle(sc->regs_res);
388 sc->mmio_len = rman_get_size(sc->regs_res);
389
390 ai = cxgb_get_adapter_info(dev);
391 if (t3_prep_adapter(sc, ai, 1) < 0) {
392 printf("prep adapter failed\n");
393 error = ENODEV;
394 goto out;
395 }
396 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
397 * enough messages for the queue sets. If that fails, try falling
398 * back to MSI. If that fails, then try falling back to the legacy
399 * interrupt pin model.
400 */
401#ifdef MSI_SUPPORTED
402
403 sc->msix_regs_rid = 0x20;
404 if ((msi_allowed >= 2) &&
405 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
406 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
407
408 msi_needed = sc->msi_count = SGE_MSIX_COUNT;
409
410 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
411 (sc->msi_count != msi_needed)) {
412 device_printf(dev, "msix allocation failed - msi_count = %d"
413 " msi_needed=%d will try msi err=%d\n", sc->msi_count,
414 msi_needed, error);
415 sc->msi_count = 0;
416 pci_release_msi(dev);
417 bus_release_resource(dev, SYS_RES_MEMORY,
418 sc->msix_regs_rid, sc->msix_regs_res);
419 sc->msix_regs_res = NULL;
420 } else {
421 sc->flags |= USING_MSIX;
422 sc->cxgb_intr = t3_intr_msix;
423 }
424 }
425
426 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
427 sc->msi_count = 1;
428 if (pci_alloc_msi(dev, &sc->msi_count)) {
429 device_printf(dev, "alloc msi failed - will try INTx\n");
430 sc->msi_count = 0;
431 pci_release_msi(dev);
432 } else {
433 sc->flags |= USING_MSI;
434 sc->irq_rid = 1;
435 sc->cxgb_intr = t3_intr_msi;
436 }
437 }
438#endif
439 if (sc->msi_count == 0) {
440 device_printf(dev, "using line interrupts\n");
441 sc->irq_rid = 0;
442 sc->cxgb_intr = t3b_intr;
443 }
444
445
446 /* Create a private taskqueue thread for handling driver events */
447#ifdef TASKQUEUE_CURRENT
448 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
449 taskqueue_thread_enqueue, &sc->tq);
450#else
451 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
452 taskqueue_thread_enqueue, &sc->tq);
453#endif
454 if (sc->tq == NULL) {
455 device_printf(dev, "failed to allocate controller task queue\n");
456 goto out;
457 }
458
459 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
460 device_get_nameunit(dev));
461 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
462 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
463
464
465 /* Create a periodic callout for checking adapter status */
466 callout_init(&sc->cxgb_tick_ch, TRUE);
467
468 if (t3_check_fw_version(sc) != 0) {
469 /*
470 * Warn user that a firmware update will be attempted in init.
471 */
472 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
473 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
474 sc->flags &= ~FW_UPTODATE;
475 } else {
476 sc->flags |= FW_UPTODATE;
477 }
478
479 if ((sc->flags & USING_MSIX) && !singleq)
480 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
481
482 /*
483 * Create a child device for each MAC. The ethernet attachment
484 * will be done in these children.
485 */
486 for (i = 0; i < (sc)->params.nports; i++) {
487 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
488 device_printf(dev, "failed to add child port\n");
489 error = EINVAL;
490 goto out;
491 }
492 sc->portdev[i] = child;
493 sc->port[i].adapter = sc;
494 sc->port[i].nqsets = port_qsets;
495 sc->port[i].first_qset = i*port_qsets;
496 sc->port[i].port = i;
497 device_set_softc(child, &sc->port[i]);
498 }
499 if ((error = bus_generic_attach(dev)) != 0)
500 goto out;
501
502 /*
503 * XXX need to poll for link status
504 */
505 sc->params.stats_update_period = 1;
506
507 /* initialize sge private state */
508 t3_sge_init_adapter(sc);
509
510 t3_led_ready(sc);
511
512 cxgb_offload_init();
513 if (is_offload(sc)) {
514 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
515 cxgb_adapter_ofld(sc);
516 }
517 error = t3_get_fw_version(sc, &vers);
518 if (error)
519 goto out;
520
521 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
522 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
523 G_FW_VERSION_MICRO(vers));
524
525 t3_add_sysctls(sc);
526out:
527 if (error)
528 cxgb_free(sc);
529
530 return (error);
531}
532
533static int
534cxgb_controller_detach(device_t dev)
535{
536 struct adapter *sc;
537
538 sc = device_get_softc(dev);
539
540 cxgb_free(sc);
541
542 return (0);
543}
544
545static void
546cxgb_free(struct adapter *sc)
547{
548 int i;
549
550 ADAPTER_LOCK(sc);
551 /*
552 * drops the lock
553 */
554 cxgb_down_locked(sc);
555
556#ifdef MSI_SUPPORTED
557 if (sc->flags & (USING_MSI | USING_MSIX)) {
558 device_printf(sc->dev, "releasing msi message(s)\n");
559 pci_release_msi(sc->dev);
560 } else {
561 device_printf(sc->dev, "no msi message to release\n");
562 }
563#endif
564 if (sc->msix_regs_res != NULL) {
565 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
566 sc->msix_regs_res);
567 }
568
569 t3_sge_deinit_sw(sc);
570
571 if (sc->tq != NULL) {
572 taskqueue_drain(sc->tq, &sc->ext_intr_task);
573 taskqueue_drain(sc->tq, &sc->tick_task);
574 taskqueue_free(sc->tq);
575 }
576
577 tsleep(&sc, 0, "cxgb unload", hz);
578
579 for (i = 0; i < (sc)->params.nports; ++i) {
580 if (sc->portdev[i] != NULL)
581 device_delete_child(sc->dev, sc->portdev[i]);
582 }
583
584 bus_generic_detach(sc->dev);
585#ifdef notyet
586 if (is_offload(sc)) {
587 cxgb_adapter_unofld(sc);
588 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
589 offload_close(&sc->tdev);
590 }
591#endif
592 t3_free_sge_resources(sc);
593 t3_sge_free(sc);
594
595 cxgb_offload_exit();
596
597 if (sc->regs_res != NULL)
598 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
599 sc->regs_res);
600
601 MTX_DESTROY(&sc->mdio_lock);
602 MTX_DESTROY(&sc->sge.reg_lock);
603 MTX_DESTROY(&sc->elmer_lock);
604 ADAPTER_LOCK_DEINIT(sc);
605
606 return;
607}
608
609/**
610 * setup_sge_qsets - configure SGE Tx/Rx/response queues
611 * @sc: the controller softc
612 *
613 * Determines how many sets of SGE queues to use and initializes them.
614 * We support multiple queue sets per port if we have MSI-X, otherwise
615 * just one queue set per port.
616 */
617static int
618setup_sge_qsets(adapter_t *sc)
619{
620 int i, j, err, irq_idx, qset_idx;
621 u_int ntxq = SGE_TXQ_PER_SET;
622
623 if ((err = t3_sge_alloc(sc)) != 0) {
624 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
625 return (err);
626 }
627
628 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
629 irq_idx = -1;
630 else
631 irq_idx = 0;
632
633 for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) {
634 struct port_info *pi = &sc->port[i];
635
636 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
637 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
638 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
639 &sc->params.sge.qset[qset_idx], ntxq, pi);
640 if (err) {
641 t3_free_sge_resources(sc);
642 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err);
643 return (err);
644 }
645 }
646 }
647
648 return (0);
649}
650
651static void
652cxgb_teardown_msix(adapter_t *sc)
653{
654 int i, nqsets;
655
656 for (nqsets = i = 0; i < (sc)->params.nports; i++)
657 nqsets += sc->port[i].nqsets;
658
659 for (i = 0; i < nqsets; i++) {
660 if (sc->msix_intr_tag[i] != NULL) {
661 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
662 sc->msix_intr_tag[i]);
663 sc->msix_intr_tag[i] = NULL;
664 }
665 if (sc->msix_irq_res[i] != NULL) {
666 bus_release_resource(sc->dev, SYS_RES_IRQ,
667 sc->msix_irq_rid[i], sc->msix_irq_res[i]);
668 sc->msix_irq_res[i] = NULL;
669 }
670 }
671}
672
673static int
674cxgb_setup_msix(adapter_t *sc, int msix_count)
675{
676 int i, j, k, nqsets, rid;
677
678 /* The first message indicates link changes and error conditions */
679 sc->irq_rid = 1;
680 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
681 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
682 device_printf(sc->dev, "Cannot allocate msix interrupt\n");
683 return (EINVAL);
684 }
685
686 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
687#ifdef INTR_FILTERS
688 NULL,
689#endif
690 cxgb_async_intr, sc, &sc->intr_tag)) {
691 device_printf(sc->dev, "Cannot set up interrupt\n");
692 return (EINVAL);
693 }
694 for (i = k = 0; i < (sc)->params.nports; i++) {
695 nqsets = sc->port[i].nqsets;
696 for (j = 0; j < nqsets; j++, k++) {
697 struct sge_qset *qs = &sc->sge.qs[k];
698
699 rid = k + 2;
700 if (cxgb_debug)
701 printf("rid=%d ", rid);
702 if ((sc->msix_irq_res[k] = bus_alloc_resource_any(
703 sc->dev, SYS_RES_IRQ, &rid,
704 RF_SHAREABLE | RF_ACTIVE)) == NULL) {
705 device_printf(sc->dev, "Cannot allocate "
706 "interrupt for message %d\n", rid);
707 return (EINVAL);
708 }
709 sc->msix_irq_rid[k] = rid;
710 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
711 INTR_MPSAFE|INTR_TYPE_NET,
712#ifdef INTR_FILTERS
713 NULL,
714#endif
715 t3_intr_msix, qs, &sc->msix_intr_tag[k])) {
716 device_printf(sc->dev, "Cannot set up "
717 "interrupt for message %d\n", rid);
718 return (EINVAL);
719 }
720 }
721 }
722
723
724 return (0);
725}
726
727static int
728cxgb_port_probe(device_t dev)
729{
730 struct port_info *p;
731 char buf[80];
732
733 p = device_get_softc(dev);
734
735 snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc);
736 device_set_desc_copy(dev, buf);
737 return (0);
738}
739
740
741static int
742cxgb_makedev(struct port_info *pi)
743{
744
745 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
746 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
747
748 if (pi->port_cdev == NULL)
749 return (ENOMEM);
750
751 pi->port_cdev->si_drv1 = (void *)pi;
752
753 return (0);
754}
755
756
757#ifdef TSO_SUPPORTED
758#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU)
759/* Don't enable TSO6 yet */
760#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU)
761#else
762#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
763/* Don't enable TSO6 yet */
764#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
765#define IFCAP_TSO4 0x0
766#define CSUM_TSO 0x0
767#endif
768
769
770static int
771cxgb_port_attach(device_t dev)
772{
773 struct port_info *p;
774 struct ifnet *ifp;
775 int err, media_flags;
776
777 p = device_get_softc(dev);
778
779 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
780 device_get_unit(device_get_parent(dev)), p->port);
781 PORT_LOCK_INIT(p, p->lockbuf);
782
783 /* Allocate an ifnet object and set it up */
784 ifp = p->ifp = if_alloc(IFT_ETHER);
785 if (ifp == NULL) {
786 device_printf(dev, "Cannot allocate ifnet\n");
787 return (ENOMEM);
788 }
789
790 /*
791 * Note that there is currently no watchdog timer.
792 */
793 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
794 ifp->if_init = cxgb_init;
795 ifp->if_softc = p;
796 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
797 ifp->if_ioctl = cxgb_ioctl;
798 ifp->if_start = cxgb_start;
799 ifp->if_timer = 0; /* Disable ifnet watchdog */
800 ifp->if_watchdog = NULL;
801
802 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE;
803 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
804 IFQ_SET_READY(&ifp->if_snd);
805
806 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
807 ifp->if_capabilities |= CXGB_CAP;
808 ifp->if_capenable |= CXGB_CAP_ENABLE;
809 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
810
811 ether_ifattach(ifp, p->hw_addr);
812#ifdef DEFAULT_JUMBO
813 ifp->if_mtu = 9000;
814#endif
815 if ((err = cxgb_makedev(p)) != 0) {
816 printf("makedev failed %d\n", err);
817 return (err);
818 }
819 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
820 cxgb_media_status);
821
822 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) {
823 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
824 } else if (!strcmp(p->port_type->desc, "10GBASE-SR")) {
825 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
826 } else if (!strcmp(p->port_type->desc, "10GBASE-XR")) {
827 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
828 } else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) {
829 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
830 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
831 0, NULL);
832 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX,
833 0, NULL);
834 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
835 0, NULL);
836 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
837 0, NULL);
838 media_flags = 0;
839 } else {
840 printf("unsupported media type %s\n", p->port_type->desc);
841 return (ENXIO);
842 }
843 if (media_flags) {
844 ifmedia_add(&p->media, media_flags, 0, NULL);
845 ifmedia_set(&p->media, media_flags);
846 } else {
847 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
848 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO);
849 }
850
851
852 snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port);
853#ifdef TASKQUEUE_CURRENT
854 /* Create a port for handling TX without starvation */
855 p->tq = taskqueue_create(p->taskqbuf, M_NOWAIT,
856 taskqueue_thread_enqueue, &p->tq);
857#else
858 /* Create a port for handling TX without starvation */
859 p->tq = taskqueue_create_fast(buf, M_NOWAIT,
860 taskqueue_thread_enqueue, &p->tq);
861#endif
862
863 if (p->tq == NULL) {
864 device_printf(dev, "failed to allocate port task queue\n");
865 return (ENOMEM);
866 }
867 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq",
868 device_get_nameunit(dev));
869 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp);
870
871 t3_sge_init_port(p);
872
873 return (0);
874}
875
876static int
877cxgb_port_detach(device_t dev)
878{
879 struct port_info *p;
880
881 p = device_get_softc(dev);
882
883 PORT_LOCK(p);
884 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING)
885 cxgb_stop_locked(p);
886 PORT_UNLOCK(p);
887
888 if (p->tq != NULL) {
889 taskqueue_drain(p->tq, &p->start_task);
890 taskqueue_free(p->tq);
891 p->tq = NULL;
892 }
893
894 PORT_LOCK_DEINIT(p);
895 ether_ifdetach(p->ifp);
896 if_free(p->ifp);
897
898 if (p->port_cdev != NULL)
899 destroy_dev(p->port_cdev);
900
901 return (0);
902}
903
904void
905t3_fatal_err(struct adapter *sc)
906{
907 u_int fw_status[4];
908
909 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
910 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
911 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
912 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
913}
914
915int
916t3_os_find_pci_capability(adapter_t *sc, int cap)
917{
918 device_t dev;
919 struct pci_devinfo *dinfo;
920 pcicfgregs *cfg;
921 uint32_t status;
922 uint8_t ptr;
923
924 dev = sc->dev;
925 dinfo = device_get_ivars(dev);
926 cfg = &dinfo->cfg;
927
928 status = pci_read_config(dev, PCIR_STATUS, 2);
929 if (!(status & PCIM_STATUS_CAPPRESENT))
930 return (0);
931
932 switch (cfg->hdrtype & PCIM_HDRTYPE) {
933 case 0:
934 case 1:
935 ptr = PCIR_CAP_PTR;
936 break;
937 case 2:
938 ptr = PCIR_CAP_PTR_2;
939 break;
940 default:
941 return (0);
942 break;
943 }
944 ptr = pci_read_config(dev, ptr, 1);
945
946 while (ptr != 0) {
947 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
948 return (ptr);
949 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
950 }
951
952 return (0);
953}
954
955int
956t3_os_pci_save_state(struct adapter *sc)
957{
958 device_t dev;
959 struct pci_devinfo *dinfo;
960
961 dev = sc->dev;
962 dinfo = device_get_ivars(dev);
963
964 pci_cfg_save(dev, dinfo, 0);
965 return (0);
966}
967
968int
969t3_os_pci_restore_state(struct adapter *sc)
970{
971 device_t dev;
972 struct pci_devinfo *dinfo;
973
974 dev = sc->dev;
975 dinfo = device_get_ivars(dev);
976
977 pci_cfg_restore(dev, dinfo);
978 return (0);
979}
980
981/**
982 * t3_os_link_changed - handle link status changes
983 * @adapter: the adapter associated with the link change
984 * @port_id: the port index whose limk status has changed
985 * @link_stat: the new status of the link
986 * @speed: the new speed setting
987 * @duplex: the new duplex setting
988 * @fc: the new flow-control setting
989 *
990 * This is the OS-dependent handler for link status changes. The OS
991 * neutral handler takes care of most of the processing for these events,
992 * then calls this handler for any OS-specific processing.
993 */
994void
995t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
996 int duplex, int fc)
997{
998 struct port_info *pi = &adapter->port[port_id];
999 struct cmac *mac = &adapter->port[port_id].mac;
1000
1001 if ((pi->ifp->if_flags & IFF_UP) == 0)
1002 return;
1003
1004 if (link_status) {
1005 t3_mac_enable(mac, MAC_DIRECTION_RX);
1006 if_link_state_change(pi->ifp, LINK_STATE_UP);
1007 } else {
1008 if_link_state_change(pi->ifp, LINK_STATE_DOWN);
1009 pi->phy.ops->power_down(&pi->phy, 1);
1010 t3_mac_disable(mac, MAC_DIRECTION_RX);
1011 t3_link_start(&pi->phy, mac, &pi->link_config);
1012 }
1013}
1014
1015
1016/*
1017 * Interrupt-context handler for external (PHY) interrupts.
1018 */
1019void
1020t3_os_ext_intr_handler(adapter_t *sc)
1021{
1022 if (cxgb_debug)
1023 printf("t3_os_ext_intr_handler\n");
1024 /*
1025 * Schedule a task to handle external interrupts as they may be slow
1026 * and we use a mutex to protect MDIO registers. We disable PHY
1027 * interrupts in the meantime and let the task reenable them when
1028 * it's done.
1029 */
1030 ADAPTER_LOCK(sc);
1031 if (sc->slow_intr_mask) {
1032 sc->slow_intr_mask &= ~F_T3DBG;
1033 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1034 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1035 }
1036 ADAPTER_UNLOCK(sc);
1037}
1038
1039void
1040t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1041{
1042
1043 /*
1044 * The ifnet might not be allocated before this gets called,
1045 * as this is called early on in attach by t3_prep_adapter
1046 * save the address off in the port structure
1047 */
1048 if (cxgb_debug)
1049 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1050 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1051}
1052
1053/**
1054 * link_start - enable a port
1055 * @p: the port to enable
1056 *
1057 * Performs the MAC and PHY actions needed to enable a port.
1058 */
1059static void
1060cxgb_link_start(struct port_info *p)
1061{
1062 struct ifnet *ifp;
1063 struct t3_rx_mode rm;
1064 struct cmac *mac = &p->mac;
1065
1066 ifp = p->ifp;
1067
1068 t3_init_rx_mode(&rm, p);
1069 t3_mac_reset(mac);
1070 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1071 t3_mac_set_address(mac, 0, p->hw_addr);
1072 t3_mac_set_rx_mode(mac, &rm);
1073 t3_link_start(&p->phy, mac, &p->link_config);
1074 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1075}
1076
1077/**
1078 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1079 * @adap: the adapter
1080 *
1081 * Sets up RSS to distribute packets to multiple receive queues. We
1082 * configure the RSS CPU lookup table to distribute to the number of HW
1083 * receive queues, and the response queue lookup table to narrow that
1084 * down to the response queues actually configured for each port.
1085 * We always configure the RSS mapping for two ports since the mapping
1086 * table has plenty of entries.
1087 */
1088static void
1089setup_rss(adapter_t *adap)
1090{
1091 int i;
1092 u_int nq0 = adap->port[0].nqsets;
1093 u_int nq1 = max((u_int)adap->port[1].nqsets, 1U);
1094 uint8_t cpus[SGE_QSETS + 1];
1095 uint16_t rspq_map[RSS_TABLE_SIZE];
1096
1097 for (i = 0; i < SGE_QSETS; ++i)
1098 cpus[i] = i;
1099 cpus[SGE_QSETS] = 0xff;
1100
1101 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1102 rspq_map[i] = i % nq0;
1103 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
1104 }
1105
1106 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1107 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
1108 V_RRCPLCPUSIZE(6), cpus, rspq_map);
1109}
1110
1111/*
1112 * Sends an mbuf to an offload queue driver
1113 * after dealing with any active network taps.
1114 */
1115static inline int
1116offload_tx(struct toedev *tdev, struct mbuf *m)
1117{
1118 int ret;
1119
1120 critical_enter();
1121 ret = t3_offload_tx(tdev, m);
1122 critical_exit();
1123 return (ret);
1124}
1125
1126static int
1127write_smt_entry(struct adapter *adapter, int idx)
1128{
1129 struct port_info *pi = &adapter->port[idx];
1130 struct cpl_smt_write_req *req;
1131 struct mbuf *m;
1132
1133 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1134 return (ENOMEM);
1135
1136 req = mtod(m, struct cpl_smt_write_req *);
1137 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1138 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1139 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
1140 req->iff = idx;
1141 memset(req->src_mac1, 0, sizeof(req->src_mac1));
1142 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1143
1144 m_set_priority(m, 1);
1145
1146 offload_tx(&adapter->tdev, m);
1147
1148 return (0);
1149}
1150
1151static int
1152init_smt(struct adapter *adapter)
1153{
1154 int i;
1155
1156 for_each_port(adapter, i)
1157 write_smt_entry(adapter, i);
1158 return 0;
1159}
1160
1161static void
1162init_port_mtus(adapter_t *adapter)
1163{
1164 unsigned int mtus = adapter->port[0].ifp->if_mtu;
1165
1166 if (adapter->port[1].ifp)
1167 mtus |= adapter->port[1].ifp->if_mtu << 16;
1168 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1169}
1170
1171static void
1172send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1173 int hi, int port)
1174{
1175 struct mbuf *m;
1176 struct mngt_pktsched_wr *req;
1177
1178 m = m_gethdr(M_NOWAIT, MT_DATA);
1179 if (m) {
1180 req = mtod(m, struct mngt_pktsched_wr *);
1181 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1182 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1183 req->sched = sched;
1184 req->idx = qidx;
1185 req->min = lo;
1186 req->max = hi;
1187 req->binding = port;
1188 m->m_len = m->m_pkthdr.len = sizeof(*req);
1189 t3_mgmt_tx(adap, m);
1190 }
1191}
1192
1193static void
1194bind_qsets(adapter_t *sc)
1195{
1196 int i, j;
1197
1198 for (i = 0; i < (sc)->params.nports; ++i) {
1199 const struct port_info *pi = adap2pinfo(sc, i);
1200
1201 for (j = 0; j < pi->nqsets; ++j)
1202 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1203 -1, i);
1204 }
1205}
1206
1207/**
1208 * cxgb_up - enable the adapter
1209 * @adap: adapter being enabled
1210 *
1211 * Called when the first port is enabled, this function performs the
1212 * actions necessary to make an adapter operational, such as completing
1213 * the initialization of HW modules, and enabling interrupts.
1214 *
1215 */
1216static int
1217cxgb_up(struct adapter *sc)
1218{
1219 int err = 0;
1220
1221 if ((sc->flags & FULL_INIT_DONE) == 0) {
1222
1223 if ((sc->flags & FW_UPTODATE) == 0)
1224 err = upgrade_fw(sc);
1225
1226 if (err)
1227 goto out;
1228
1229 err = t3_init_hw(sc, 0);
1230 if (err)
1231 goto out;
1232
1233 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1234
1235 err = setup_sge_qsets(sc);
1236 if (err)
1237 goto out;
1238
1239 setup_rss(sc);
1240 sc->flags |= FULL_INIT_DONE;
1241 }
1242
1243 t3_intr_clear(sc);
1244
1245 /* If it's MSI or INTx, allocate a single interrupt for everything */
1246 if ((sc->flags & USING_MSIX) == 0) {
1247 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
1248 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
1249 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid);
1250 err = EINVAL;
1251 goto out;
1252 }
1253 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
1254
1255 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
1256#ifdef INTR_FILTERS
1257 NULL,
1258#endif
1259 sc->cxgb_intr, sc, &sc->intr_tag)) {
1260 device_printf(sc->dev, "Cannot set up interrupt\n");
1261 err = EINVAL;
1262 goto irq_err;
1263 }
1264 } else {
1265 cxgb_setup_msix(sc, sc->msi_count);
1266 }
1267
1268 t3_sge_start(sc);
1269 t3_intr_enable(sc);
1270
1271 if ((sc->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
1272 bind_qsets(sc);
1273 sc->flags |= QUEUES_BOUND;
1274out:
1275 return (err);
1276irq_err:
1277 CH_ERR(sc, "request_irq failed, err %d\n", err);
1278 goto out;
1279}
1280
1281
1282/*
1283 * Release resources when all the ports and offloading have been stopped.
1284 */
1285static void
1286cxgb_down_locked(struct adapter *sc)
1287{
1288 int i;
1289
1290 t3_sge_stop(sc);
1291 t3_intr_disable(sc);
1292
1293 if (sc->intr_tag != NULL) {
1294 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
1295 sc->intr_tag = NULL;
1296 }
1297 if (sc->irq_res != NULL) {
1298 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
1299 sc->irq_rid, sc->irq_res);
1300 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
1301 sc->irq_res);
1302 sc->irq_res = NULL;
1303 }
1304
1305 if (sc->flags & USING_MSIX)
1306 cxgb_teardown_msix(sc);
1307 ADAPTER_UNLOCK(sc);
1308
1309 callout_drain(&sc->cxgb_tick_ch);
1310 callout_drain(&sc->sge_timer_ch);
1311
1312 if (sc->tq != NULL)
1313 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1314 for (i = 0; i < sc->params.nports; i++)
1315 if (sc->port[i].tq != NULL)
1316 taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task);
1317
1318}
1319
1320static int
1321offload_open(struct port_info *pi)
1322{
1323 struct adapter *adapter = pi->adapter;
1324 struct toedev *tdev = TOEDEV(pi->ifp);
1325 int adap_up = adapter->open_device_map & PORT_MASK;
1326 int err = 0;
1327
1328 if (atomic_cmpset_int(&adapter->open_device_map,
1329 (adapter->open_device_map & ~OFFLOAD_DEVMAP_BIT),
1330 (adapter->open_device_map | OFFLOAD_DEVMAP_BIT)) == 0)
1331 return (0);
1332
1333 ADAPTER_LOCK(pi->adapter);
1334 if (!adap_up)
1335 err = cxgb_up(adapter);
1336 ADAPTER_UNLOCK(pi->adapter);
1337 if (err < 0)
1338 return (err);
1339
1340 t3_tp_set_offload_mode(adapter, 1);
1341 tdev->lldev = adapter->port[0].ifp;
1342 err = cxgb_offload_activate(adapter);
1343 if (err)
1344 goto out;
1345
1346 init_port_mtus(adapter);
1347 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1348 adapter->params.b_wnd,
1349 adapter->params.rev == 0 ?
1350 adapter->port[0].ifp->if_mtu : 0xffff);
1351 init_smt(adapter);
1352
1353 /* Call back all registered clients */
1354 cxgb_add_clients(tdev);
1355
1356out:
1357 /* restore them in case the offload module has changed them */
1358 if (err) {
1359 t3_tp_set_offload_mode(adapter, 0);
1360 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1361 cxgb_set_dummy_ops(tdev);
1362 }
1363 return (err);
1364}
1365#ifdef notyet
1366static int
1367offload_close(struct toedev *tdev)
1368{
1369 struct adapter *adapter = tdev2adap(tdev);
1370
1371 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1372 return (0);
1373
1374 /* Call back all registered clients */
1375 cxgb_remove_clients(tdev);
1376 tdev->lldev = NULL;
1377 cxgb_set_dummy_ops(tdev);
1378 t3_tp_set_offload_mode(adapter, 0);
1379 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1380
1381 if (!adapter->open_device_map)
1382 cxgb_down(adapter);
1383
1384 cxgb_offload_deactivate(adapter);
1385 return (0);
1386}
1387#endif
1388
1389static void
1390cxgb_init(void *arg)
1391{
1392 struct port_info *p = arg;
1393
1394 PORT_LOCK(p);
1395 cxgb_init_locked(p);
1396 PORT_UNLOCK(p);
1397}
1398
1399static void
1400cxgb_init_locked(struct port_info *p)
1401{
1402 struct ifnet *ifp;
1403 adapter_t *sc = p->adapter;
1404 int err;
1405
1406 PORT_LOCK_ASSERT_OWNED(p);
1407 ifp = p->ifp;
1408
1409 ADAPTER_LOCK(p->adapter);
1410 if ((sc->open_device_map == 0) && ((err = cxgb_up(sc)) < 0)) {
1411 ADAPTER_UNLOCK(p->adapter);
1412 cxgb_stop_locked(p);
1413 return;
1414 }
1415 if (p->adapter->open_device_map == 0) {
1416 t3_intr_clear(sc);
1417 t3_sge_init_adapter(sc);
1418 }
1419 setbit(&p->adapter->open_device_map, p->port);
1420 ADAPTER_UNLOCK(p->adapter);
1421
1422 if (is_offload(sc) && !ofld_disable) {
1423 err = offload_open(p);
1424 if (err)
1425 log(LOG_WARNING,
1426 "Could not initialize offload capabilities\n");
1427 }
1428 cxgb_link_start(p);
1429 t3_link_changed(sc, p->port);
1430 ifp->if_baudrate = p->link_config.speed * 1000000;
1431
1432 t3_port_intr_enable(sc, p->port);
1433
1434 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1435 cxgb_tick, sc);
1436
1437 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1438 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1439}
1440
1441static void
1442cxgb_set_rxmode(struct port_info *p)
1443{
1444 struct t3_rx_mode rm;
1445 struct cmac *mac = &p->mac;
1446
1447 PORT_LOCK_ASSERT_OWNED(p);
1448
1449 t3_init_rx_mode(&rm, p);
1450 t3_mac_set_rx_mode(mac, &rm);
1451}
1452
1453static void
1454cxgb_stop_locked(struct port_info *p)
1455{
1456 struct ifnet *ifp;
1457
1458 PORT_LOCK_ASSERT_OWNED(p);
1459 ADAPTER_LOCK_ASSERT_NOTOWNED(p->adapter);
1460
1461 ifp = p->ifp;
1462
1463 t3_port_intr_disable(p->adapter, p->port);
1464 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1465 p->phy.ops->power_down(&p->phy, 1);
1466 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1467
1468 ADAPTER_LOCK(p->adapter);
1469 clrbit(&p->adapter->open_device_map, p->port);
1470
1471
1472 if (p->adapter->open_device_map == 0) {
1473 cxgb_down_locked(p->adapter);
1474 } else
1475 ADAPTER_UNLOCK(p->adapter);
1476
1477}
1478
1479static int
1480cxgb_set_mtu(struct port_info *p, int mtu)
1481{
1482 struct ifnet *ifp = p->ifp;
1483 int error = 0;
1484
1485 if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO))
1486 error = EINVAL;
1487 else if (ifp->if_mtu != mtu) {
1488 PORT_LOCK(p);
1489 ifp->if_mtu = mtu;
1490 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1491 callout_stop(&p->adapter->cxgb_tick_ch);
1492 cxgb_stop_locked(p);
1493 cxgb_init_locked(p);
1494 }
1495 PORT_UNLOCK(p);
1496 }
1497 return (error);
1498}
1499
1500static int
1501cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1502{
1503 struct port_info *p = ifp->if_softc;
1504 struct ifaddr *ifa = (struct ifaddr *)data;
1505 struct ifreq *ifr = (struct ifreq *)data;
1506 int flags, error = 0;
1507 uint32_t mask;
1508
1509 /*
1510 * XXX need to check that we aren't in the middle of an unload
1511 */
1512 switch (command) {
1513 case SIOCSIFMTU:
1514 error = cxgb_set_mtu(p, ifr->ifr_mtu);
1515 break;
1516 case SIOCSIFADDR:
1517 case SIOCGIFADDR:
1518 PORT_LOCK(p);
1519 if (ifa->ifa_addr->sa_family == AF_INET) {
1520 ifp->if_flags |= IFF_UP;
1521 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1522 cxgb_init_locked(p);
1523 arp_ifinit(ifp, ifa);
1524 } else
1525 error = ether_ioctl(ifp, command, data);
1526 PORT_UNLOCK(p);
1527 break;
1528 case SIOCSIFFLAGS:
1529 callout_drain(&p->adapter->cxgb_tick_ch);
1530 PORT_LOCK(p);
1531 if (ifp->if_flags & IFF_UP) {
1532 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1533 flags = p->if_flags;
1534 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1535 ((ifp->if_flags ^ flags) & IFF_ALLMULTI))
1536 cxgb_set_rxmode(p);
1537 } else
1538 cxgb_init_locked(p);
1539 p->if_flags = ifp->if_flags;
1540 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1541 cxgb_stop_locked(p);
1542
1543 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1544 adapter_t *sc = p->adapter;
1545 callout_reset(&sc->cxgb_tick_ch,
1546 sc->params.stats_update_period * hz,
1547 cxgb_tick, sc);
1548 }
1549 PORT_UNLOCK(p);
1550 break;
1551 case SIOCSIFMEDIA:
1552 case SIOCGIFMEDIA:
1553 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
1554 break;
1555 case SIOCSIFCAP:
1556 PORT_LOCK(p);
1557 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1558 if (mask & IFCAP_TXCSUM) {
1559 if (IFCAP_TXCSUM & ifp->if_capenable) {
1560 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
1561 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
1562 | CSUM_TSO);
1563 } else {
1564 ifp->if_capenable |= IFCAP_TXCSUM;
1565 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1566 }
1567 } else if (mask & IFCAP_RXCSUM) {
1568 if (IFCAP_RXCSUM & ifp->if_capenable) {
1569 ifp->if_capenable &= ~IFCAP_RXCSUM;
1570 } else {
1571 ifp->if_capenable |= IFCAP_RXCSUM;
1572 }
1573 }
1574 if (mask & IFCAP_TSO4) {
1575 if (IFCAP_TSO4 & ifp->if_capenable) {
1576 ifp->if_capenable &= ~IFCAP_TSO4;
1577 ifp->if_hwassist &= ~CSUM_TSO;
1578 } else if (IFCAP_TXCSUM & ifp->if_capenable) {
1579 ifp->if_capenable |= IFCAP_TSO4;
1580 ifp->if_hwassist |= CSUM_TSO;
1581 } else {
1582 if (cxgb_debug)
1583 printf("cxgb requires tx checksum offload"
1584 " be enabled to use TSO\n");
1585 error = EINVAL;
1586 }
1587 }
1588 PORT_UNLOCK(p);
1589 break;
1590 default:
1591 error = ether_ioctl(ifp, command, data);
1592 break;
1593 }
1594 return (error);
1595}
1596
1597static int
1598cxgb_start_tx(struct ifnet *ifp, uint32_t txmax)
1599{
1600 struct sge_qset *qs;
1601 struct sge_txq *txq;
1602 struct port_info *p = ifp->if_softc;
1603 struct mbuf *m0, *m = NULL;
1604 int err, in_use_init;
1605
1606 if (!p->link_config.link_ok)
1607 return (ENXIO);
1608
1609 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1610 return (ENOBUFS);
1611
1612 qs = &p->adapter->sge.qs[p->first_qset];
1613 txq = &qs->txq[TXQ_ETH];
1614 err = 0;
1615
1616 if (txq->flags & TXQ_TRANSMITTING)
1617 return (EINPROGRESS);
1618
1619 mtx_lock(&txq->lock);
1620 txq->flags |= TXQ_TRANSMITTING;
1621 in_use_init = txq->in_use;
1622 while ((txq->in_use - in_use_init < txmax) &&
1623 (txq->size > txq->in_use + TX_MAX_DESC)) {
1624 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
1625 if (m == NULL)
1626 break;
1627 /*
1628 * Convert chain to M_IOVEC
1629 */
1630 KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early"));
1631 m0 = m;
1632#ifdef INVARIANTS
1633 /*
1634 * Clean up after net stack sloppiness
1635 * before calling m_sanity
1636 */
1637 m0 = m->m_next;
1638 while (m0) {
1639 m0->m_flags &= ~M_PKTHDR;
1640 m0 = m0->m_next;
1641 }
1642 m_sanity(m0, 0);
1643 m0 = m;
1644#endif
1645 if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES &&
1646 m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) {
1647 if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) {
1648 m = m0;
1649 m_collapse(m, TX_MAX_SEGS, &m0);
1650 } else
1651 break;
1652 }
1653 m = m0;
1654 if ((err = t3_encap(p, &m)) != 0)
1655 break;
1656 BPF_MTAP(ifp, m);
1657 }
1658 txq->flags &= ~TXQ_TRANSMITTING;
1659 mtx_unlock(&txq->lock);
1660
1661 if (__predict_false(err)) {
1662 if (err == ENOMEM) {
1663 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1664 IFQ_LOCK(&ifp->if_snd);
1665 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1666 IFQ_UNLOCK(&ifp->if_snd);
1667 }
1668 }
1669 if (err == 0 && m == NULL)
1670 err = ENOBUFS;
1671 else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) &&
1672 (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
1673 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1674 err = ENOSPC;
1675 }
1676 return (err);
1677}
1678
1679static void
1680cxgb_start_proc(void *arg, int ncount)
1681{
1682 struct ifnet *ifp = arg;
1683 struct port_info *pi = ifp->if_softc;
1684 struct sge_qset *qs;
1685 struct sge_txq *txq;
1686 int error;
1687
1688 qs = &pi->adapter->sge.qs[pi->first_qset];
1689 txq = &qs->txq[TXQ_ETH];
1690
1691 do {
1692 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1693 taskqueue_enqueue(pi->adapter->tq, &txq->qreclaim_task);
1694
1695 error = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1696 } while (error == 0);
1697}
1698
1699static void
1700cxgb_start(struct ifnet *ifp)
1701{
1702 struct port_info *pi = ifp->if_softc;
1703 struct sge_qset *qs;
1704 struct sge_txq *txq;
1705 int err;
1706
1707 qs = &pi->adapter->sge.qs[pi->first_qset];
1708 txq = &qs->txq[TXQ_ETH];
1709
1710 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1711 taskqueue_enqueue(pi->adapter->tq,
1712 &pi->timer_reclaim_task);
1713
1714 err = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1715
1716 if (err == 0)
1717 taskqueue_enqueue(pi->tq, &pi->start_task);
1718}
1719
1720
1721static int
1722cxgb_media_change(struct ifnet *ifp)
1723{
1724 if_printf(ifp, "media change not supported\n");
1725 return (ENXIO);
1726}
1727
1728static void
1729cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1730{
1731 struct port_info *p = ifp->if_softc;
1732
1733 ifmr->ifm_status = IFM_AVALID;
1734 ifmr->ifm_active = IFM_ETHER;
1735
1736 if (!p->link_config.link_ok)
1737 return;
1738
1739 ifmr->ifm_status |= IFM_ACTIVE;
1740
1741 switch (p->link_config.speed) {
1742 case 10:
1743 ifmr->ifm_active |= IFM_10_T;
1744 break;
1745 case 100:
1746 ifmr->ifm_active |= IFM_100_TX;
1747 break;
1748 case 1000:
1749 ifmr->ifm_active |= IFM_1000_T;
1750 break;
1751 }
1752
1753 if (p->link_config.duplex)
1754 ifmr->ifm_active |= IFM_FDX;
1755 else
1756 ifmr->ifm_active |= IFM_HDX;
1757}
1758
1759static void
1760cxgb_async_intr(void *data)
1761{
1762 adapter_t *sc = data;
1763
1764 if (cxgb_debug)
1765 device_printf(sc->dev, "cxgb_async_intr\n");
1766 /*
1767 * May need to sleep - defer to taskqueue
1768 */
1769 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
1770}
1771
1772static void
1773cxgb_ext_intr_handler(void *arg, int count)
1774{
1775 adapter_t *sc = (adapter_t *)arg;
1776
1777 if (cxgb_debug)
1778 printf("cxgb_ext_intr_handler\n");
1779
1780 t3_phy_intr_handler(sc);
1781
1782 /* Now reenable external interrupts */
1783 ADAPTER_LOCK(sc);
1784 if (sc->slow_intr_mask) {
1785 sc->slow_intr_mask |= F_T3DBG;
1786 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
1787 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1788 }
1789 ADAPTER_UNLOCK(sc);
1790}
1791
1792static void
1793check_link_status(adapter_t *sc)
1794{
1795 int i;
1796
1797 for (i = 0; i < (sc)->params.nports; ++i) {
1798 struct port_info *p = &sc->port[i];
1799
1800 if (!(p->port_type->caps & SUPPORTED_IRQ))
1801 t3_link_changed(sc, i);
1802 p->ifp->if_baudrate = p->link_config.speed * 1000000;
1803 }
1804}
1805
1806static void
1807check_t3b2_mac(struct adapter *adapter)
1808{
1809 int i;
1810
1811 for_each_port(adapter, i) {
1812 struct port_info *p = &adapter->port[i];
1813 struct ifnet *ifp = p->ifp;
1814 int status;
1815
1816 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1817 continue;
1818
1819 status = 0;
1820 PORT_LOCK(p);
1821 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
1822 status = t3b2_mac_watchdog_task(&p->mac);
1823 if (status == 1)
1824 p->mac.stats.num_toggled++;
1825 else if (status == 2) {
1826 struct cmac *mac = &p->mac;
1827
1828 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN
1829 + ETHER_VLAN_ENCAP_LEN);
1830 t3_mac_set_address(mac, 0, p->hw_addr);
1831 cxgb_set_rxmode(p);
1832 t3_link_start(&p->phy, mac, &p->link_config);
1833 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1834 t3_port_intr_enable(adapter, p->port);
1835 p->mac.stats.num_resets++;
1836 }
1837 PORT_UNLOCK(p);
1838 }
1839}
1840
1841static void
1842cxgb_tick(void *arg)
1843{
1844 adapter_t *sc = (adapter_t *)arg;
1845
1846 taskqueue_enqueue(sc->tq, &sc->tick_task);
1847
1848 if (sc->open_device_map != 0)
1849 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1850 cxgb_tick, sc);
1851}
1852
1853static void
1854cxgb_tick_handler(void *arg, int count)
1855{
1856 adapter_t *sc = (adapter_t *)arg;
1857 const struct adapter_params *p = &sc->params;
1858
1859 ADAPTER_LOCK(sc);
1860 if (p->linkpoll_period)
1861 check_link_status(sc);
1862
1863 /*
1864 * adapter lock can currently only be acquire after the
1865 * port lock
1866 */
1867 ADAPTER_UNLOCK(sc);
1868
1869 if (p->rev == T3_REV_B2)
1870 check_t3b2_mac(sc);
1871}
1872
1873static int
1874in_range(int val, int lo, int hi)
1875{
1876 return val < 0 || (val <= hi && val >= lo);
1877}
1878
1879static int
1880cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td)
1881{
1882 return (0);
1883}
1884
1885static int
1886cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
1887{
1888 return (0);
1889}
1890
1891static int
1892cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
1893 int fflag, struct thread *td)
1894{
1895 int mmd, error = 0;
1896 struct port_info *pi = dev->si_drv1;
1897 adapter_t *sc = pi->adapter;
1898
1899#ifdef PRIV_SUPPORTED
1900 if (priv_check(td, PRIV_DRIVER)) {
1901 if (cxgb_debug)
1902 printf("user does not have access to privileged ioctls\n");
1903 return (EPERM);
1904 }
1905#else
1906 if (suser(td)) {
1907 if (cxgb_debug)
1908 printf("user does not have access to privileged ioctls\n");
1909 return (EPERM);
1910 }
1911#endif
1912
1913 switch (cmd) {
1914 case SIOCGMIIREG: {
1915 uint32_t val;
1916 struct cphy *phy = &pi->phy;
1917 struct mii_data *mid = (struct mii_data *)data;
1918
1919 if (!phy->mdio_read)
1920 return (EOPNOTSUPP);
1921 if (is_10G(sc)) {
1922 mmd = mid->phy_id >> 8;
1923 if (!mmd)
1924 mmd = MDIO_DEV_PCS;
1925 else if (mmd > MDIO_DEV_XGXS)
1926 return -EINVAL;
1927
1928 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
1929 mid->reg_num, &val);
1930 } else
1931 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
1932 mid->reg_num & 0x1f, &val);
1933 if (error == 0)
1934 mid->val_out = val;
1935 break;
1936 }
1937 case SIOCSMIIREG: {
1938 struct cphy *phy = &pi->phy;
1939 struct mii_data *mid = (struct mii_data *)data;
1940
1941 if (!phy->mdio_write)
1942 return (EOPNOTSUPP);
1943 if (is_10G(sc)) {
1944 mmd = mid->phy_id >> 8;
1945 if (!mmd)
1946 mmd = MDIO_DEV_PCS;
1947 else if (mmd > MDIO_DEV_XGXS)
1948 return (EINVAL);
1949
1950 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
1951 mmd, mid->reg_num, mid->val_in);
1952 } else
1953 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
1954 mid->reg_num & 0x1f,
1955 mid->val_in);
1956 break;
1957 }
1958 case CHELSIO_SETREG: {
1959 struct ch_reg *edata = (struct ch_reg *)data;
1960 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1961 return (EFAULT);
1962 t3_write_reg(sc, edata->addr, edata->val);
1963 break;
1964 }
1965 case CHELSIO_GETREG: {
1966 struct ch_reg *edata = (struct ch_reg *)data;
1967 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1968 return (EFAULT);
1969 edata->val = t3_read_reg(sc, edata->addr);
1970 break;
1971 }
1972 case CHELSIO_GET_SGE_CONTEXT: {
1973 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
1974 mtx_lock(&sc->sge.reg_lock);
1975 switch (ecntxt->cntxt_type) {
1976 case CNTXT_TYPE_EGRESS:
1977 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
1978 ecntxt->data);
1979 break;
1980 case CNTXT_TYPE_FL:
1981 error = t3_sge_read_fl(sc, ecntxt->cntxt_id,
1982 ecntxt->data);
1983 break;
1984 case CNTXT_TYPE_RSP:
1985 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id,
1986 ecntxt->data);
1987 break;
1988 case CNTXT_TYPE_CQ:
1989 error = t3_sge_read_cq(sc, ecntxt->cntxt_id,
1990 ecntxt->data);
1991 break;
1992 default:
1993 error = EINVAL;
1994 break;
1995 }
1996 mtx_unlock(&sc->sge.reg_lock);
1997 break;
1998 }
1999 case CHELSIO_GET_SGE_DESC: {
2000 struct ch_desc *edesc = (struct ch_desc *)data;
2001 int ret;
2002 if (edesc->queue_num >= SGE_QSETS * 6)
2003 return (EINVAL);
2004 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2005 edesc->queue_num % 6, edesc->idx, edesc->data);
2006 if (ret < 0)
2007 return (EINVAL);
2008 edesc->size = ret;
2009 break;
2010 }
2011 case CHELSIO_SET_QSET_PARAMS: {
2012 struct qset_params *q;
2013 struct ch_qset_params *t = (struct ch_qset_params *)data;
2014
2015 if (t->qset_idx >= SGE_QSETS)
2016 return -EINVAL;
2017 if (!in_range(t->intr_lat, 0, M_NEWTIMER) ||
2018 !in_range(t->cong_thres, 0, 255) ||
2019 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES,
2020 MAX_TXQ_ENTRIES) ||
2021 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES,
2022 MAX_TXQ_ENTRIES) ||
2023 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2024 MAX_CTRL_TXQ_ENTRIES) ||
2025 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) ||
2026 !in_range(t->fl_size[1], MIN_FL_ENTRIES,
2027 MAX_RX_JUMBO_BUFFERS) ||
2028 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
2029 return -EINVAL;
2030 if ((sc->flags & FULL_INIT_DONE) &&
2031 (t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
2032 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||
2033 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 ||
2034 t->polling >= 0 || t->cong_thres >= 0))
2035 return -EBUSY;
2036
2037 q = &sc->params.sge.qset[t->qset_idx];
2038
2039 if (t->rspq_size >= 0)
2040 q->rspq_size = t->rspq_size;
2041 if (t->fl_size[0] >= 0)
2042 q->fl_size = t->fl_size[0];
2043 if (t->fl_size[1] >= 0)
2044 q->jumbo_size = t->fl_size[1];
2045 if (t->txq_size[0] >= 0)
2046 q->txq_size[0] = t->txq_size[0];
2047 if (t->txq_size[1] >= 0)
2048 q->txq_size[1] = t->txq_size[1];
2049 if (t->txq_size[2] >= 0)
2050 q->txq_size[2] = t->txq_size[2];
2051 if (t->cong_thres >= 0)
2052 q->cong_thres = t->cong_thres;
2053 if (t->intr_lat >= 0) {
2054 struct sge_qset *qs = &sc->sge.qs[t->qset_idx];
2055
2056 q->coalesce_nsecs = t->intr_lat*1000;
2057 t3_update_qset_coalesce(qs, q);
2058 }
2059 break;
2060 }
2061 case CHELSIO_GET_QSET_PARAMS: {
2062 struct qset_params *q;
2063 struct ch_qset_params *t = (struct ch_qset_params *)data;
2064
2065 if (t->qset_idx >= SGE_QSETS)
2066 return (EINVAL);
2067
2068 q = &(sc)->params.sge.qset[t->qset_idx];
2069 t->rspq_size = q->rspq_size;
2070 t->txq_size[0] = q->txq_size[0];
2071 t->txq_size[1] = q->txq_size[1];
2072 t->txq_size[2] = q->txq_size[2];
2073 t->fl_size[0] = q->fl_size;
2074 t->fl_size[1] = q->jumbo_size;
2075 t->polling = q->polling;
2076 t->intr_lat = q->coalesce_nsecs / 1000;
2077 t->cong_thres = q->cong_thres;
2078 break;
2079 }
2080 case CHELSIO_SET_QSET_NUM: {
2081 struct ch_reg *edata = (struct ch_reg *)data;
2082 unsigned int port_idx = pi->port;
2083
2084 if (sc->flags & FULL_INIT_DONE)
2085 return (EBUSY);
2086 if (edata->val < 1 ||
2087 (edata->val > 1 && !(sc->flags & USING_MSIX)))
2088 return (EINVAL);
2089 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS)
2090 return (EINVAL);
2091 sc->port[port_idx].nqsets = edata->val;
2092 sc->port[0].first_qset = 0;
2093 /*
2094 * XXX hardcode ourselves to 2 ports just like LEEENUX
2095 */
2096 sc->port[1].first_qset = sc->port[0].nqsets;
2097 break;
2098 }
2099 case CHELSIO_GET_QSET_NUM: {
2100 struct ch_reg *edata = (struct ch_reg *)data;
2101 edata->val = pi->nqsets;
2102 break;
2103 }
2104#ifdef notyet
2105 case CHELSIO_LOAD_FW:
2106 case CHELSIO_GET_PM:
2107 case CHELSIO_SET_PM:
2108 return (EOPNOTSUPP);
2109 break;
2110#endif
2111 case CHELSIO_SETMTUTAB: {
2112 struct ch_mtus *m = (struct ch_mtus *)data;
2113 int i;
2114
2115 if (!is_offload(sc))
2116 return (EOPNOTSUPP);
2117 if (offload_running(sc))
2118 return (EBUSY);
2119 if (m->nmtus != NMTUS)
2120 return (EINVAL);
2121 if (m->mtus[0] < 81) /* accommodate SACK */
2122 return (EINVAL);
2123
2124 /*
2125 * MTUs must be in ascending order
2126 */
2127 for (i = 1; i < NMTUS; ++i)
2128 if (m->mtus[i] < m->mtus[i - 1])
2129 return (EINVAL);
2130
2131 memcpy(sc->params.mtus, m->mtus,
2132 sizeof(sc->params.mtus));
2133 break;
2134 }
2135 case CHELSIO_GETMTUTAB: {
2136 struct ch_mtus *m = (struct ch_mtus *)data;
2137
2138 if (!is_offload(sc))
2139 return (EOPNOTSUPP);
2140
2141 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2142 m->nmtus = NMTUS;
2143 break;
2144 }
2145 case CHELSIO_DEVUP:
2146 if (!is_offload(sc))
2147 return (EOPNOTSUPP);
2148 return offload_open(pi);
2149 break;
2150 case CHELSIO_GET_MEM: {
2151 struct ch_mem_range *t = (struct ch_mem_range *)data;
2152 struct mc7 *mem;
2153 uint8_t *useraddr;
2154 u64 buf[32];
2155
2156 if (!is_offload(sc))
2157 return (EOPNOTSUPP);
2158 if (!(sc->flags & FULL_INIT_DONE))
2159 return (EIO); /* need the memory controllers */
2160 if ((t->addr & 0x7) || (t->len & 0x7))
2161 return (EINVAL);
2162 if (t->mem_id == MEM_CM)
2163 mem = &sc->cm;
2164 else if (t->mem_id == MEM_PMRX)
2165 mem = &sc->pmrx;
2166 else if (t->mem_id == MEM_PMTX)
2167 mem = &sc->pmtx;
2168 else
2169 return (EINVAL);
2170
2171 /*
2172 * Version scheme:
2173 * bits 0..9: chip version
2174 * bits 10..15: chip revision
2175 */
2176 t->version = 3 | (sc->params.rev << 10);
2177
2178 /*
2179 * Read 256 bytes at a time as len can be large and we don't
2180 * want to use huge intermediate buffers.
2181 */
2182 useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */
2183 while (t->len) {
2184 unsigned int chunk = min(t->len, sizeof(buf));
2185
2186 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf);
2187 if (error)
2188 return (-error);
2189 if (copyout(buf, useraddr, chunk))
2190 return (EFAULT);
2191 useraddr += chunk;
2192 t->addr += chunk;
2193 t->len -= chunk;
2194 }
2195 break;
2196 }
2197 case CHELSIO_READ_TCAM_WORD: {
2198 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2199
2200 if (!is_offload(sc))
2201 return (EOPNOTSUPP);
2202 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2203 break;
2204 }
2205 case CHELSIO_SET_TRACE_FILTER: {
2206 struct ch_trace *t = (struct ch_trace *)data;
2207 const struct trace_params *tp;
2208
2209 tp = (const struct trace_params *)&t->sip;
2210 if (t->config_tx)
2211 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2212 t->trace_tx);
2213 if (t->config_rx)
2214 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2215 t->trace_rx);
2216 break;
2217 }
2218 case CHELSIO_SET_PKTSCHED: {
2219 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2220 if (sc->open_device_map == 0)
2221 return (EAGAIN);
2222 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2223 p->binding);
2224 break;
2225 }
2226 case CHELSIO_IFCONF_GETREGS: {
2227 struct ifconf_regs *regs = (struct ifconf_regs *)data;
2228 int reglen = cxgb_get_regs_len();
2229 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT);
2230 if (buf == NULL) {
2231 return (ENOMEM);
2232 } if (regs->len > reglen)
2233 regs->len = reglen;
2234 else if (regs->len < reglen) {
2235 error = E2BIG;
2236 goto done;
2237 }
2238 cxgb_get_regs(sc, regs, buf);
2239 error = copyout(buf, regs->data, reglen);
2240
2241 done:
2242 free(buf, M_DEVBUF);
2243
2244 break;
2245 }
2246 case CHELSIO_SET_HW_SCHED: {
2247 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2248 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2249
2250 if ((sc->flags & FULL_INIT_DONE) == 0)
2251 return (EAGAIN); /* need TP to be initialized */
2252 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2253 !in_range(t->channel, 0, 1) ||
2254 !in_range(t->kbps, 0, 10000000) ||
2255 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2256 !in_range(t->flow_ipg, 0,
2257 dack_ticks_to_usec(sc, 0x7ff)))
2258 return (EINVAL);
2259
2260 if (t->kbps >= 0) {
2261 error = t3_config_sched(sc, t->kbps, t->sched);
2262 if (error < 0)
2263 return (-error);
2264 }
2265 if (t->class_ipg >= 0)
2266 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2267 if (t->flow_ipg >= 0) {
2268 t->flow_ipg *= 1000; /* us -> ns */
2269 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2270 }
2271 if (t->mode >= 0) {
2272 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2273
2274 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2275 bit, t->mode ? bit : 0);
2276 }
2277 if (t->channel >= 0)
2278 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2279 1 << t->sched, t->channel << t->sched);
2280 break;
2281 }
2282 default:
2283 return (EOPNOTSUPP);
2284 break;
2285 }
2286
2287 return (error);
2288}
2289
2290static __inline void
2291reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
2292 unsigned int end)
2293{
2294 uint32_t *p = (uint32_t *)buf + start;
2295
2296 for ( ; start <= end; start += sizeof(uint32_t))
2297 *p++ = t3_read_reg(ap, start);
2298}
2299
2300#define T3_REGMAP_SIZE (3 * 1024)
2301static int
2302cxgb_get_regs_len(void)
2303{
2304 return T3_REGMAP_SIZE;
2305}
2306#undef T3_REGMAP_SIZE
2307
2308static void
2309cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf)
2310{
2311
2312 /*
2313 * Version scheme:
2314 * bits 0..9: chip version
2315 * bits 10..15: chip revision
2316 * bit 31: set for PCIe cards
2317 */
2318 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
2319
2320 /*
2321 * We skip the MAC statistics registers because they are clear-on-read.
2322 * Also reading multi-register stats would need to synchronize with the
2323 * periodic mac stats accumulation. Hard to justify the complexity.
2324 */
2325 memset(buf, 0, REGDUMP_SIZE);
2326 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
2327 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
2328 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
2329 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
2330 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
2331 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
2332 XGM_REG(A_XGM_SERDES_STAT3, 1));
2333 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
2334 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
2335}