1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
122. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_main.c 170789 2007-06-15 20:02:02Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/bus.h>
37#include <sys/module.h>
38#include <sys/pciio.h>
39#include <sys/conf.h>
40#include <machine/bus.h>
41#include <machine/resource.h>
42#include <sys/bus_dma.h>
43#include <sys/rman.h>
44#include <sys/ioccom.h>
45#include <sys/mbuf.h>
46#include <sys/linker.h>
47#include <sys/firmware.h>
48#include <sys/socket.h>
49#include <sys/sockio.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <sys/queue.h>
53#include <sys/taskqueue.h>
54
55#include <net/bpf.h>
56#include <net/ethernet.h>
57#include <net/if.h>
58#include <net/if_arp.h>
59#include <net/if_dl.h>
60#include <net/if_media.h>
61#include <net/if_types.h>
62
63#include <netinet/in_systm.h>
64#include <netinet/in.h>
65#include <netinet/if_ether.h>
66#include <netinet/ip.h>
67#include <netinet/ip.h>
68#include <netinet/tcp.h>
69#include <netinet/udp.h>
70
71#include <dev/pci/pcireg.h>
72#include <dev/pci/pcivar.h>
73#include <dev/pci/pci_private.h>
74
75#ifdef CONFIG_DEFINED
76#include <cxgb_include.h>
77#else
78#include <dev/cxgb/cxgb_include.h>
79#endif
80
81#ifdef PRIV_SUPPORTED
82#include <sys/priv.h>
83#endif
84
85static int cxgb_setup_msix(adapter_t *, int);
86static void cxgb_teardown_msix(adapter_t *);
87static void cxgb_init(void *);
88static void cxgb_init_locked(struct port_info *);
89static void cxgb_stop_locked(struct port_info *);
90static void cxgb_set_rxmode(struct port_info *);
91static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
92static void cxgb_start(struct ifnet *);
93static void cxgb_start_proc(void *, int ncount);
94static int cxgb_media_change(struct ifnet *);
95static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
96static int setup_sge_qsets(adapter_t *);
97static void cxgb_async_intr(void *);
98static void cxgb_ext_intr_handler(void *, int);
99static void cxgb_down(struct adapter *sc);
100static void cxgb_tick(void *);
101static void setup_rss(adapter_t *sc);
102
103/* Attachment glue for the PCI controller end of the device. Each port of
104 * the device is attached separately, as defined later.
105 */
106static int cxgb_controller_probe(device_t);
107static int cxgb_controller_attach(device_t);
108static int cxgb_controller_detach(device_t);
109static void cxgb_free(struct adapter *);
110static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
111 unsigned int end);
112static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf);
113static int cxgb_get_regs_len(void);
114static int offload_open(struct port_info *pi);
115#ifdef notyet
116static int offload_close(struct toedev *tdev);
117#endif
118
119
120static device_method_t cxgb_controller_methods[] = {
121 DEVMETHOD(device_probe, cxgb_controller_probe),
122 DEVMETHOD(device_attach, cxgb_controller_attach),
123 DEVMETHOD(device_detach, cxgb_controller_detach),
124
125 /* bus interface */
126 DEVMETHOD(bus_print_child, bus_generic_print_child),
127 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
128
129 { 0, 0 }
130};
131
132static driver_t cxgb_controller_driver = {
133 "cxgbc",
134 cxgb_controller_methods,
135 sizeof(struct adapter)
136};
137
138static devclass_t cxgb_controller_devclass;
139DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
140
141/*
142 * Attachment glue for the ports. Attachment is done directly to the
143 * controller device.
144 */
145static int cxgb_port_probe(device_t);
146static int cxgb_port_attach(device_t);
147static int cxgb_port_detach(device_t);
148
149static device_method_t cxgb_port_methods[] = {
150 DEVMETHOD(device_probe, cxgb_port_probe),
151 DEVMETHOD(device_attach, cxgb_port_attach),
152 DEVMETHOD(device_detach, cxgb_port_detach),
153 { 0, 0 }
154};
155
156static driver_t cxgb_port_driver = {
157 "cxgb",
158 cxgb_port_methods,
159 0
160};
161
162static d_ioctl_t cxgb_extension_ioctl;
163static d_open_t cxgb_extension_open;
164static d_close_t cxgb_extension_close;
165
166static struct cdevsw cxgb_cdevsw = {
167 .d_version = D_VERSION,
168 .d_flags = 0,
169 .d_open = cxgb_extension_open,
170 .d_close = cxgb_extension_close,
171 .d_ioctl = cxgb_extension_ioctl,
172 .d_name = "cxgb",
173};
174
175static devclass_t cxgb_port_devclass;
176DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
177
178#define SGE_MSIX_COUNT (SGE_QSETS + 1)
179
180extern int collapse_mbufs;
181/*
182 * The driver uses the best interrupt scheme available on a platform in the
183 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
184 * of these schemes the driver may consider as follows:
185 *
186 * msi = 2: choose from among all three options
187 * msi = 1 : only consider MSI and pin interrupts
188 * msi = 0: force pin interrupts
189 */
190static int msi_allowed = 2;
191
192TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
193SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
194SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
195 "MSI-X, MSI, INTx selector");
196
197/*
198 * The driver enables offload as a default.
199 * To disable it, use ofld_disable = 1.
200 */
201static int ofld_disable = 0;
202TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
203SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
204 "disable ULP offload");
205
206/*
207 * The driver uses an auto-queue algorithm by default.
208 * To disable it and force a single queue-set per port, use singleq = 1.
209 */
210static int singleq = 1;
211TUNABLE_INT("hw.cxgb.singleq", &singleq);
212SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
213 "use a single queue-set per port");
214
215enum {
216 MAX_TXQ_ENTRIES = 16384,
217 MAX_CTRL_TXQ_ENTRIES = 1024,
218 MAX_RSPQ_ENTRIES = 16384,
219 MAX_RX_BUFFERS = 16384,
220 MAX_RX_JUMBO_BUFFERS = 16384,
221 MIN_TXQ_ENTRIES = 4,
222 MIN_CTRL_TXQ_ENTRIES = 4,
223 MIN_RSPQ_ENTRIES = 32,
224 MIN_FL_ENTRIES = 32
225};
226
227#define PORT_MASK ((1 << MAX_NPORTS) - 1)
228
229/* Table for probing the cards. The desc field isn't actually used */
230struct cxgb_ident {
231 uint16_t vendor;
232 uint16_t device;
233 int index;
234 char *desc;
235} cxgb_identifiers[] = {
236 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
237 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
238 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
239 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
240 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
241 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
242 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
243 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
244 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
245 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
246 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
247 {0, 0, 0, NULL}
248};
249
250static struct cxgb_ident *
251cxgb_get_ident(device_t dev)
252{
253 struct cxgb_ident *id;
254
255 for (id = cxgb_identifiers; id->desc != NULL; id++) {
256 if ((id->vendor == pci_get_vendor(dev)) &&
257 (id->device == pci_get_device(dev))) {
258 return (id);
259 }
260 }
261 return (NULL);
262}
263
264static const struct adapter_info *
265cxgb_get_adapter_info(device_t dev)
266{
267 struct cxgb_ident *id;
268 const struct adapter_info *ai;
269
270 id = cxgb_get_ident(dev);
271 if (id == NULL)
272 return (NULL);
273
274 ai = t3_get_adapter_info(id->index);
275
276 return (ai);
277}
278
279static int
280cxgb_controller_probe(device_t dev)
281{
282 const struct adapter_info *ai;
283 char *ports, buf[80];
284 int nports;
285
286 ai = cxgb_get_adapter_info(dev);
287 if (ai == NULL)
288 return (ENXIO);
289
290 nports = ai->nports0 + ai->nports1;
291 if (nports == 1)
292 ports = "port";
293 else
294 ports = "ports";
295
296 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports);
297 device_set_desc_copy(dev, buf);
298 return (BUS_PROBE_DEFAULT);
299}
300
301static int
302upgrade_fw(adapter_t *sc)
303{
304 char buf[32];
305#ifdef FIRMWARE_LATEST
306 const struct firmware *fw;
307#else
308 struct firmware *fw;
309#endif
310 int status;
311
312 snprintf(&buf[0], sizeof(buf), "t3fw%d%d%d", FW_VERSION_MAJOR,
313 FW_VERSION_MINOR, FW_VERSION_MICRO);
314
315 fw = firmware_get(buf);
316
317 if (fw == NULL) {
318 device_printf(sc->dev, "Could not find firmware image %s\n", buf);
319 return (ENOENT);
320 }
321
322 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
323
324 firmware_put(fw, FIRMWARE_UNLOAD);
325
326 return (status);
327}
328
329static int
330cxgb_controller_attach(device_t dev)
331{
332 device_t child;
333 const struct adapter_info *ai;
334 struct adapter *sc;
335 int i, reg, msi_needed, error = 0;
336 uint32_t vers;
337 int port_qsets = 1;
338
339 sc = device_get_softc(dev);
340 sc->dev = dev;
341 sc->msi_count = 0;
342
343 /* find the PCIe link width and set max read request to 4KB*/
344 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
345 uint16_t lnk, pectl;
346 lnk = pci_read_config(dev, reg + 0x12, 2);
347 sc->link_width = (lnk >> 4) & 0x3f;
348
349 pectl = pci_read_config(dev, reg + 0x8, 2);
350 pectl = (pectl & ~0x7000) | (5 << 12);
351 pci_write_config(dev, reg + 0x8, pectl, 2);
352 }
353 if (sc->link_width != 0 && sc->link_width <= 4) {
354 device_printf(sc->dev,
355 "PCIe x%d Link, expect reduced performance\n",
356 sc->link_width);
357 }
358
359 pci_enable_busmaster(dev);
360 /*
361 * Allocate the registers and make them available to the driver.
362 * The registers that we care about for NIC mode are in BAR 0
363 */
364 sc->regs_rid = PCIR_BAR(0);
365 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
366 &sc->regs_rid, RF_ACTIVE)) == NULL) {
367 device_printf(dev, "Cannot allocate BAR\n");
368 return (ENXIO);
369 }
370
371 mtx_init(&sc->sge.reg_lock, "SGE reg lock", NULL, MTX_DEF);
372 mtx_init(&sc->lock, "cxgb controller lock", NULL, MTX_DEF);
373 mtx_init(&sc->mdio_lock, "cxgb mdio", NULL, MTX_DEF);
374 mtx_init(&sc->elmer_lock, "cxgb elmer", NULL, MTX_DEF);
375
376 sc->bt = rman_get_bustag(sc->regs_res);
377 sc->bh = rman_get_bushandle(sc->regs_res);
378 sc->mmio_len = rman_get_size(sc->regs_res);
379
380 ai = cxgb_get_adapter_info(dev);
381 if (t3_prep_adapter(sc, ai, 1) < 0) {
382 printf("prep adapter failed\n");
383 error = ENODEV;
384 goto out;
385 }
386 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
387 * enough messages for the queue sets. If that fails, try falling
388 * back to MSI. If that fails, then try falling back to the legacy
389 * interrupt pin model.
390 */
391#ifdef MSI_SUPPORTED
392
393 sc->msix_regs_rid = 0x20;
394 if ((msi_allowed >= 2) &&
395 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
396 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
397
398 msi_needed = sc->msi_count = SGE_MSIX_COUNT;
399
400 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
401 (sc->msi_count != msi_needed)) {
402 device_printf(dev, "msix allocation failed - msi_count = %d"
403 " msi_needed=%d will try msi err=%d\n", sc->msi_count,
404 msi_needed, error);
405 sc->msi_count = 0;
406 pci_release_msi(dev);
407 bus_release_resource(dev, SYS_RES_MEMORY,
408 sc->msix_regs_rid, sc->msix_regs_res);
409 sc->msix_regs_res = NULL;
410 } else {
411 sc->flags |= USING_MSIX;
412 sc->cxgb_intr = t3_intr_msix;
413 }
414 }
415
416 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
417 sc->msi_count = 1;
418 if (pci_alloc_msi(dev, &sc->msi_count)) {
419 device_printf(dev, "alloc msi failed - will try INTx\n");
420 sc->msi_count = 0;
421 pci_release_msi(dev);
422 } else {
423 sc->flags |= USING_MSI;
424 sc->irq_rid = 1;
425 sc->cxgb_intr = t3_intr_msi;
426 }
427 }
428#endif
429 if (sc->msi_count == 0) {
430 device_printf(dev, "using line interrupts\n");
431 sc->irq_rid = 0;
432 sc->cxgb_intr = t3b_intr;
433 }
434
435
436 /* Create a private taskqueue thread for handling driver events */
437#ifdef TASKQUEUE_CURRENT
438 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
439 taskqueue_thread_enqueue, &sc->tq);
440#else
441 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
442 taskqueue_thread_enqueue, &sc->tq);
443#endif
444 if (sc->tq == NULL) {
445 device_printf(dev, "failed to allocate controller task queue\n");
446 goto out;
447 }
448
449 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
450 device_get_nameunit(dev));
451 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
452
453
454 /* Create a periodic callout for checking adapter status */
455 callout_init_mtx(&sc->cxgb_tick_ch, &sc->lock, CALLOUT_RETURNUNLOCKED);
456
457 if (t3_check_fw_version(sc) != 0) {
458 /*
459 * Warn user that a firmware update will be attempted in init.
460 */
461 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
462 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
463 sc->flags &= ~FW_UPTODATE;
464 } else {
465 sc->flags |= FW_UPTODATE;
466 }
467
468 if ((sc->flags & USING_MSIX) && !singleq)
469 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
470
471 /*
472 * Create a child device for each MAC. The ethernet attachment
473 * will be done in these children.
474 */
475 for (i = 0; i < (sc)->params.nports; i++) {
476 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
477 device_printf(dev, "failed to add child port\n");
478 error = EINVAL;
479 goto out;
480 }
481 sc->portdev[i] = child;
482 sc->port[i].adapter = sc;
483 sc->port[i].nqsets = port_qsets;
484 sc->port[i].first_qset = i*port_qsets;
485 sc->port[i].port = i;
486 device_set_softc(child, &sc->port[i]);
487 }
488 if ((error = bus_generic_attach(dev)) != 0)
489 goto out;
490
491 /*
492 * XXX need to poll for link status
493 */
494 sc->params.stats_update_period = 1;
495
496 /* initialize sge private state */
497 t3_sge_init_adapter(sc);
498
499 t3_led_ready(sc);
500
501 cxgb_offload_init();
502 if (is_offload(sc)) {
503 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
504 cxgb_adapter_ofld(sc);
505 }
506 error = t3_get_fw_version(sc, &vers);
507 if (error)
508 goto out;
509
510 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
511 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
512 G_FW_VERSION_MICRO(vers));
513
514 t3_add_sysctls(sc);
515out:
516 if (error)
517 cxgb_free(sc);
518
519 return (error);
520}
521
522static int
523cxgb_controller_detach(device_t dev)
524{
525 struct adapter *sc;
526
527 sc = device_get_softc(dev);
528
529 cxgb_free(sc);
530
531 return (0);
532}
533
534static void
535cxgb_free(struct adapter *sc)
536{
537 int i;
538
539 cxgb_down(sc);
540
541#ifdef MSI_SUPPORTED
542 if (sc->flags & (USING_MSI | USING_MSIX)) {
543 device_printf(sc->dev, "releasing msi message(s)\n");
544 pci_release_msi(sc->dev);
545 } else {
546 device_printf(sc->dev, "no msi message to release\n");
547 }
548#endif
549 if (sc->msix_regs_res != NULL) {
550 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
551 sc->msix_regs_res);
552 }
553
554 /*
555 * XXX need to drain the ifq by hand until
556 * it is taught about mbuf iovecs
557 */
558 callout_drain(&sc->cxgb_tick_ch);
559
560 t3_sge_deinit_sw(sc);
561
562 if (sc->tq != NULL) {
563 taskqueue_drain(sc->tq, &sc->ext_intr_task);
564 taskqueue_free(sc->tq);
565 }
566
567 for (i = 0; i < (sc)->params.nports; ++i) {
568 if (sc->portdev[i] != NULL)
569 device_delete_child(sc->dev, sc->portdev[i]);
570 }
571
572 bus_generic_detach(sc->dev);
573#ifdef notyet
574 if (is_offload(sc)) {
575 cxgb_adapter_unofld(sc);
576 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
577 offload_close(&sc->tdev);
578 }
579#endif
580 t3_free_sge_resources(sc);
581 t3_sge_free(sc);
582
583 if (sc->regs_res != NULL)
584 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
585 sc->regs_res);
586
587 mtx_destroy(&sc->mdio_lock);
588 mtx_destroy(&sc->sge.reg_lock);
589 mtx_destroy(&sc->lock);
590
591 return;
592}
593
594/**
595 * setup_sge_qsets - configure SGE Tx/Rx/response queues
596 * @sc: the controller softc
597 *
598 * Determines how many sets of SGE queues to use and initializes them.
599 * We support multiple queue sets per port if we have MSI-X, otherwise
600 * just one queue set per port.
601 */
602static int
603setup_sge_qsets(adapter_t *sc)
604{
605 int i, j, err, irq_idx, qset_idx;
606 u_int ntxq = SGE_TXQ_PER_SET;
607
608 if ((err = t3_sge_alloc(sc)) != 0) {
609 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
610 return (err);
611 }
612
613 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
614 irq_idx = -1;
615 else
616 irq_idx = 0;
617
618 for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) {
619 struct port_info *pi = &sc->port[i];
620
621 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
622 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
623 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
624 &sc->params.sge.qset[qset_idx], ntxq, pi);
625 if (err) {
626 t3_free_sge_resources(sc);
627 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err);
628 return (err);
629 }
630 }
631 }
632
633 return (0);
634}
635
636static void
637cxgb_teardown_msix(adapter_t *sc)
638{
639 int i, nqsets;
640
641 for (nqsets = i = 0; i < (sc)->params.nports; i++)
642 nqsets += sc->port[i].nqsets;
643
644 for (i = 0; i < nqsets; i++) {
645 if (sc->msix_intr_tag[i] != NULL) {
646 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
647 sc->msix_intr_tag[i]);
648 sc->msix_intr_tag[i] = NULL;
649 }
650 if (sc->msix_irq_res[i] != NULL) {
651 bus_release_resource(sc->dev, SYS_RES_IRQ,
652 sc->msix_irq_rid[i], sc->msix_irq_res[i]);
653 sc->msix_irq_res[i] = NULL;
654 }
655 }
656}
657
658static int
659cxgb_setup_msix(adapter_t *sc, int msix_count)
660{
661 int i, j, k, nqsets, rid;
662
663 /* The first message indicates link changes and error conditions */
664 sc->irq_rid = 1;
665 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
666 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
667 device_printf(sc->dev, "Cannot allocate msix interrupt\n");
668 return (EINVAL);
669 }
670
671 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
672#ifdef INTR_FILTERS
673 NULL,
674#endif
675 cxgb_async_intr, sc, &sc->intr_tag)) {
676 device_printf(sc->dev, "Cannot set up interrupt\n");
677 return (EINVAL);
678 }
679 for (i = k = 0; i < (sc)->params.nports; i++) {
680 nqsets = sc->port[i].nqsets;
681 for (j = 0; j < nqsets; j++, k++) {
682 struct sge_qset *qs = &sc->sge.qs[k];
683
684 rid = k + 2;
685 if (cxgb_debug)
686 printf("rid=%d ", rid);
687 if ((sc->msix_irq_res[k] = bus_alloc_resource_any(
688 sc->dev, SYS_RES_IRQ, &rid,
689 RF_SHAREABLE | RF_ACTIVE)) == NULL) {
690 device_printf(sc->dev, "Cannot allocate "
691 "interrupt for message %d\n", rid);
692 return (EINVAL);
693 }
694 sc->msix_irq_rid[k] = rid;
695 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
696 INTR_MPSAFE|INTR_TYPE_NET,
697#ifdef INTR_FILTERS
698 NULL,
699#endif
700 t3_intr_msix, qs, &sc->msix_intr_tag[k])) {
701 device_printf(sc->dev, "Cannot set up "
702 "interrupt for message %d\n", rid);
703 return (EINVAL);
704 }
705 }
706 }
707
708
709 return (0);
710}
711
712static int
713cxgb_port_probe(device_t dev)
714{
715 struct port_info *p;
716 char buf[80];
717
718 p = device_get_softc(dev);
719
720 snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc);
721 device_set_desc_copy(dev, buf);
722 return (0);
723}
724
725
726static int
727cxgb_makedev(struct port_info *pi)
728{
729
730 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
731 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
732
733 if (pi->port_cdev == NULL)
734 return (ENOMEM);
735
736 pi->port_cdev->si_drv1 = (void *)pi;
737
738 return (0);
739}
740
741
742#ifdef TSO_SUPPORTED
743#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU)
744/* Don't enable TSO6 yet */
745#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU)
746#else
747#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
748/* Don't enable TSO6 yet */
749#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
750#define IFCAP_TSO4 0x0
751#define CSUM_TSO 0x0
752#endif
753
754
755static int
756cxgb_port_attach(device_t dev)
757{
758 struct port_info *p;
759 struct ifnet *ifp;
760 int err, media_flags;
761 char buf[64];
762
763 p = device_get_softc(dev);
764
765 snprintf(buf, sizeof(buf), "cxgb port %d", p->port);
766 mtx_init(&p->lock, buf, 0, MTX_DEF);
767
768 /* Allocate an ifnet object and set it up */
769 ifp = p->ifp = if_alloc(IFT_ETHER);
770 if (ifp == NULL) {
771 device_printf(dev, "Cannot allocate ifnet\n");
772 return (ENOMEM);
773 }
774
775 /*
776 * Note that there is currently no watchdog timer.
777 */
778 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
779 ifp->if_init = cxgb_init;
780 ifp->if_softc = p;
781 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
782 ifp->if_ioctl = cxgb_ioctl;
783 ifp->if_start = cxgb_start;
784 ifp->if_timer = 0; /* Disable ifnet watchdog */
785 ifp->if_watchdog = NULL;
786
787 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE;
788 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
789 IFQ_SET_READY(&ifp->if_snd);
790
791 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
792 ifp->if_capabilities |= CXGB_CAP;
793 ifp->if_capenable |= CXGB_CAP_ENABLE;
794 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
795
796 ether_ifattach(ifp, p->hw_addr);
797#ifdef DEFAULT_JUMBO
798 ifp->if_mtu = 9000;
799#endif
800 if ((err = cxgb_makedev(p)) != 0) {
801 printf("makedev failed %d\n", err);
802 return (err);
803 }
804 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
805 cxgb_media_status);
806
807 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) {
808 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
809 } else if (!strcmp(p->port_type->desc, "10GBASE-SR")) {
810 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
811 } else if (!strcmp(p->port_type->desc, "10GBASE-XR")) {
812 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
813 } else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) {
814 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
815 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
816 0, NULL);
817 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX,
818 0, NULL);
819 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
820 0, NULL);
821 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
822 0, NULL);
823 media_flags = 0;
824 } else {
825 printf("unsupported media type %s\n", p->port_type->desc);
826 return (ENXIO);
827 }
828 if (media_flags) {
829 ifmedia_add(&p->media, media_flags, 0, NULL);
830 ifmedia_set(&p->media, media_flags);
831 } else {
832 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
833 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO);
834 }
835
836
837 snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port);
838#ifdef TASKQUEUE_CURRENT
839 /* Create a port for handling TX without starvation */
840 p->tq = taskqueue_create(buf, M_NOWAIT,
841 taskqueue_thread_enqueue, &p->tq);
842#else
843 /* Create a port for handling TX without starvation */
844 p->tq = taskqueue_create_fast(buf, M_NOWAIT,
845 taskqueue_thread_enqueue, &p->tq);
846#endif
847
848 if (p->tq == NULL) {
849 device_printf(dev, "failed to allocate port task queue\n");
850 return (ENOMEM);
851 }
852 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq",
853 device_get_nameunit(dev));
854 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp);
855
856 t3_sge_init_port(p);
857
858 return (0);
859}
860
861static int
862cxgb_port_detach(device_t dev)
863{
864 struct port_info *p;
865
866 p = device_get_softc(dev);
867
868 PORT_LOCK(p);
869 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING)
870 cxgb_stop_locked(p);
871 PORT_UNLOCK(p);
872
873 mtx_destroy(&p->lock);
874 if (p->tq != NULL) {
875 taskqueue_drain(p->tq, &p->start_task);
876 taskqueue_free(p->tq);
877 p->tq = NULL;
878 }
879
880 ether_ifdetach(p->ifp);
881 if_free(p->ifp);
882
883 if (p->port_cdev != NULL)
884 destroy_dev(p->port_cdev);
885
886 return (0);
887}
888
889void
890t3_fatal_err(struct adapter *sc)
891{
892 u_int fw_status[4];
893
894 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
895 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
896 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
897 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
898}
899
900int
901t3_os_find_pci_capability(adapter_t *sc, int cap)
902{
903 device_t dev;
904 struct pci_devinfo *dinfo;
905 pcicfgregs *cfg;
906 uint32_t status;
907 uint8_t ptr;
908
909 dev = sc->dev;
910 dinfo = device_get_ivars(dev);
911 cfg = &dinfo->cfg;
912
913 status = pci_read_config(dev, PCIR_STATUS, 2);
914 if (!(status & PCIM_STATUS_CAPPRESENT))
915 return (0);
916
917 switch (cfg->hdrtype & PCIM_HDRTYPE) {
918 case 0:
919 case 1:
920 ptr = PCIR_CAP_PTR;
921 break;
922 case 2:
923 ptr = PCIR_CAP_PTR_2;
924 break;
925 default:
926 return (0);
927 break;
928 }
929 ptr = pci_read_config(dev, ptr, 1);
930
931 while (ptr != 0) {
932 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
933 return (ptr);
934 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
935 }
936
937 return (0);
938}
939
940int
941t3_os_pci_save_state(struct adapter *sc)
942{
943 device_t dev;
944 struct pci_devinfo *dinfo;
945
946 dev = sc->dev;
947 dinfo = device_get_ivars(dev);
948
949 pci_cfg_save(dev, dinfo, 0);
950 return (0);
951}
952
953int
954t3_os_pci_restore_state(struct adapter *sc)
955{
956 device_t dev;
957 struct pci_devinfo *dinfo;
958
959 dev = sc->dev;
960 dinfo = device_get_ivars(dev);
961
962 pci_cfg_restore(dev, dinfo);
963 return (0);
964}
965
966/**
967 * t3_os_link_changed - handle link status changes
968 * @adapter: the adapter associated with the link change
969 * @port_id: the port index whose limk status has changed
970 * @link_stat: the new status of the link
971 * @speed: the new speed setting
972 * @duplex: the new duplex setting
973 * @fc: the new flow-control setting
974 *
975 * This is the OS-dependent handler for link status changes. The OS
976 * neutral handler takes care of most of the processing for these events,
977 * then calls this handler for any OS-specific processing.
978 */
979void
980t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
981 int duplex, int fc)
982{
983 struct port_info *pi = &adapter->port[port_id];
984 struct cmac *mac = &adapter->port[port_id].mac;
985
986 if ((pi->ifp->if_flags & IFF_UP) == 0)
987 return;
988
989 if (link_status) {
990 t3_mac_enable(mac, MAC_DIRECTION_RX);
991 if_link_state_change(pi->ifp, LINK_STATE_UP);
992 } else {
993 if_link_state_change(pi->ifp, LINK_STATE_DOWN);
994 pi->phy.ops->power_down(&pi->phy, 1);
995 t3_mac_disable(mac, MAC_DIRECTION_RX);
996 t3_link_start(&pi->phy, mac, &pi->link_config);
997 }
998}
999
1000
1001/*
1002 * Interrupt-context handler for external (PHY) interrupts.
1003 */
1004void
1005t3_os_ext_intr_handler(adapter_t *sc)
1006{
1007 if (cxgb_debug)
1008 printf("t3_os_ext_intr_handler\n");
1009 /*
1010 * Schedule a task to handle external interrupts as they may be slow
1011 * and we use a mutex to protect MDIO registers. We disable PHY
1012 * interrupts in the meantime and let the task reenable them when
1013 * it's done.
1014 */
1015 ADAPTER_LOCK(sc);
1016 if (sc->slow_intr_mask) {
1017 sc->slow_intr_mask &= ~F_T3DBG;
1018 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1019 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1020 }
1021 ADAPTER_UNLOCK(sc);
1022}
1023
1024void
1025t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1026{
1027
1028 /*
1029 * The ifnet might not be allocated before this gets called,
1030 * as this is called early on in attach by t3_prep_adapter
1031 * save the address off in the port structure
1032 */
1033 if (cxgb_debug)
1034 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1035 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1036}
1037
1038/**
1039 * link_start - enable a port
1040 * @p: the port to enable
1041 *
1042 * Performs the MAC and PHY actions needed to enable a port.
1043 */
1044static void
1045cxgb_link_start(struct port_info *p)
1046{
1047 struct ifnet *ifp;
1048 struct t3_rx_mode rm;
1049 struct cmac *mac = &p->mac;
1050
1051 ifp = p->ifp;
1052
1053 t3_init_rx_mode(&rm, p);
1054 t3_mac_reset(mac);
1055 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1056 t3_mac_set_address(mac, 0, p->hw_addr);
1057 t3_mac_set_rx_mode(mac, &rm);
1058 t3_link_start(&p->phy, mac, &p->link_config);
1059 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1060}
1061
1062/**
1063 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1064 * @adap: the adapter
1065 *
1066 * Sets up RSS to distribute packets to multiple receive queues. We
1067 * configure the RSS CPU lookup table to distribute to the number of HW
1068 * receive queues, and the response queue lookup table to narrow that
1069 * down to the response queues actually configured for each port.
1070 * We always configure the RSS mapping for two ports since the mapping
1071 * table has plenty of entries.
1072 */
1073static void
1074setup_rss(adapter_t *adap)
1075{
1076 int i;
1077 u_int nq0 = adap->port[0].nqsets;
1078 u_int nq1 = max((u_int)adap->port[1].nqsets, 1U);
1079 uint8_t cpus[SGE_QSETS + 1];
1080 uint16_t rspq_map[RSS_TABLE_SIZE];
1081
1082 for (i = 0; i < SGE_QSETS; ++i)
1083 cpus[i] = i;
1084 cpus[SGE_QSETS] = 0xff;
1085
1086 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1087 rspq_map[i] = i % nq0;
1088 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
1089 }
1090
1091 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1092 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
1093 V_RRCPLCPUSIZE(6), cpus, rspq_map);
1094}
1095
1096/*
1097 * Sends an mbuf to an offload queue driver
1098 * after dealing with any active network taps.
1099 */
1100static inline int
1101offload_tx(struct toedev *tdev, struct mbuf *m)
1102{
1103 int ret;
1104
1105 critical_enter();
1106 ret = t3_offload_tx(tdev, m);
1107 critical_exit();
1108 return (ret);
1109}
1110
1111static int
1112write_smt_entry(struct adapter *adapter, int idx)
1113{
1114 struct port_info *pi = &adapter->port[idx];
1115 struct cpl_smt_write_req *req;
1116 struct mbuf *m;
1117
1118 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1119 return (ENOMEM);
1120
1121 req = mtod(m, struct cpl_smt_write_req *);
1122 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1123 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1124 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
1125 req->iff = idx;
1126 memset(req->src_mac1, 0, sizeof(req->src_mac1));
1127 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1128
1129 m_set_priority(m, 1);
1130
1131 offload_tx(&adapter->tdev, m);
1132
1133 return (0);
1134}
1135
1136static int
1137init_smt(struct adapter *adapter)
1138{
1139 int i;
1140
1141 for_each_port(adapter, i)
1142 write_smt_entry(adapter, i);
1143 return 0;
1144}
1145
1146static void
1147init_port_mtus(adapter_t *adapter)
1148{
1149 unsigned int mtus = adapter->port[0].ifp->if_mtu;
1150
1151 if (adapter->port[1].ifp)
1152 mtus |= adapter->port[1].ifp->if_mtu << 16;
1153 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1154}
1155
1156static void
1157send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1158 int hi, int port)
1159{
1160 struct mbuf *m;
1161 struct mngt_pktsched_wr *req;
1162
1163 m = m_gethdr(M_NOWAIT, MT_DATA);
1164 if (m) {
1165 req = mtod(m, struct mngt_pktsched_wr *);
1166 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1167 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1168 req->sched = sched;
1169 req->idx = qidx;
1170 req->min = lo;
1171 req->max = hi;
1172 req->binding = port;
1173 m->m_len = m->m_pkthdr.len = sizeof(*req);
1174 t3_mgmt_tx(adap, m);
1175 }
1176}
1177
1178static void
1179bind_qsets(adapter_t *sc)
1180{
1181 int i, j;
1182
1183 for (i = 0; i < (sc)->params.nports; ++i) {
1184 const struct port_info *pi = adap2pinfo(sc, i);
1185
1186 for (j = 0; j < pi->nqsets; ++j)
1187 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1188 -1, i);
1189 }
1190}
1191
1192/**
1193 * cxgb_up - enable the adapter
1194 * @adap: adapter being enabled
1195 *
1196 * Called when the first port is enabled, this function performs the
1197 * actions necessary to make an adapter operational, such as completing
1198 * the initialization of HW modules, and enabling interrupts.
1199 *
1200 */
1201static int
1202cxgb_up(struct adapter *sc)
1203{
1204 int err = 0;
1205
1206 if ((sc->flags & FULL_INIT_DONE) == 0) {
1207
1208 if ((sc->flags & FW_UPTODATE) == 0)
1209 err = upgrade_fw(sc);
1210
1211 if (err)
1212 goto out;
1213
1214 err = t3_init_hw(sc, 0);
1215 if (err)
1216 goto out;
1217
1218 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1219
1220 err = setup_sge_qsets(sc);
1221 if (err)
1222 goto out;
1223
1224 setup_rss(sc);
1225 sc->flags |= FULL_INIT_DONE;
1226 }
1227
1228 t3_intr_clear(sc);
1229
1230 /* If it's MSI or INTx, allocate a single interrupt for everything */
1231 if ((sc->flags & USING_MSIX) == 0) {
1232 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
1233 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
1234 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid);
1235 err = EINVAL;
1236 goto out;
1237 }
1238 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
1239
1240 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
1241#ifdef INTR_FILTERS
1242 NULL,
1243#endif
1244 sc->cxgb_intr, sc, &sc->intr_tag)) {
1245 device_printf(sc->dev, "Cannot set up interrupt\n");
1246 err = EINVAL;
1247 goto irq_err;
1248 }
1249 } else {
1250 cxgb_setup_msix(sc, sc->msi_count);
1251 }
1252
1253 t3_sge_start(sc);
1254 t3_intr_enable(sc);
1255
1256 if ((sc->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
1257 bind_qsets(sc);
1258 sc->flags |= QUEUES_BOUND;
1259out:
1260 return (err);
1261irq_err:
1262 CH_ERR(sc, "request_irq failed, err %d\n", err);
1263 goto out;
1264}
1265
1266
1267/*
1268 * Release resources when all the ports and offloading have been stopped.
1269 */
1270static void
1271cxgb_down(struct adapter *sc)
1272{
1273 int i;
1274
1275 t3_sge_stop(sc);
1276 ADAPTER_LOCK(sc);
1277 t3_intr_disable(sc);
1278 ADAPTER_UNLOCK(sc);
1279
1280
1281 if (sc->intr_tag != NULL) {
1282 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
1283 sc->intr_tag = NULL;
1284 }
1285 if (sc->irq_res != NULL) {
1286 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
1287 sc->irq_rid, sc->irq_res);
1288 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
1289 sc->irq_res);
1290 sc->irq_res = NULL;
1291 }
1292
1293 if (sc->flags & USING_MSIX)
1294 cxgb_teardown_msix(sc);
1295
1296 callout_drain(&sc->sge_timer_ch);
1297 if (sc->tq != NULL)
1298 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1299 for (i = 0; i < sc->params.nports; i++)
1300 if (sc->port[i].tq != NULL)
1301 taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task);
1302
1303}
1304
1305static int
1306offload_open(struct port_info *pi)
1307{
1308 struct adapter *adapter = pi->adapter;
1309 struct toedev *tdev = TOEDEV(pi->ifp);
1310 int adap_up = adapter->open_device_map & PORT_MASK;
1311 int err = 0;
1312
1313 if (atomic_cmpset_int(&adapter->open_device_map,
1314 (adapter->open_device_map & ~OFFLOAD_DEVMAP_BIT),
1315 (adapter->open_device_map | OFFLOAD_DEVMAP_BIT)) == 0)
1316 return (0);
1317
1318 ADAPTER_LOCK(pi->adapter);
1319 if (!adap_up)
1320 err = cxgb_up(adapter);
1321 ADAPTER_UNLOCK(pi->adapter);
1322 if (err < 0)
1323 return (err);
1324
1325 t3_tp_set_offload_mode(adapter, 1);
1326 tdev->lldev = adapter->port[0].ifp;
1327 err = cxgb_offload_activate(adapter);
1328 if (err)
1329 goto out;
1330
1331 init_port_mtus(adapter);
1332 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1333 adapter->params.b_wnd,
1334 adapter->params.rev == 0 ?
1335 adapter->port[0].ifp->if_mtu : 0xffff);
1336 init_smt(adapter);
1337
1338 /* Call back all registered clients */
1339 cxgb_add_clients(tdev);
1340
1341out:
1342 /* restore them in case the offload module has changed them */
1343 if (err) {
1344 t3_tp_set_offload_mode(adapter, 0);
1345 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1346 cxgb_set_dummy_ops(tdev);
1347 }
1348 return (err);
1349}
1350#ifdef notyet
1351static int
1352offload_close(struct toedev *tdev)
1353{
1354 struct adapter *adapter = tdev2adap(tdev);
1355
1356 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1357 return (0);
1358
1359 /* Call back all registered clients */
1360 cxgb_remove_clients(tdev);
1361 tdev->lldev = NULL;
1362 cxgb_set_dummy_ops(tdev);
1363 t3_tp_set_offload_mode(adapter, 0);
1364 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1365
1366 ADAPTER_LOCK(adapter);
1367 if (!adapter->open_device_map)
1368 cxgb_down(adapter);
1369 ADAPTER_UNLOCK(adapter);
1370
1371 cxgb_offload_deactivate(adapter);
1372 return (0);
1373}
1374#endif
1375
1376static void
1377cxgb_init(void *arg)
1378{
1379 struct port_info *p = arg;
1380
1381 PORT_LOCK(p);
1382 cxgb_init_locked(p);
1383 PORT_UNLOCK(p);
1384}
1385
1386static void
1387cxgb_init_locked(struct port_info *p)
1388{
1389 struct ifnet *ifp;
1390 adapter_t *sc = p->adapter;
1391 int err;
1392
1393 mtx_assert(&p->lock, MA_OWNED);
1394 ifp = p->ifp;
1395
1396 ADAPTER_LOCK(p->adapter);
1397 if ((sc->open_device_map == 0) && ((err = cxgb_up(sc)) < 0)) {
1398 ADAPTER_UNLOCK(p->adapter);
1399 cxgb_stop_locked(p);
1400 return;
1401 }
1402 if (p->adapter->open_device_map == 0)
1403 t3_intr_clear(sc);
1404
1405 setbit(&p->adapter->open_device_map, p->port);
1406 ADAPTER_UNLOCK(p->adapter);
1407
1408 if (is_offload(sc) && !ofld_disable) {
1409 err = offload_open(p);
1410 if (err)
1411 log(LOG_WARNING,
1412 "Could not initialize offload capabilities\n");
1413 }
1414 cxgb_link_start(p);
1415 t3_link_changed(sc, p->port);
1416 ifp->if_baudrate = p->link_config.speed * 1000000;
1417
1418 t3_port_intr_enable(sc, p->port);
1419
1420 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1421 cxgb_tick, sc);
1422
1423 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1424 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1425}
1426
1427static void
1428cxgb_set_rxmode(struct port_info *p)
1429{
1430 struct t3_rx_mode rm;
1431 struct cmac *mac = &p->mac;
1432
1433 mtx_assert(&p->lock, MA_OWNED);
1434
1435 t3_init_rx_mode(&rm, p);
1436 t3_mac_set_rx_mode(mac, &rm);
1437}
1438
1439static void
1440cxgb_stop_locked(struct port_info *p)
1441{
1442 struct ifnet *ifp;
1443
1444 mtx_assert(&p->lock, MA_OWNED);
1445 mtx_assert(&p->adapter->lock, MA_NOTOWNED);
1446
1447 ifp = p->ifp;
1448
1449 t3_port_intr_disable(p->adapter, p->port);
1450 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1451 p->phy.ops->power_down(&p->phy, 1);
1452 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1453
1454 ADAPTER_LOCK(p->adapter);
1455 clrbit(&p->adapter->open_device_map, p->port);
1456 /*
1457 * XXX cancel check_task
1458 */
1459 if (p->adapter->open_device_map == 0)
1460 cxgb_down(p->adapter);
1461 ADAPTER_UNLOCK(p->adapter);
1462}
1463
1464static int
1465cxgb_set_mtu(struct port_info *p, int mtu)
1466{
1467 struct ifnet *ifp = p->ifp;
1468 int error = 0;
1469
1470 if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO))
1471 error = EINVAL;
1472 else if (ifp->if_mtu != mtu) {
1473 PORT_LOCK(p);
1474 ifp->if_mtu = mtu;
1475 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1476 callout_stop(&p->adapter->cxgb_tick_ch);
1477 cxgb_stop_locked(p);
1478 cxgb_init_locked(p);
1479 }
1480 PORT_UNLOCK(p);
1481 }
1482 return (error);
1483}
1484
1485static int
1486cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1487{
1488 struct port_info *p = ifp->if_softc;
1489 struct ifaddr *ifa = (struct ifaddr *)data;
1490 struct ifreq *ifr = (struct ifreq *)data;
1491 int flags, error = 0;
1492 uint32_t mask;
1493
1494 /*
1495 * XXX need to check that we aren't in the middle of an unload
1496 */
1497 switch (command) {
1498 case SIOCSIFMTU:
1499 error = cxgb_set_mtu(p, ifr->ifr_mtu);
1500 break;
1501 case SIOCSIFADDR:
1502 case SIOCGIFADDR:
1503 PORT_LOCK(p);
1504 if (ifa->ifa_addr->sa_family == AF_INET) {
1505 ifp->if_flags |= IFF_UP;
1506 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1507 cxgb_init_locked(p);
1508 arp_ifinit(ifp, ifa);
1509 } else
1510 error = ether_ioctl(ifp, command, data);
1511 PORT_UNLOCK(p);
1512 break;
1513 case SIOCSIFFLAGS:
1514 PORT_LOCK(p);
1515 if (ifp->if_flags & IFF_UP) {
1516 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1517 flags = p->if_flags;
1518 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1519 ((ifp->if_flags ^ flags) & IFF_ALLMULTI))
1520 cxgb_set_rxmode(p);
1521
1522 } else
1523 cxgb_init_locked(p);
1524 p->if_flags = ifp->if_flags;
1525 } else {
1526 callout_stop(&p->adapter->cxgb_tick_ch);
1527 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1528 cxgb_stop_locked(p);
1529 } else {
1530 adapter_t *sc = p->adapter;
1531 callout_reset(&sc->cxgb_tick_ch,
1532 sc->params.stats_update_period * hz,
1533 cxgb_tick, sc);
1534 }
1535 }
1536 PORT_UNLOCK(p);
1537 break;
1538 case SIOCSIFMEDIA:
1539 case SIOCGIFMEDIA:
1540 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
1541 break;
1542 case SIOCSIFCAP:
1543 PORT_LOCK(p);
1544 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1545 if (mask & IFCAP_TXCSUM) {
1546 if (IFCAP_TXCSUM & ifp->if_capenable) {
1547 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
1548 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
1549 | CSUM_TSO);
1550 } else {
1551 ifp->if_capenable |= IFCAP_TXCSUM;
1552 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1553 }
1554 } else if (mask & IFCAP_RXCSUM) {
1555 if (IFCAP_RXCSUM & ifp->if_capenable) {
1556 ifp->if_capenable &= ~IFCAP_RXCSUM;
1557 } else {
1558 ifp->if_capenable |= IFCAP_RXCSUM;
1559 }
1560 }
1561 if (mask & IFCAP_TSO4) {
1562 if (IFCAP_TSO4 & ifp->if_capenable) {
1563 ifp->if_capenable &= ~IFCAP_TSO4;
1564 ifp->if_hwassist &= ~CSUM_TSO;
1565 } else if (IFCAP_TXCSUM & ifp->if_capenable) {
1566 ifp->if_capenable |= IFCAP_TSO4;
1567 ifp->if_hwassist |= CSUM_TSO;
1568 } else {
1569 if (cxgb_debug)
1570 printf("cxgb requires tx checksum offload"
1571 " be enabled to use TSO\n");
1572 error = EINVAL;
1573 }
1574 }
1575 PORT_UNLOCK(p);
1576 break;
1577 default:
1578 error = ether_ioctl(ifp, command, data);
1579 break;
1580 }
1581 return (error);
1582}
1583
1584static int
1585cxgb_start_tx(struct ifnet *ifp, uint32_t txmax)
1586{
1587 struct sge_qset *qs;
1588 struct sge_txq *txq;
1589 struct port_info *p = ifp->if_softc;
1590 struct mbuf *m0, *m = NULL;
1591 int err, in_use_init;
1592
1593 if (!p->link_config.link_ok)
1594 return (ENXIO);
1595
1596 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1597 return (ENOBUFS);
1598
1599 qs = &p->adapter->sge.qs[p->first_qset];
1600 txq = &qs->txq[TXQ_ETH];
1601 err = 0;
1602
1603 mtx_lock(&txq->lock);
1604 in_use_init = txq->in_use;
1605 while ((txq->in_use - in_use_init < txmax) &&
1606 (txq->size > txq->in_use + TX_MAX_DESC)) {
1607 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
1608 if (m == NULL)
1609 break;
1610 /*
1611 * Convert chain to M_IOVEC
1612 */
1613 KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early"));
1614 m0 = m;
1615#ifdef INVARIANTS
1616 /*
1617 * Clean up after net stack sloppiness
1618 * before calling m_sanity
1619 */
1620 m0 = m->m_next;
1621 while (m0) {
1622 m0->m_flags &= ~M_PKTHDR;
1623 m0 = m0->m_next;
1624 }
1625 m_sanity(m0, 0);
1626 m0 = m;
1627#endif
1628 if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES &&
1629 m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) {
1630 if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) {
1631 m = m0;
1632 m_collapse(m, TX_MAX_SEGS, &m0);
1633 } else
1634 break;
1635 }
1636 m = m0;
1637 if ((err = t3_encap(p, &m)) != 0)
1638 break;
1639 BPF_MTAP(ifp, m);
1640 }
1641 mtx_unlock(&txq->lock);
1642
1643 if (__predict_false(err)) {
1644 if (err == ENOMEM) {
1645 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1646 IFQ_LOCK(&ifp->if_snd);
1647 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1648 IFQ_UNLOCK(&ifp->if_snd);
1649 }
1650 }
1651 if (err == 0 && m == NULL)
1652 err = ENOBUFS;
1653 else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) &&
1654 (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
1655 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1656 err = ENOSPC;
1657 }
1658 return (err);
1659}
1660
1661static void
1662cxgb_start_proc(void *arg, int ncount)
1663{
1664 struct ifnet *ifp = arg;
1665 struct port_info *pi = ifp->if_softc;
1666 struct sge_qset *qs;
1667 struct sge_txq *txq;
1668 int error;
1669
1670 qs = &pi->adapter->sge.qs[pi->first_qset];
1671 txq = &qs->txq[TXQ_ETH];
1672
1673 do {
1674 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1675 taskqueue_enqueue(pi->adapter->tq,
1676 &pi->timer_reclaim_task);
1677
1678 error = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1679 } while (error == 0);
1680}
1681
1682static void
1683cxgb_start(struct ifnet *ifp)
1684{
1685 struct port_info *pi = ifp->if_softc;
1686 struct sge_qset *qs;
1687 struct sge_txq *txq;
1688 int err;
1689
1690 qs = &pi->adapter->sge.qs[pi->first_qset];
1691 txq = &qs->txq[TXQ_ETH];
1692
1693 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1694 taskqueue_enqueue(pi->adapter->tq,
1695 &pi->timer_reclaim_task);
1696
1697 err = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1698
1699 if (err == 0)
1700 taskqueue_enqueue(pi->tq, &pi->start_task);
1701}
1702
1703
1704static int
1705cxgb_media_change(struct ifnet *ifp)
1706{
1707 if_printf(ifp, "media change not supported\n");
1708 return (ENXIO);
1709}
1710
1711static void
1712cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1713{
1714 struct port_info *p = ifp->if_softc;
1715
1716 ifmr->ifm_status = IFM_AVALID;
1717 ifmr->ifm_active = IFM_ETHER;
1718
1719 if (!p->link_config.link_ok)
1720 return;
1721
1722 ifmr->ifm_status |= IFM_ACTIVE;
1723
1724 switch (p->link_config.speed) {
1725 case 10:
1726 ifmr->ifm_active |= IFM_10_T;
1727 break;
1728 case 100:
1729 ifmr->ifm_active |= IFM_100_TX;
1730 break;
1731 case 1000:
1732 ifmr->ifm_active |= IFM_1000_T;
1733 break;
1734 }
1735
1736 if (p->link_config.duplex)
1737 ifmr->ifm_active |= IFM_FDX;
1738 else
1739 ifmr->ifm_active |= IFM_HDX;
1740}
1741
1742static void
1743cxgb_async_intr(void *data)
1744{
1745 adapter_t *sc = data;
1746
1747 if (cxgb_debug)
1748 device_printf(sc->dev, "cxgb_async_intr\n");
1749
1750 t3_slow_intr_handler(sc);
1751
1752}
1753
1754static void
1755cxgb_ext_intr_handler(void *arg, int count)
1756{
1757 adapter_t *sc = (adapter_t *)arg;
1758
1759 if (cxgb_debug)
1760 printf("cxgb_ext_intr_handler\n");
1761
1762 t3_phy_intr_handler(sc);
1763
1764 /* Now reenable external interrupts */
1765 ADAPTER_LOCK(sc);
1766 if (sc->slow_intr_mask) {
1767 sc->slow_intr_mask |= F_T3DBG;
1768 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
1769 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1770 }
1771 ADAPTER_UNLOCK(sc);
1772}
1773
1774static void
1775check_link_status(adapter_t *sc)
1776{
1777 int i;
1778
1779 for (i = 0; i < (sc)->params.nports; ++i) {
1780 struct port_info *p = &sc->port[i];
1781
1782 if (!(p->port_type->caps & SUPPORTED_IRQ))
1783 t3_link_changed(sc, i);
1784 p->ifp->if_baudrate = p->link_config.speed * 1000000;
1785 }
1786}
1787
1788static void
1789check_t3b2_mac(struct adapter *adapter)
1790{
1791 int i;
1792
1793 for_each_port(adapter, i) {
1794 struct port_info *p = &adapter->port[i];
1795 struct ifnet *ifp = p->ifp;
1796 int status;
1797
1798 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1799 continue;
1800
1801 status = 0;
1802 PORT_LOCK(p);
1803 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
1804 status = t3b2_mac_watchdog_task(&p->mac);
1805 if (status == 1)
1806 p->mac.stats.num_toggled++;
1807 else if (status == 2) {
1808 struct cmac *mac = &p->mac;
1809
1810 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN
1811 + ETHER_VLAN_ENCAP_LEN);
1812 t3_mac_set_address(mac, 0, p->hw_addr);
1813 cxgb_set_rxmode(p);
1814 t3_link_start(&p->phy, mac, &p->link_config);
1815 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1816 t3_port_intr_enable(adapter, p->port);
1817 p->mac.stats.num_resets++;
1818 }
1819 PORT_UNLOCK(p);
1820 }
1821}
1822
1823static void
1824cxgb_tick(void *arg)
1825{
1826 adapter_t *sc = (adapter_t *)arg;
1827 const struct adapter_params *p = &sc->params;
1828
1829 if (p->linkpoll_period)
1830 check_link_status(sc);
1831 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1832 cxgb_tick, sc);
1833
1834 /*
1835 * adapter lock can currently only be acquire after the
1836 * port lock
1837 */
1838 ADAPTER_UNLOCK(sc);
1839
1840 if (p->rev == T3_REV_B2)
1841 check_t3b2_mac(sc);
1842}
1843
1844static int
1845in_range(int val, int lo, int hi)
1846{
1847 return val < 0 || (val <= hi && val >= lo);
1848}
1849
1850static int
1851cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td)
1852{
1853 return (0);
1854}
1855
1856static int
1857cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
1858{
1859 return (0);
1860}
1861
1862static int
1863cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
1864 int fflag, struct thread *td)
1865{
1866 int mmd, error = 0;
1867 struct port_info *pi = dev->si_drv1;
1868 adapter_t *sc = pi->adapter;
1869
1870#ifdef PRIV_SUPPORTED
1871 if (priv_check(td, PRIV_DRIVER)) {
1872 if (cxgb_debug)
1873 printf("user does not have access to privileged ioctls\n");
1874 return (EPERM);
1875 }
1876#else
1877 if (suser(td)) {
1878 if (cxgb_debug)
1879 printf("user does not have access to privileged ioctls\n");
1880 return (EPERM);
1881 }
1882#endif
1883
1884 switch (cmd) {
1885 case SIOCGMIIREG: {
1886 uint32_t val;
1887 struct cphy *phy = &pi->phy;
1888 struct mii_data *mid = (struct mii_data *)data;
1889
1890 if (!phy->mdio_read)
1891 return (EOPNOTSUPP);
1892 if (is_10G(sc)) {
1893 mmd = mid->phy_id >> 8;
1894 if (!mmd)
1895 mmd = MDIO_DEV_PCS;
1896 else if (mmd > MDIO_DEV_XGXS)
1897 return -EINVAL;
1898
1899 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
1900 mid->reg_num, &val);
1901 } else
1902 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
1903 mid->reg_num & 0x1f, &val);
1904 if (error == 0)
1905 mid->val_out = val;
1906 break;
1907 }
1908 case SIOCSMIIREG: {
1909 struct cphy *phy = &pi->phy;
1910 struct mii_data *mid = (struct mii_data *)data;
1911
1912 if (!phy->mdio_write)
1913 return (EOPNOTSUPP);
1914 if (is_10G(sc)) {
1915 mmd = mid->phy_id >> 8;
1916 if (!mmd)
1917 mmd = MDIO_DEV_PCS;
1918 else if (mmd > MDIO_DEV_XGXS)
1919 return (EINVAL);
1920
1921 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
1922 mmd, mid->reg_num, mid->val_in);
1923 } else
1924 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
1925 mid->reg_num & 0x1f,
1926 mid->val_in);
1927 break;
1928 }
1929 case CHELSIO_SETREG: {
1930 struct ch_reg *edata = (struct ch_reg *)data;
1931 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1932 return (EFAULT);
1933 t3_write_reg(sc, edata->addr, edata->val);
1934 break;
1935 }
1936 case CHELSIO_GETREG: {
1937 struct ch_reg *edata = (struct ch_reg *)data;
1938 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1939 return (EFAULT);
1940 edata->val = t3_read_reg(sc, edata->addr);
1941 break;
1942 }
1943 case CHELSIO_GET_SGE_CONTEXT: {
1944 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
1945 mtx_lock(&sc->sge.reg_lock);
1946 switch (ecntxt->cntxt_type) {
1947 case CNTXT_TYPE_EGRESS:
1948 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
1949 ecntxt->data);
1950 break;
1951 case CNTXT_TYPE_FL:
1952 error = t3_sge_read_fl(sc, ecntxt->cntxt_id,
1953 ecntxt->data);
1954 break;
1955 case CNTXT_TYPE_RSP:
1956 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id,
1957 ecntxt->data);
1958 break;
1959 case CNTXT_TYPE_CQ:
1960 error = t3_sge_read_cq(sc, ecntxt->cntxt_id,
1961 ecntxt->data);
1962 break;
1963 default:
1964 error = EINVAL;
1965 break;
1966 }
1967 mtx_unlock(&sc->sge.reg_lock);
1968 break;
1969 }
1970 case CHELSIO_GET_SGE_DESC: {
1971 struct ch_desc *edesc = (struct ch_desc *)data;
1972 int ret;
1973 if (edesc->queue_num >= SGE_QSETS * 6)
1974 return (EINVAL);
1975 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
1976 edesc->queue_num % 6, edesc->idx, edesc->data);
1977 if (ret < 0)
1978 return (EINVAL);
1979 edesc->size = ret;
1980 break;
1981 }
1982 case CHELSIO_SET_QSET_PARAMS: {
1983 struct qset_params *q;
1984 struct ch_qset_params *t = (struct ch_qset_params *)data;
1985
1986 if (t->qset_idx >= SGE_QSETS)
1987 return -EINVAL;
1988 if (!in_range(t->intr_lat, 0, M_NEWTIMER) ||
1989 !in_range(t->cong_thres, 0, 255) ||
1990 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES,
1991 MAX_TXQ_ENTRIES) ||
1992 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES,
1993 MAX_TXQ_ENTRIES) ||
1994 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1995 MAX_CTRL_TXQ_ENTRIES) ||
1996 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) ||
1997 !in_range(t->fl_size[1], MIN_FL_ENTRIES,
1998 MAX_RX_JUMBO_BUFFERS) ||
1999 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
2000 return -EINVAL;
2001 if ((sc->flags & FULL_INIT_DONE) &&
2002 (t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
2003 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||
2004 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 ||
2005 t->polling >= 0 || t->cong_thres >= 0))
2006 return -EBUSY;
2007
2008 q = &sc->params.sge.qset[t->qset_idx];
2009
2010 if (t->rspq_size >= 0)
2011 q->rspq_size = t->rspq_size;
2012 if (t->fl_size[0] >= 0)
2013 q->fl_size = t->fl_size[0];
2014 if (t->fl_size[1] >= 0)
2015 q->jumbo_size = t->fl_size[1];
2016 if (t->txq_size[0] >= 0)
2017 q->txq_size[0] = t->txq_size[0];
2018 if (t->txq_size[1] >= 0)
2019 q->txq_size[1] = t->txq_size[1];
2020 if (t->txq_size[2] >= 0)
2021 q->txq_size[2] = t->txq_size[2];
2022 if (t->cong_thres >= 0)
2023 q->cong_thres = t->cong_thres;
2024 if (t->intr_lat >= 0) {
2025 struct sge_qset *qs = &sc->sge.qs[t->qset_idx];
2026
2027 q->coalesce_nsecs = t->intr_lat*1000;
2028 t3_update_qset_coalesce(qs, q);
2029 }
2030 break;
2031 }
2032 case CHELSIO_GET_QSET_PARAMS: {
2033 struct qset_params *q;
2034 struct ch_qset_params *t = (struct ch_qset_params *)data;
2035
2036 if (t->qset_idx >= SGE_QSETS)
2037 return (EINVAL);
2038
2039 q = &(sc)->params.sge.qset[t->qset_idx];
2040 t->rspq_size = q->rspq_size;
2041 t->txq_size[0] = q->txq_size[0];
2042 t->txq_size[1] = q->txq_size[1];
2043 t->txq_size[2] = q->txq_size[2];
2044 t->fl_size[0] = q->fl_size;
2045 t->fl_size[1] = q->jumbo_size;
2046 t->polling = q->polling;
2047 t->intr_lat = q->coalesce_nsecs / 1000;
2048 t->cong_thres = q->cong_thres;
2049 break;
2050 }
2051 case CHELSIO_SET_QSET_NUM: {
2052 struct ch_reg *edata = (struct ch_reg *)data;
2053 unsigned int port_idx = pi->port;
2054
2055 if (sc->flags & FULL_INIT_DONE)
2056 return (EBUSY);
2057 if (edata->val < 1 ||
2058 (edata->val > 1 && !(sc->flags & USING_MSIX)))
2059 return (EINVAL);
2060 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS)
2061 return (EINVAL);
2062 sc->port[port_idx].nqsets = edata->val;
2063 sc->port[0].first_qset = 0;
2064 /*
2065 * XXX hardcode ourselves to 2 ports just like LEEENUX
2066 */
2067 sc->port[1].first_qset = sc->port[0].nqsets;
2068 break;
2069 }
2070 case CHELSIO_GET_QSET_NUM: {
2071 struct ch_reg *edata = (struct ch_reg *)data;
2072 edata->val = pi->nqsets;
2073 break;
2074 }
2075#ifdef notyet
2076 case CHELSIO_LOAD_FW:
2077 case CHELSIO_GET_PM:
2078 case CHELSIO_SET_PM:
2079 return (EOPNOTSUPP);
2080 break;
2081#endif
2082 case CHELSIO_SETMTUTAB: {
2083 struct ch_mtus *m = (struct ch_mtus *)data;
2084 int i;
2085
2086 if (!is_offload(sc))
2087 return (EOPNOTSUPP);
2088 if (offload_running(sc))
2089 return (EBUSY);
2090 if (m->nmtus != NMTUS)
2091 return (EINVAL);
2092 if (m->mtus[0] < 81) /* accommodate SACK */
2093 return (EINVAL);
2094
2095 /*
2096 * MTUs must be in ascending order
2097 */
2098 for (i = 1; i < NMTUS; ++i)
2099 if (m->mtus[i] < m->mtus[i - 1])
2100 return (EINVAL);
2101
2102 memcpy(sc->params.mtus, m->mtus,
2103 sizeof(sc->params.mtus));
2104 break;
2105 }
2106 case CHELSIO_GETMTUTAB: {
2107 struct ch_mtus *m = (struct ch_mtus *)data;
2108
2109 if (!is_offload(sc))
2110 return (EOPNOTSUPP);
2111
2112 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2113 m->nmtus = NMTUS;
2114 break;
2115 }
2116 case CHELSIO_DEVUP:
2117 if (!is_offload(sc))
2118 return (EOPNOTSUPP);
2119 return offload_open(pi);
2120 break;
2121 case CHELSIO_GET_MEM: {
2122 struct ch_mem_range *t = (struct ch_mem_range *)data;
2123 struct mc7 *mem;
2124 uint8_t *useraddr;
2125 u64 buf[32];
2126
2127 if (!is_offload(sc))
2128 return (EOPNOTSUPP);
2129 if (!(sc->flags & FULL_INIT_DONE))
2130 return (EIO); /* need the memory controllers */
2131 if ((t->addr & 0x7) || (t->len & 0x7))
2132 return (EINVAL);
2133 if (t->mem_id == MEM_CM)
2134 mem = &sc->cm;
2135 else if (t->mem_id == MEM_PMRX)
2136 mem = &sc->pmrx;
2137 else if (t->mem_id == MEM_PMTX)
2138 mem = &sc->pmtx;
2139 else
2140 return (EINVAL);
2141
2142 /*
2143 * Version scheme:
2144 * bits 0..9: chip version
2145 * bits 10..15: chip revision
2146 */
2147 t->version = 3 | (sc->params.rev << 10);
2148
2149 /*
2150 * Read 256 bytes at a time as len can be large and we don't
2151 * want to use huge intermediate buffers.
2152 */
2153 useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */
2154 while (t->len) {
2155 unsigned int chunk = min(t->len, sizeof(buf));
2156
2157 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf);
2158 if (error)
2159 return (-error);
2160 if (copyout(buf, useraddr, chunk))
2161 return (EFAULT);
2162 useraddr += chunk;
2163 t->addr += chunk;
2164 t->len -= chunk;
2165 }
2166 break;
2167 }
2168 case CHELSIO_READ_TCAM_WORD: {
2169 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2170
2171 if (!is_offload(sc))
2172 return (EOPNOTSUPP);
2173 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2174 break;
2175 }
2176 case CHELSIO_SET_TRACE_FILTER: {
2177 struct ch_trace *t = (struct ch_trace *)data;
2178 const struct trace_params *tp;
2179
2180 tp = (const struct trace_params *)&t->sip;
2181 if (t->config_tx)
2182 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2183 t->trace_tx);
2184 if (t->config_rx)
2185 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2186 t->trace_rx);
2187 break;
2188 }
2189 case CHELSIO_SET_PKTSCHED: {
2190 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2191 if (sc->open_device_map == 0)
2192 return (EAGAIN);
2193 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2194 p->binding);
2195 break;
2196 }
2197 case CHELSIO_IFCONF_GETREGS: {
2198 struct ifconf_regs *regs = (struct ifconf_regs *)data;
2199 int reglen = cxgb_get_regs_len();
2200 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT);
2201 if (buf == NULL) {
2202 return (ENOMEM);
2203 } if (regs->len > reglen)
2204 regs->len = reglen;
2205 else if (regs->len < reglen) {
2206 error = E2BIG;
2207 goto done;
2208 }
2209 cxgb_get_regs(sc, regs, buf);
2210 error = copyout(buf, regs->data, reglen);
2211
2212 done:
2213 free(buf, M_DEVBUF);
2214
2215 break;
2216 }
2217 case CHELSIO_SET_HW_SCHED: {
2218 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2219 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2220
2221 if ((sc->flags & FULL_INIT_DONE) == 0)
2222 return (EAGAIN); /* need TP to be initialized */
2223 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2224 !in_range(t->channel, 0, 1) ||
2225 !in_range(t->kbps, 0, 10000000) ||
2226 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2227 !in_range(t->flow_ipg, 0,
2228 dack_ticks_to_usec(sc, 0x7ff)))
2229 return (EINVAL);
2230
2231 if (t->kbps >= 0) {
2232 error = t3_config_sched(sc, t->kbps, t->sched);
2233 if (error < 0)
2234 return (-error);
2235 }
2236 if (t->class_ipg >= 0)
2237 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2238 if (t->flow_ipg >= 0) {
2239 t->flow_ipg *= 1000; /* us -> ns */
2240 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2241 }
2242 if (t->mode >= 0) {
2243 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2244
2245 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2246 bit, t->mode ? bit : 0);
2247 }
2248 if (t->channel >= 0)
2249 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2250 1 << t->sched, t->channel << t->sched);
2251 break;
2252 }
2253 default:
2254 return (EOPNOTSUPP);
2255 break;
2256 }
2257
2258 return (error);
2259}
2260
2261static __inline void
2262reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
2263 unsigned int end)
2264{
2265 uint32_t *p = (uint32_t *)buf + start;
2266
2267 for ( ; start <= end; start += sizeof(uint32_t))
2268 *p++ = t3_read_reg(ap, start);
2269}
2270
2271#define T3_REGMAP_SIZE (3 * 1024)
2272static int
2273cxgb_get_regs_len(void)
2274{
2275 return T3_REGMAP_SIZE;
2276}
2277#undef T3_REGMAP_SIZE
2278
2279static void
2280cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf)
2281{
2282
2283 /*
2284 * Version scheme:
2285 * bits 0..9: chip version
2286 * bits 10..15: chip revision
2287 * bit 31: set for PCIe cards
2288 */
2289 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
2290
2291 /*
2292 * We skip the MAC statistics registers because they are clear-on-read.
2293 * Also reading multi-register stats would need to synchronize with the
2294 * periodic mac stats accumulation. Hard to justify the complexity.
2295 */
2296 memset(buf, 0, REGDUMP_SIZE);
2297 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
2298 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
2299 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
2300 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
2301 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
2302 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
2303 XGM_REG(A_XGM_SERDES_STAT3, 1));
2304 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
2305 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
2306}
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
122. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26POSSIBILITY OF SUCH DAMAGE.
27
28***************************************************************************/
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_main.c 170789 2007-06-15 20:02:02Z kmacy $");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/bus.h>
37#include <sys/module.h>
38#include <sys/pciio.h>
39#include <sys/conf.h>
40#include <machine/bus.h>
41#include <machine/resource.h>
42#include <sys/bus_dma.h>
43#include <sys/rman.h>
44#include <sys/ioccom.h>
45#include <sys/mbuf.h>
46#include <sys/linker.h>
47#include <sys/firmware.h>
48#include <sys/socket.h>
49#include <sys/sockio.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <sys/queue.h>
53#include <sys/taskqueue.h>
54
55#include <net/bpf.h>
56#include <net/ethernet.h>
57#include <net/if.h>
58#include <net/if_arp.h>
59#include <net/if_dl.h>
60#include <net/if_media.h>
61#include <net/if_types.h>
62
63#include <netinet/in_systm.h>
64#include <netinet/in.h>
65#include <netinet/if_ether.h>
66#include <netinet/ip.h>
67#include <netinet/ip.h>
68#include <netinet/tcp.h>
69#include <netinet/udp.h>
70
71#include <dev/pci/pcireg.h>
72#include <dev/pci/pcivar.h>
73#include <dev/pci/pci_private.h>
74
75#ifdef CONFIG_DEFINED
76#include <cxgb_include.h>
77#else
78#include <dev/cxgb/cxgb_include.h>
79#endif
80
81#ifdef PRIV_SUPPORTED
82#include <sys/priv.h>
83#endif
84
85static int cxgb_setup_msix(adapter_t *, int);
86static void cxgb_teardown_msix(adapter_t *);
87static void cxgb_init(void *);
88static void cxgb_init_locked(struct port_info *);
89static void cxgb_stop_locked(struct port_info *);
90static void cxgb_set_rxmode(struct port_info *);
91static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
92static void cxgb_start(struct ifnet *);
93static void cxgb_start_proc(void *, int ncount);
94static int cxgb_media_change(struct ifnet *);
95static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
96static int setup_sge_qsets(adapter_t *);
97static void cxgb_async_intr(void *);
98static void cxgb_ext_intr_handler(void *, int);
99static void cxgb_down(struct adapter *sc);
100static void cxgb_tick(void *);
101static void setup_rss(adapter_t *sc);
102
103/* Attachment glue for the PCI controller end of the device. Each port of
104 * the device is attached separately, as defined later.
105 */
106static int cxgb_controller_probe(device_t);
107static int cxgb_controller_attach(device_t);
108static int cxgb_controller_detach(device_t);
109static void cxgb_free(struct adapter *);
110static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
111 unsigned int end);
112static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf);
113static int cxgb_get_regs_len(void);
114static int offload_open(struct port_info *pi);
115#ifdef notyet
116static int offload_close(struct toedev *tdev);
117#endif
118
119
120static device_method_t cxgb_controller_methods[] = {
121 DEVMETHOD(device_probe, cxgb_controller_probe),
122 DEVMETHOD(device_attach, cxgb_controller_attach),
123 DEVMETHOD(device_detach, cxgb_controller_detach),
124
125 /* bus interface */
126 DEVMETHOD(bus_print_child, bus_generic_print_child),
127 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
128
129 { 0, 0 }
130};
131
132static driver_t cxgb_controller_driver = {
133 "cxgbc",
134 cxgb_controller_methods,
135 sizeof(struct adapter)
136};
137
138static devclass_t cxgb_controller_devclass;
139DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
140
141/*
142 * Attachment glue for the ports. Attachment is done directly to the
143 * controller device.
144 */
145static int cxgb_port_probe(device_t);
146static int cxgb_port_attach(device_t);
147static int cxgb_port_detach(device_t);
148
149static device_method_t cxgb_port_methods[] = {
150 DEVMETHOD(device_probe, cxgb_port_probe),
151 DEVMETHOD(device_attach, cxgb_port_attach),
152 DEVMETHOD(device_detach, cxgb_port_detach),
153 { 0, 0 }
154};
155
156static driver_t cxgb_port_driver = {
157 "cxgb",
158 cxgb_port_methods,
159 0
160};
161
162static d_ioctl_t cxgb_extension_ioctl;
163static d_open_t cxgb_extension_open;
164static d_close_t cxgb_extension_close;
165
166static struct cdevsw cxgb_cdevsw = {
167 .d_version = D_VERSION,
168 .d_flags = 0,
169 .d_open = cxgb_extension_open,
170 .d_close = cxgb_extension_close,
171 .d_ioctl = cxgb_extension_ioctl,
172 .d_name = "cxgb",
173};
174
175static devclass_t cxgb_port_devclass;
176DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
177
178#define SGE_MSIX_COUNT (SGE_QSETS + 1)
179
180extern int collapse_mbufs;
181/*
182 * The driver uses the best interrupt scheme available on a platform in the
183 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
184 * of these schemes the driver may consider as follows:
185 *
186 * msi = 2: choose from among all three options
187 * msi = 1 : only consider MSI and pin interrupts
188 * msi = 0: force pin interrupts
189 */
190static int msi_allowed = 2;
191
192TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
193SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
194SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
195 "MSI-X, MSI, INTx selector");
196
197/*
198 * The driver enables offload as a default.
199 * To disable it, use ofld_disable = 1.
200 */
201static int ofld_disable = 0;
202TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
203SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
204 "disable ULP offload");
205
206/*
207 * The driver uses an auto-queue algorithm by default.
208 * To disable it and force a single queue-set per port, use singleq = 1.
209 */
210static int singleq = 1;
211TUNABLE_INT("hw.cxgb.singleq", &singleq);
212SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
213 "use a single queue-set per port");
214
215enum {
216 MAX_TXQ_ENTRIES = 16384,
217 MAX_CTRL_TXQ_ENTRIES = 1024,
218 MAX_RSPQ_ENTRIES = 16384,
219 MAX_RX_BUFFERS = 16384,
220 MAX_RX_JUMBO_BUFFERS = 16384,
221 MIN_TXQ_ENTRIES = 4,
222 MIN_CTRL_TXQ_ENTRIES = 4,
223 MIN_RSPQ_ENTRIES = 32,
224 MIN_FL_ENTRIES = 32
225};
226
227#define PORT_MASK ((1 << MAX_NPORTS) - 1)
228
229/* Table for probing the cards. The desc field isn't actually used */
230struct cxgb_ident {
231 uint16_t vendor;
232 uint16_t device;
233 int index;
234 char *desc;
235} cxgb_identifiers[] = {
236 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
237 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
238 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
239 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
240 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
241 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
242 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
243 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
244 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
245 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
246 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
247 {0, 0, 0, NULL}
248};
249
250static struct cxgb_ident *
251cxgb_get_ident(device_t dev)
252{
253 struct cxgb_ident *id;
254
255 for (id = cxgb_identifiers; id->desc != NULL; id++) {
256 if ((id->vendor == pci_get_vendor(dev)) &&
257 (id->device == pci_get_device(dev))) {
258 return (id);
259 }
260 }
261 return (NULL);
262}
263
264static const struct adapter_info *
265cxgb_get_adapter_info(device_t dev)
266{
267 struct cxgb_ident *id;
268 const struct adapter_info *ai;
269
270 id = cxgb_get_ident(dev);
271 if (id == NULL)
272 return (NULL);
273
274 ai = t3_get_adapter_info(id->index);
275
276 return (ai);
277}
278
279static int
280cxgb_controller_probe(device_t dev)
281{
282 const struct adapter_info *ai;
283 char *ports, buf[80];
284 int nports;
285
286 ai = cxgb_get_adapter_info(dev);
287 if (ai == NULL)
288 return (ENXIO);
289
290 nports = ai->nports0 + ai->nports1;
291 if (nports == 1)
292 ports = "port";
293 else
294 ports = "ports";
295
296 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports);
297 device_set_desc_copy(dev, buf);
298 return (BUS_PROBE_DEFAULT);
299}
300
301static int
302upgrade_fw(adapter_t *sc)
303{
304 char buf[32];
305#ifdef FIRMWARE_LATEST
306 const struct firmware *fw;
307#else
308 struct firmware *fw;
309#endif
310 int status;
311
312 snprintf(&buf[0], sizeof(buf), "t3fw%d%d%d", FW_VERSION_MAJOR,
313 FW_VERSION_MINOR, FW_VERSION_MICRO);
314
315 fw = firmware_get(buf);
316
317 if (fw == NULL) {
318 device_printf(sc->dev, "Could not find firmware image %s\n", buf);
319 return (ENOENT);
320 }
321
322 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
323
324 firmware_put(fw, FIRMWARE_UNLOAD);
325
326 return (status);
327}
328
329static int
330cxgb_controller_attach(device_t dev)
331{
332 device_t child;
333 const struct adapter_info *ai;
334 struct adapter *sc;
335 int i, reg, msi_needed, error = 0;
336 uint32_t vers;
337 int port_qsets = 1;
338
339 sc = device_get_softc(dev);
340 sc->dev = dev;
341 sc->msi_count = 0;
342
343 /* find the PCIe link width and set max read request to 4KB*/
344 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
345 uint16_t lnk, pectl;
346 lnk = pci_read_config(dev, reg + 0x12, 2);
347 sc->link_width = (lnk >> 4) & 0x3f;
348
349 pectl = pci_read_config(dev, reg + 0x8, 2);
350 pectl = (pectl & ~0x7000) | (5 << 12);
351 pci_write_config(dev, reg + 0x8, pectl, 2);
352 }
353 if (sc->link_width != 0 && sc->link_width <= 4) {
354 device_printf(sc->dev,
355 "PCIe x%d Link, expect reduced performance\n",
356 sc->link_width);
357 }
358
359 pci_enable_busmaster(dev);
360 /*
361 * Allocate the registers and make them available to the driver.
362 * The registers that we care about for NIC mode are in BAR 0
363 */
364 sc->regs_rid = PCIR_BAR(0);
365 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
366 &sc->regs_rid, RF_ACTIVE)) == NULL) {
367 device_printf(dev, "Cannot allocate BAR\n");
368 return (ENXIO);
369 }
370
371 mtx_init(&sc->sge.reg_lock, "SGE reg lock", NULL, MTX_DEF);
372 mtx_init(&sc->lock, "cxgb controller lock", NULL, MTX_DEF);
373 mtx_init(&sc->mdio_lock, "cxgb mdio", NULL, MTX_DEF);
374 mtx_init(&sc->elmer_lock, "cxgb elmer", NULL, MTX_DEF);
375
376 sc->bt = rman_get_bustag(sc->regs_res);
377 sc->bh = rman_get_bushandle(sc->regs_res);
378 sc->mmio_len = rman_get_size(sc->regs_res);
379
380 ai = cxgb_get_adapter_info(dev);
381 if (t3_prep_adapter(sc, ai, 1) < 0) {
382 printf("prep adapter failed\n");
383 error = ENODEV;
384 goto out;
385 }
386 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
387 * enough messages for the queue sets. If that fails, try falling
388 * back to MSI. If that fails, then try falling back to the legacy
389 * interrupt pin model.
390 */
391#ifdef MSI_SUPPORTED
392
393 sc->msix_regs_rid = 0x20;
394 if ((msi_allowed >= 2) &&
395 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
396 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
397
398 msi_needed = sc->msi_count = SGE_MSIX_COUNT;
399
400 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
401 (sc->msi_count != msi_needed)) {
402 device_printf(dev, "msix allocation failed - msi_count = %d"
403 " msi_needed=%d will try msi err=%d\n", sc->msi_count,
404 msi_needed, error);
405 sc->msi_count = 0;
406 pci_release_msi(dev);
407 bus_release_resource(dev, SYS_RES_MEMORY,
408 sc->msix_regs_rid, sc->msix_regs_res);
409 sc->msix_regs_res = NULL;
410 } else {
411 sc->flags |= USING_MSIX;
412 sc->cxgb_intr = t3_intr_msix;
413 }
414 }
415
416 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
417 sc->msi_count = 1;
418 if (pci_alloc_msi(dev, &sc->msi_count)) {
419 device_printf(dev, "alloc msi failed - will try INTx\n");
420 sc->msi_count = 0;
421 pci_release_msi(dev);
422 } else {
423 sc->flags |= USING_MSI;
424 sc->irq_rid = 1;
425 sc->cxgb_intr = t3_intr_msi;
426 }
427 }
428#endif
429 if (sc->msi_count == 0) {
430 device_printf(dev, "using line interrupts\n");
431 sc->irq_rid = 0;
432 sc->cxgb_intr = t3b_intr;
433 }
434
435
436 /* Create a private taskqueue thread for handling driver events */
437#ifdef TASKQUEUE_CURRENT
438 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
439 taskqueue_thread_enqueue, &sc->tq);
440#else
441 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
442 taskqueue_thread_enqueue, &sc->tq);
443#endif
444 if (sc->tq == NULL) {
445 device_printf(dev, "failed to allocate controller task queue\n");
446 goto out;
447 }
448
449 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
450 device_get_nameunit(dev));
451 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
452
453
454 /* Create a periodic callout for checking adapter status */
455 callout_init_mtx(&sc->cxgb_tick_ch, &sc->lock, CALLOUT_RETURNUNLOCKED);
456
457 if (t3_check_fw_version(sc) != 0) {
458 /*
459 * Warn user that a firmware update will be attempted in init.
460 */
461 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
462 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
463 sc->flags &= ~FW_UPTODATE;
464 } else {
465 sc->flags |= FW_UPTODATE;
466 }
467
468 if ((sc->flags & USING_MSIX) && !singleq)
469 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
470
471 /*
472 * Create a child device for each MAC. The ethernet attachment
473 * will be done in these children.
474 */
475 for (i = 0; i < (sc)->params.nports; i++) {
476 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
477 device_printf(dev, "failed to add child port\n");
478 error = EINVAL;
479 goto out;
480 }
481 sc->portdev[i] = child;
482 sc->port[i].adapter = sc;
483 sc->port[i].nqsets = port_qsets;
484 sc->port[i].first_qset = i*port_qsets;
485 sc->port[i].port = i;
486 device_set_softc(child, &sc->port[i]);
487 }
488 if ((error = bus_generic_attach(dev)) != 0)
489 goto out;
490
491 /*
492 * XXX need to poll for link status
493 */
494 sc->params.stats_update_period = 1;
495
496 /* initialize sge private state */
497 t3_sge_init_adapter(sc);
498
499 t3_led_ready(sc);
500
501 cxgb_offload_init();
502 if (is_offload(sc)) {
503 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
504 cxgb_adapter_ofld(sc);
505 }
506 error = t3_get_fw_version(sc, &vers);
507 if (error)
508 goto out;
509
510 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
511 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
512 G_FW_VERSION_MICRO(vers));
513
514 t3_add_sysctls(sc);
515out:
516 if (error)
517 cxgb_free(sc);
518
519 return (error);
520}
521
522static int
523cxgb_controller_detach(device_t dev)
524{
525 struct adapter *sc;
526
527 sc = device_get_softc(dev);
528
529 cxgb_free(sc);
530
531 return (0);
532}
533
534static void
535cxgb_free(struct adapter *sc)
536{
537 int i;
538
539 cxgb_down(sc);
540
541#ifdef MSI_SUPPORTED
542 if (sc->flags & (USING_MSI | USING_MSIX)) {
543 device_printf(sc->dev, "releasing msi message(s)\n");
544 pci_release_msi(sc->dev);
545 } else {
546 device_printf(sc->dev, "no msi message to release\n");
547 }
548#endif
549 if (sc->msix_regs_res != NULL) {
550 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
551 sc->msix_regs_res);
552 }
553
554 /*
555 * XXX need to drain the ifq by hand until
556 * it is taught about mbuf iovecs
557 */
558 callout_drain(&sc->cxgb_tick_ch);
559
560 t3_sge_deinit_sw(sc);
561
562 if (sc->tq != NULL) {
563 taskqueue_drain(sc->tq, &sc->ext_intr_task);
564 taskqueue_free(sc->tq);
565 }
566
567 for (i = 0; i < (sc)->params.nports; ++i) {
568 if (sc->portdev[i] != NULL)
569 device_delete_child(sc->dev, sc->portdev[i]);
570 }
571
572 bus_generic_detach(sc->dev);
573#ifdef notyet
574 if (is_offload(sc)) {
575 cxgb_adapter_unofld(sc);
576 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
577 offload_close(&sc->tdev);
578 }
579#endif
580 t3_free_sge_resources(sc);
581 t3_sge_free(sc);
582
583 if (sc->regs_res != NULL)
584 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
585 sc->regs_res);
586
587 mtx_destroy(&sc->mdio_lock);
588 mtx_destroy(&sc->sge.reg_lock);
589 mtx_destroy(&sc->lock);
590
591 return;
592}
593
594/**
595 * setup_sge_qsets - configure SGE Tx/Rx/response queues
596 * @sc: the controller softc
597 *
598 * Determines how many sets of SGE queues to use and initializes them.
599 * We support multiple queue sets per port if we have MSI-X, otherwise
600 * just one queue set per port.
601 */
602static int
603setup_sge_qsets(adapter_t *sc)
604{
605 int i, j, err, irq_idx, qset_idx;
606 u_int ntxq = SGE_TXQ_PER_SET;
607
608 if ((err = t3_sge_alloc(sc)) != 0) {
609 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
610 return (err);
611 }
612
613 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
614 irq_idx = -1;
615 else
616 irq_idx = 0;
617
618 for (qset_idx = 0, i = 0; i < (sc)->params.nports; ++i) {
619 struct port_info *pi = &sc->port[i];
620
621 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
622 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
623 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
624 &sc->params.sge.qset[qset_idx], ntxq, pi);
625 if (err) {
626 t3_free_sge_resources(sc);
627 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", err);
628 return (err);
629 }
630 }
631 }
632
633 return (0);
634}
635
636static void
637cxgb_teardown_msix(adapter_t *sc)
638{
639 int i, nqsets;
640
641 for (nqsets = i = 0; i < (sc)->params.nports; i++)
642 nqsets += sc->port[i].nqsets;
643
644 for (i = 0; i < nqsets; i++) {
645 if (sc->msix_intr_tag[i] != NULL) {
646 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
647 sc->msix_intr_tag[i]);
648 sc->msix_intr_tag[i] = NULL;
649 }
650 if (sc->msix_irq_res[i] != NULL) {
651 bus_release_resource(sc->dev, SYS_RES_IRQ,
652 sc->msix_irq_rid[i], sc->msix_irq_res[i]);
653 sc->msix_irq_res[i] = NULL;
654 }
655 }
656}
657
658static int
659cxgb_setup_msix(adapter_t *sc, int msix_count)
660{
661 int i, j, k, nqsets, rid;
662
663 /* The first message indicates link changes and error conditions */
664 sc->irq_rid = 1;
665 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
666 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
667 device_printf(sc->dev, "Cannot allocate msix interrupt\n");
668 return (EINVAL);
669 }
670
671 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
672#ifdef INTR_FILTERS
673 NULL,
674#endif
675 cxgb_async_intr, sc, &sc->intr_tag)) {
676 device_printf(sc->dev, "Cannot set up interrupt\n");
677 return (EINVAL);
678 }
679 for (i = k = 0; i < (sc)->params.nports; i++) {
680 nqsets = sc->port[i].nqsets;
681 for (j = 0; j < nqsets; j++, k++) {
682 struct sge_qset *qs = &sc->sge.qs[k];
683
684 rid = k + 2;
685 if (cxgb_debug)
686 printf("rid=%d ", rid);
687 if ((sc->msix_irq_res[k] = bus_alloc_resource_any(
688 sc->dev, SYS_RES_IRQ, &rid,
689 RF_SHAREABLE | RF_ACTIVE)) == NULL) {
690 device_printf(sc->dev, "Cannot allocate "
691 "interrupt for message %d\n", rid);
692 return (EINVAL);
693 }
694 sc->msix_irq_rid[k] = rid;
695 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
696 INTR_MPSAFE|INTR_TYPE_NET,
697#ifdef INTR_FILTERS
698 NULL,
699#endif
700 t3_intr_msix, qs, &sc->msix_intr_tag[k])) {
701 device_printf(sc->dev, "Cannot set up "
702 "interrupt for message %d\n", rid);
703 return (EINVAL);
704 }
705 }
706 }
707
708
709 return (0);
710}
711
712static int
713cxgb_port_probe(device_t dev)
714{
715 struct port_info *p;
716 char buf[80];
717
718 p = device_get_softc(dev);
719
720 snprintf(buf, sizeof(buf), "Port %d %s", p->port, p->port_type->desc);
721 device_set_desc_copy(dev, buf);
722 return (0);
723}
724
725
726static int
727cxgb_makedev(struct port_info *pi)
728{
729
730 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
731 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
732
733 if (pi->port_cdev == NULL)
734 return (ENOMEM);
735
736 pi->port_cdev->si_drv1 = (void *)pi;
737
738 return (0);
739}
740
741
742#ifdef TSO_SUPPORTED
743#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU)
744/* Don't enable TSO6 yet */
745#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU)
746#else
747#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
748/* Don't enable TSO6 yet */
749#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
750#define IFCAP_TSO4 0x0
751#define CSUM_TSO 0x0
752#endif
753
754
755static int
756cxgb_port_attach(device_t dev)
757{
758 struct port_info *p;
759 struct ifnet *ifp;
760 int err, media_flags;
761 char buf[64];
762
763 p = device_get_softc(dev);
764
765 snprintf(buf, sizeof(buf), "cxgb port %d", p->port);
766 mtx_init(&p->lock, buf, 0, MTX_DEF);
767
768 /* Allocate an ifnet object and set it up */
769 ifp = p->ifp = if_alloc(IFT_ETHER);
770 if (ifp == NULL) {
771 device_printf(dev, "Cannot allocate ifnet\n");
772 return (ENOMEM);
773 }
774
775 /*
776 * Note that there is currently no watchdog timer.
777 */
778 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
779 ifp->if_init = cxgb_init;
780 ifp->if_softc = p;
781 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
782 ifp->if_ioctl = cxgb_ioctl;
783 ifp->if_start = cxgb_start;
784 ifp->if_timer = 0; /* Disable ifnet watchdog */
785 ifp->if_watchdog = NULL;
786
787 ifp->if_snd.ifq_drv_maxlen = TX_ETH_Q_SIZE;
788 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
789 IFQ_SET_READY(&ifp->if_snd);
790
791 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
792 ifp->if_capabilities |= CXGB_CAP;
793 ifp->if_capenable |= CXGB_CAP_ENABLE;
794 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
795
796 ether_ifattach(ifp, p->hw_addr);
797#ifdef DEFAULT_JUMBO
798 ifp->if_mtu = 9000;
799#endif
800 if ((err = cxgb_makedev(p)) != 0) {
801 printf("makedev failed %d\n", err);
802 return (err);
803 }
804 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
805 cxgb_media_status);
806
807 if (!strcmp(p->port_type->desc, "10GBASE-CX4")) {
808 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
809 } else if (!strcmp(p->port_type->desc, "10GBASE-SR")) {
810 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
811 } else if (!strcmp(p->port_type->desc, "10GBASE-XR")) {
812 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
813 } else if (!strcmp(p->port_type->desc, "10/100/1000BASE-T")) {
814 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
815 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
816 0, NULL);
817 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX,
818 0, NULL);
819 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
820 0, NULL);
821 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
822 0, NULL);
823 media_flags = 0;
824 } else {
825 printf("unsupported media type %s\n", p->port_type->desc);
826 return (ENXIO);
827 }
828 if (media_flags) {
829 ifmedia_add(&p->media, media_flags, 0, NULL);
830 ifmedia_set(&p->media, media_flags);
831 } else {
832 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
833 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO);
834 }
835
836
837 snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port);
838#ifdef TASKQUEUE_CURRENT
839 /* Create a port for handling TX without starvation */
840 p->tq = taskqueue_create(buf, M_NOWAIT,
841 taskqueue_thread_enqueue, &p->tq);
842#else
843 /* Create a port for handling TX without starvation */
844 p->tq = taskqueue_create_fast(buf, M_NOWAIT,
845 taskqueue_thread_enqueue, &p->tq);
846#endif
847
848 if (p->tq == NULL) {
849 device_printf(dev, "failed to allocate port task queue\n");
850 return (ENOMEM);
851 }
852 taskqueue_start_threads(&p->tq, 1, PI_NET, "%s taskq",
853 device_get_nameunit(dev));
854 TASK_INIT(&p->start_task, 0, cxgb_start_proc, ifp);
855
856 t3_sge_init_port(p);
857
858 return (0);
859}
860
861static int
862cxgb_port_detach(device_t dev)
863{
864 struct port_info *p;
865
866 p = device_get_softc(dev);
867
868 PORT_LOCK(p);
869 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING)
870 cxgb_stop_locked(p);
871 PORT_UNLOCK(p);
872
873 mtx_destroy(&p->lock);
874 if (p->tq != NULL) {
875 taskqueue_drain(p->tq, &p->start_task);
876 taskqueue_free(p->tq);
877 p->tq = NULL;
878 }
879
880 ether_ifdetach(p->ifp);
881 if_free(p->ifp);
882
883 if (p->port_cdev != NULL)
884 destroy_dev(p->port_cdev);
885
886 return (0);
887}
888
889void
890t3_fatal_err(struct adapter *sc)
891{
892 u_int fw_status[4];
893
894 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
895 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
896 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
897 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
898}
899
900int
901t3_os_find_pci_capability(adapter_t *sc, int cap)
902{
903 device_t dev;
904 struct pci_devinfo *dinfo;
905 pcicfgregs *cfg;
906 uint32_t status;
907 uint8_t ptr;
908
909 dev = sc->dev;
910 dinfo = device_get_ivars(dev);
911 cfg = &dinfo->cfg;
912
913 status = pci_read_config(dev, PCIR_STATUS, 2);
914 if (!(status & PCIM_STATUS_CAPPRESENT))
915 return (0);
916
917 switch (cfg->hdrtype & PCIM_HDRTYPE) {
918 case 0:
919 case 1:
920 ptr = PCIR_CAP_PTR;
921 break;
922 case 2:
923 ptr = PCIR_CAP_PTR_2;
924 break;
925 default:
926 return (0);
927 break;
928 }
929 ptr = pci_read_config(dev, ptr, 1);
930
931 while (ptr != 0) {
932 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
933 return (ptr);
934 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
935 }
936
937 return (0);
938}
939
940int
941t3_os_pci_save_state(struct adapter *sc)
942{
943 device_t dev;
944 struct pci_devinfo *dinfo;
945
946 dev = sc->dev;
947 dinfo = device_get_ivars(dev);
948
949 pci_cfg_save(dev, dinfo, 0);
950 return (0);
951}
952
953int
954t3_os_pci_restore_state(struct adapter *sc)
955{
956 device_t dev;
957 struct pci_devinfo *dinfo;
958
959 dev = sc->dev;
960 dinfo = device_get_ivars(dev);
961
962 pci_cfg_restore(dev, dinfo);
963 return (0);
964}
965
966/**
967 * t3_os_link_changed - handle link status changes
968 * @adapter: the adapter associated with the link change
969 * @port_id: the port index whose limk status has changed
970 * @link_stat: the new status of the link
971 * @speed: the new speed setting
972 * @duplex: the new duplex setting
973 * @fc: the new flow-control setting
974 *
975 * This is the OS-dependent handler for link status changes. The OS
976 * neutral handler takes care of most of the processing for these events,
977 * then calls this handler for any OS-specific processing.
978 */
979void
980t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
981 int duplex, int fc)
982{
983 struct port_info *pi = &adapter->port[port_id];
984 struct cmac *mac = &adapter->port[port_id].mac;
985
986 if ((pi->ifp->if_flags & IFF_UP) == 0)
987 return;
988
989 if (link_status) {
990 t3_mac_enable(mac, MAC_DIRECTION_RX);
991 if_link_state_change(pi->ifp, LINK_STATE_UP);
992 } else {
993 if_link_state_change(pi->ifp, LINK_STATE_DOWN);
994 pi->phy.ops->power_down(&pi->phy, 1);
995 t3_mac_disable(mac, MAC_DIRECTION_RX);
996 t3_link_start(&pi->phy, mac, &pi->link_config);
997 }
998}
999
1000
1001/*
1002 * Interrupt-context handler for external (PHY) interrupts.
1003 */
1004void
1005t3_os_ext_intr_handler(adapter_t *sc)
1006{
1007 if (cxgb_debug)
1008 printf("t3_os_ext_intr_handler\n");
1009 /*
1010 * Schedule a task to handle external interrupts as they may be slow
1011 * and we use a mutex to protect MDIO registers. We disable PHY
1012 * interrupts in the meantime and let the task reenable them when
1013 * it's done.
1014 */
1015 ADAPTER_LOCK(sc);
1016 if (sc->slow_intr_mask) {
1017 sc->slow_intr_mask &= ~F_T3DBG;
1018 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1019 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1020 }
1021 ADAPTER_UNLOCK(sc);
1022}
1023
1024void
1025t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1026{
1027
1028 /*
1029 * The ifnet might not be allocated before this gets called,
1030 * as this is called early on in attach by t3_prep_adapter
1031 * save the address off in the port structure
1032 */
1033 if (cxgb_debug)
1034 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1035 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1036}
1037
1038/**
1039 * link_start - enable a port
1040 * @p: the port to enable
1041 *
1042 * Performs the MAC and PHY actions needed to enable a port.
1043 */
1044static void
1045cxgb_link_start(struct port_info *p)
1046{
1047 struct ifnet *ifp;
1048 struct t3_rx_mode rm;
1049 struct cmac *mac = &p->mac;
1050
1051 ifp = p->ifp;
1052
1053 t3_init_rx_mode(&rm, p);
1054 t3_mac_reset(mac);
1055 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1056 t3_mac_set_address(mac, 0, p->hw_addr);
1057 t3_mac_set_rx_mode(mac, &rm);
1058 t3_link_start(&p->phy, mac, &p->link_config);
1059 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1060}
1061
1062/**
1063 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1064 * @adap: the adapter
1065 *
1066 * Sets up RSS to distribute packets to multiple receive queues. We
1067 * configure the RSS CPU lookup table to distribute to the number of HW
1068 * receive queues, and the response queue lookup table to narrow that
1069 * down to the response queues actually configured for each port.
1070 * We always configure the RSS mapping for two ports since the mapping
1071 * table has plenty of entries.
1072 */
1073static void
1074setup_rss(adapter_t *adap)
1075{
1076 int i;
1077 u_int nq0 = adap->port[0].nqsets;
1078 u_int nq1 = max((u_int)adap->port[1].nqsets, 1U);
1079 uint8_t cpus[SGE_QSETS + 1];
1080 uint16_t rspq_map[RSS_TABLE_SIZE];
1081
1082 for (i = 0; i < SGE_QSETS; ++i)
1083 cpus[i] = i;
1084 cpus[SGE_QSETS] = 0xff;
1085
1086 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1087 rspq_map[i] = i % nq0;
1088 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
1089 }
1090
1091 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1092 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
1093 V_RRCPLCPUSIZE(6), cpus, rspq_map);
1094}
1095
1096/*
1097 * Sends an mbuf to an offload queue driver
1098 * after dealing with any active network taps.
1099 */
1100static inline int
1101offload_tx(struct toedev *tdev, struct mbuf *m)
1102{
1103 int ret;
1104
1105 critical_enter();
1106 ret = t3_offload_tx(tdev, m);
1107 critical_exit();
1108 return (ret);
1109}
1110
1111static int
1112write_smt_entry(struct adapter *adapter, int idx)
1113{
1114 struct port_info *pi = &adapter->port[idx];
1115 struct cpl_smt_write_req *req;
1116 struct mbuf *m;
1117
1118 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1119 return (ENOMEM);
1120
1121 req = mtod(m, struct cpl_smt_write_req *);
1122 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1123 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1124 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
1125 req->iff = idx;
1126 memset(req->src_mac1, 0, sizeof(req->src_mac1));
1127 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1128
1129 m_set_priority(m, 1);
1130
1131 offload_tx(&adapter->tdev, m);
1132
1133 return (0);
1134}
1135
1136static int
1137init_smt(struct adapter *adapter)
1138{
1139 int i;
1140
1141 for_each_port(adapter, i)
1142 write_smt_entry(adapter, i);
1143 return 0;
1144}
1145
1146static void
1147init_port_mtus(adapter_t *adapter)
1148{
1149 unsigned int mtus = adapter->port[0].ifp->if_mtu;
1150
1151 if (adapter->port[1].ifp)
1152 mtus |= adapter->port[1].ifp->if_mtu << 16;
1153 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1154}
1155
1156static void
1157send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1158 int hi, int port)
1159{
1160 struct mbuf *m;
1161 struct mngt_pktsched_wr *req;
1162
1163 m = m_gethdr(M_NOWAIT, MT_DATA);
1164 if (m) {
1165 req = mtod(m, struct mngt_pktsched_wr *);
1166 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1167 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1168 req->sched = sched;
1169 req->idx = qidx;
1170 req->min = lo;
1171 req->max = hi;
1172 req->binding = port;
1173 m->m_len = m->m_pkthdr.len = sizeof(*req);
1174 t3_mgmt_tx(adap, m);
1175 }
1176}
1177
1178static void
1179bind_qsets(adapter_t *sc)
1180{
1181 int i, j;
1182
1183 for (i = 0; i < (sc)->params.nports; ++i) {
1184 const struct port_info *pi = adap2pinfo(sc, i);
1185
1186 for (j = 0; j < pi->nqsets; ++j)
1187 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1188 -1, i);
1189 }
1190}
1191
1192/**
1193 * cxgb_up - enable the adapter
1194 * @adap: adapter being enabled
1195 *
1196 * Called when the first port is enabled, this function performs the
1197 * actions necessary to make an adapter operational, such as completing
1198 * the initialization of HW modules, and enabling interrupts.
1199 *
1200 */
1201static int
1202cxgb_up(struct adapter *sc)
1203{
1204 int err = 0;
1205
1206 if ((sc->flags & FULL_INIT_DONE) == 0) {
1207
1208 if ((sc->flags & FW_UPTODATE) == 0)
1209 err = upgrade_fw(sc);
1210
1211 if (err)
1212 goto out;
1213
1214 err = t3_init_hw(sc, 0);
1215 if (err)
1216 goto out;
1217
1218 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1219
1220 err = setup_sge_qsets(sc);
1221 if (err)
1222 goto out;
1223
1224 setup_rss(sc);
1225 sc->flags |= FULL_INIT_DONE;
1226 }
1227
1228 t3_intr_clear(sc);
1229
1230 /* If it's MSI or INTx, allocate a single interrupt for everything */
1231 if ((sc->flags & USING_MSIX) == 0) {
1232 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
1233 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
1234 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid);
1235 err = EINVAL;
1236 goto out;
1237 }
1238 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
1239
1240 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
1241#ifdef INTR_FILTERS
1242 NULL,
1243#endif
1244 sc->cxgb_intr, sc, &sc->intr_tag)) {
1245 device_printf(sc->dev, "Cannot set up interrupt\n");
1246 err = EINVAL;
1247 goto irq_err;
1248 }
1249 } else {
1250 cxgb_setup_msix(sc, sc->msi_count);
1251 }
1252
1253 t3_sge_start(sc);
1254 t3_intr_enable(sc);
1255
1256 if ((sc->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
1257 bind_qsets(sc);
1258 sc->flags |= QUEUES_BOUND;
1259out:
1260 return (err);
1261irq_err:
1262 CH_ERR(sc, "request_irq failed, err %d\n", err);
1263 goto out;
1264}
1265
1266
1267/*
1268 * Release resources when all the ports and offloading have been stopped.
1269 */
1270static void
1271cxgb_down(struct adapter *sc)
1272{
1273 int i;
1274
1275 t3_sge_stop(sc);
1276 ADAPTER_LOCK(sc);
1277 t3_intr_disable(sc);
1278 ADAPTER_UNLOCK(sc);
1279
1280
1281 if (sc->intr_tag != NULL) {
1282 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
1283 sc->intr_tag = NULL;
1284 }
1285 if (sc->irq_res != NULL) {
1286 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
1287 sc->irq_rid, sc->irq_res);
1288 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
1289 sc->irq_res);
1290 sc->irq_res = NULL;
1291 }
1292
1293 if (sc->flags & USING_MSIX)
1294 cxgb_teardown_msix(sc);
1295
1296 callout_drain(&sc->sge_timer_ch);
1297 if (sc->tq != NULL)
1298 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1299 for (i = 0; i < sc->params.nports; i++)
1300 if (sc->port[i].tq != NULL)
1301 taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task);
1302
1303}
1304
1305static int
1306offload_open(struct port_info *pi)
1307{
1308 struct adapter *adapter = pi->adapter;
1309 struct toedev *tdev = TOEDEV(pi->ifp);
1310 int adap_up = adapter->open_device_map & PORT_MASK;
1311 int err = 0;
1312
1313 if (atomic_cmpset_int(&adapter->open_device_map,
1314 (adapter->open_device_map & ~OFFLOAD_DEVMAP_BIT),
1315 (adapter->open_device_map | OFFLOAD_DEVMAP_BIT)) == 0)
1316 return (0);
1317
1318 ADAPTER_LOCK(pi->adapter);
1319 if (!adap_up)
1320 err = cxgb_up(adapter);
1321 ADAPTER_UNLOCK(pi->adapter);
1322 if (err < 0)
1323 return (err);
1324
1325 t3_tp_set_offload_mode(adapter, 1);
1326 tdev->lldev = adapter->port[0].ifp;
1327 err = cxgb_offload_activate(adapter);
1328 if (err)
1329 goto out;
1330
1331 init_port_mtus(adapter);
1332 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1333 adapter->params.b_wnd,
1334 adapter->params.rev == 0 ?
1335 adapter->port[0].ifp->if_mtu : 0xffff);
1336 init_smt(adapter);
1337
1338 /* Call back all registered clients */
1339 cxgb_add_clients(tdev);
1340
1341out:
1342 /* restore them in case the offload module has changed them */
1343 if (err) {
1344 t3_tp_set_offload_mode(adapter, 0);
1345 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1346 cxgb_set_dummy_ops(tdev);
1347 }
1348 return (err);
1349}
1350#ifdef notyet
1351static int
1352offload_close(struct toedev *tdev)
1353{
1354 struct adapter *adapter = tdev2adap(tdev);
1355
1356 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1357 return (0);
1358
1359 /* Call back all registered clients */
1360 cxgb_remove_clients(tdev);
1361 tdev->lldev = NULL;
1362 cxgb_set_dummy_ops(tdev);
1363 t3_tp_set_offload_mode(adapter, 0);
1364 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1365
1366 ADAPTER_LOCK(adapter);
1367 if (!adapter->open_device_map)
1368 cxgb_down(adapter);
1369 ADAPTER_UNLOCK(adapter);
1370
1371 cxgb_offload_deactivate(adapter);
1372 return (0);
1373}
1374#endif
1375
1376static void
1377cxgb_init(void *arg)
1378{
1379 struct port_info *p = arg;
1380
1381 PORT_LOCK(p);
1382 cxgb_init_locked(p);
1383 PORT_UNLOCK(p);
1384}
1385
1386static void
1387cxgb_init_locked(struct port_info *p)
1388{
1389 struct ifnet *ifp;
1390 adapter_t *sc = p->adapter;
1391 int err;
1392
1393 mtx_assert(&p->lock, MA_OWNED);
1394 ifp = p->ifp;
1395
1396 ADAPTER_LOCK(p->adapter);
1397 if ((sc->open_device_map == 0) && ((err = cxgb_up(sc)) < 0)) {
1398 ADAPTER_UNLOCK(p->adapter);
1399 cxgb_stop_locked(p);
1400 return;
1401 }
1402 if (p->adapter->open_device_map == 0)
1403 t3_intr_clear(sc);
1404
1405 setbit(&p->adapter->open_device_map, p->port);
1406 ADAPTER_UNLOCK(p->adapter);
1407
1408 if (is_offload(sc) && !ofld_disable) {
1409 err = offload_open(p);
1410 if (err)
1411 log(LOG_WARNING,
1412 "Could not initialize offload capabilities\n");
1413 }
1414 cxgb_link_start(p);
1415 t3_link_changed(sc, p->port);
1416 ifp->if_baudrate = p->link_config.speed * 1000000;
1417
1418 t3_port_intr_enable(sc, p->port);
1419
1420 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1421 cxgb_tick, sc);
1422
1423 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1424 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1425}
1426
1427static void
1428cxgb_set_rxmode(struct port_info *p)
1429{
1430 struct t3_rx_mode rm;
1431 struct cmac *mac = &p->mac;
1432
1433 mtx_assert(&p->lock, MA_OWNED);
1434
1435 t3_init_rx_mode(&rm, p);
1436 t3_mac_set_rx_mode(mac, &rm);
1437}
1438
1439static void
1440cxgb_stop_locked(struct port_info *p)
1441{
1442 struct ifnet *ifp;
1443
1444 mtx_assert(&p->lock, MA_OWNED);
1445 mtx_assert(&p->adapter->lock, MA_NOTOWNED);
1446
1447 ifp = p->ifp;
1448
1449 t3_port_intr_disable(p->adapter, p->port);
1450 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1451 p->phy.ops->power_down(&p->phy, 1);
1452 t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1453
1454 ADAPTER_LOCK(p->adapter);
1455 clrbit(&p->adapter->open_device_map, p->port);
1456 /*
1457 * XXX cancel check_task
1458 */
1459 if (p->adapter->open_device_map == 0)
1460 cxgb_down(p->adapter);
1461 ADAPTER_UNLOCK(p->adapter);
1462}
1463
1464static int
1465cxgb_set_mtu(struct port_info *p, int mtu)
1466{
1467 struct ifnet *ifp = p->ifp;
1468 int error = 0;
1469
1470 if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO))
1471 error = EINVAL;
1472 else if (ifp->if_mtu != mtu) {
1473 PORT_LOCK(p);
1474 ifp->if_mtu = mtu;
1475 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1476 callout_stop(&p->adapter->cxgb_tick_ch);
1477 cxgb_stop_locked(p);
1478 cxgb_init_locked(p);
1479 }
1480 PORT_UNLOCK(p);
1481 }
1482 return (error);
1483}
1484
1485static int
1486cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1487{
1488 struct port_info *p = ifp->if_softc;
1489 struct ifaddr *ifa = (struct ifaddr *)data;
1490 struct ifreq *ifr = (struct ifreq *)data;
1491 int flags, error = 0;
1492 uint32_t mask;
1493
1494 /*
1495 * XXX need to check that we aren't in the middle of an unload
1496 */
1497 switch (command) {
1498 case SIOCSIFMTU:
1499 error = cxgb_set_mtu(p, ifr->ifr_mtu);
1500 break;
1501 case SIOCSIFADDR:
1502 case SIOCGIFADDR:
1503 PORT_LOCK(p);
1504 if (ifa->ifa_addr->sa_family == AF_INET) {
1505 ifp->if_flags |= IFF_UP;
1506 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1507 cxgb_init_locked(p);
1508 arp_ifinit(ifp, ifa);
1509 } else
1510 error = ether_ioctl(ifp, command, data);
1511 PORT_UNLOCK(p);
1512 break;
1513 case SIOCSIFFLAGS:
1514 PORT_LOCK(p);
1515 if (ifp->if_flags & IFF_UP) {
1516 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1517 flags = p->if_flags;
1518 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1519 ((ifp->if_flags ^ flags) & IFF_ALLMULTI))
1520 cxgb_set_rxmode(p);
1521
1522 } else
1523 cxgb_init_locked(p);
1524 p->if_flags = ifp->if_flags;
1525 } else {
1526 callout_stop(&p->adapter->cxgb_tick_ch);
1527 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1528 cxgb_stop_locked(p);
1529 } else {
1530 adapter_t *sc = p->adapter;
1531 callout_reset(&sc->cxgb_tick_ch,
1532 sc->params.stats_update_period * hz,
1533 cxgb_tick, sc);
1534 }
1535 }
1536 PORT_UNLOCK(p);
1537 break;
1538 case SIOCSIFMEDIA:
1539 case SIOCGIFMEDIA:
1540 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
1541 break;
1542 case SIOCSIFCAP:
1543 PORT_LOCK(p);
1544 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1545 if (mask & IFCAP_TXCSUM) {
1546 if (IFCAP_TXCSUM & ifp->if_capenable) {
1547 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
1548 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
1549 | CSUM_TSO);
1550 } else {
1551 ifp->if_capenable |= IFCAP_TXCSUM;
1552 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1553 }
1554 } else if (mask & IFCAP_RXCSUM) {
1555 if (IFCAP_RXCSUM & ifp->if_capenable) {
1556 ifp->if_capenable &= ~IFCAP_RXCSUM;
1557 } else {
1558 ifp->if_capenable |= IFCAP_RXCSUM;
1559 }
1560 }
1561 if (mask & IFCAP_TSO4) {
1562 if (IFCAP_TSO4 & ifp->if_capenable) {
1563 ifp->if_capenable &= ~IFCAP_TSO4;
1564 ifp->if_hwassist &= ~CSUM_TSO;
1565 } else if (IFCAP_TXCSUM & ifp->if_capenable) {
1566 ifp->if_capenable |= IFCAP_TSO4;
1567 ifp->if_hwassist |= CSUM_TSO;
1568 } else {
1569 if (cxgb_debug)
1570 printf("cxgb requires tx checksum offload"
1571 " be enabled to use TSO\n");
1572 error = EINVAL;
1573 }
1574 }
1575 PORT_UNLOCK(p);
1576 break;
1577 default:
1578 error = ether_ioctl(ifp, command, data);
1579 break;
1580 }
1581 return (error);
1582}
1583
1584static int
1585cxgb_start_tx(struct ifnet *ifp, uint32_t txmax)
1586{
1587 struct sge_qset *qs;
1588 struct sge_txq *txq;
1589 struct port_info *p = ifp->if_softc;
1590 struct mbuf *m0, *m = NULL;
1591 int err, in_use_init;
1592
1593 if (!p->link_config.link_ok)
1594 return (ENXIO);
1595
1596 if (IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1597 return (ENOBUFS);
1598
1599 qs = &p->adapter->sge.qs[p->first_qset];
1600 txq = &qs->txq[TXQ_ETH];
1601 err = 0;
1602
1603 mtx_lock(&txq->lock);
1604 in_use_init = txq->in_use;
1605 while ((txq->in_use - in_use_init < txmax) &&
1606 (txq->size > txq->in_use + TX_MAX_DESC)) {
1607 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
1608 if (m == NULL)
1609 break;
1610 /*
1611 * Convert chain to M_IOVEC
1612 */
1613 KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early"));
1614 m0 = m;
1615#ifdef INVARIANTS
1616 /*
1617 * Clean up after net stack sloppiness
1618 * before calling m_sanity
1619 */
1620 m0 = m->m_next;
1621 while (m0) {
1622 m0->m_flags &= ~M_PKTHDR;
1623 m0 = m0->m_next;
1624 }
1625 m_sanity(m0, 0);
1626 m0 = m;
1627#endif
1628 if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES &&
1629 m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) {
1630 if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) {
1631 m = m0;
1632 m_collapse(m, TX_MAX_SEGS, &m0);
1633 } else
1634 break;
1635 }
1636 m = m0;
1637 if ((err = t3_encap(p, &m)) != 0)
1638 break;
1639 BPF_MTAP(ifp, m);
1640 }
1641 mtx_unlock(&txq->lock);
1642
1643 if (__predict_false(err)) {
1644 if (err == ENOMEM) {
1645 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1646 IFQ_LOCK(&ifp->if_snd);
1647 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1648 IFQ_UNLOCK(&ifp->if_snd);
1649 }
1650 }
1651 if (err == 0 && m == NULL)
1652 err = ENOBUFS;
1653 else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) &&
1654 (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
1655 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1656 err = ENOSPC;
1657 }
1658 return (err);
1659}
1660
1661static void
1662cxgb_start_proc(void *arg, int ncount)
1663{
1664 struct ifnet *ifp = arg;
1665 struct port_info *pi = ifp->if_softc;
1666 struct sge_qset *qs;
1667 struct sge_txq *txq;
1668 int error;
1669
1670 qs = &pi->adapter->sge.qs[pi->first_qset];
1671 txq = &qs->txq[TXQ_ETH];
1672
1673 do {
1674 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1675 taskqueue_enqueue(pi->adapter->tq,
1676 &pi->timer_reclaim_task);
1677
1678 error = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1679 } while (error == 0);
1680}
1681
1682static void
1683cxgb_start(struct ifnet *ifp)
1684{
1685 struct port_info *pi = ifp->if_softc;
1686 struct sge_qset *qs;
1687 struct sge_txq *txq;
1688 int err;
1689
1690 qs = &pi->adapter->sge.qs[pi->first_qset];
1691 txq = &qs->txq[TXQ_ETH];
1692
1693 if (desc_reclaimable(txq) > TX_CLEAN_MAX_DESC)
1694 taskqueue_enqueue(pi->adapter->tq,
1695 &pi->timer_reclaim_task);
1696
1697 err = cxgb_start_tx(ifp, TX_START_MAX_DESC);
1698
1699 if (err == 0)
1700 taskqueue_enqueue(pi->tq, &pi->start_task);
1701}
1702
1703
1704static int
1705cxgb_media_change(struct ifnet *ifp)
1706{
1707 if_printf(ifp, "media change not supported\n");
1708 return (ENXIO);
1709}
1710
1711static void
1712cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1713{
1714 struct port_info *p = ifp->if_softc;
1715
1716 ifmr->ifm_status = IFM_AVALID;
1717 ifmr->ifm_active = IFM_ETHER;
1718
1719 if (!p->link_config.link_ok)
1720 return;
1721
1722 ifmr->ifm_status |= IFM_ACTIVE;
1723
1724 switch (p->link_config.speed) {
1725 case 10:
1726 ifmr->ifm_active |= IFM_10_T;
1727 break;
1728 case 100:
1729 ifmr->ifm_active |= IFM_100_TX;
1730 break;
1731 case 1000:
1732 ifmr->ifm_active |= IFM_1000_T;
1733 break;
1734 }
1735
1736 if (p->link_config.duplex)
1737 ifmr->ifm_active |= IFM_FDX;
1738 else
1739 ifmr->ifm_active |= IFM_HDX;
1740}
1741
1742static void
1743cxgb_async_intr(void *data)
1744{
1745 adapter_t *sc = data;
1746
1747 if (cxgb_debug)
1748 device_printf(sc->dev, "cxgb_async_intr\n");
1749
1750 t3_slow_intr_handler(sc);
1751
1752}
1753
1754static void
1755cxgb_ext_intr_handler(void *arg, int count)
1756{
1757 adapter_t *sc = (adapter_t *)arg;
1758
1759 if (cxgb_debug)
1760 printf("cxgb_ext_intr_handler\n");
1761
1762 t3_phy_intr_handler(sc);
1763
1764 /* Now reenable external interrupts */
1765 ADAPTER_LOCK(sc);
1766 if (sc->slow_intr_mask) {
1767 sc->slow_intr_mask |= F_T3DBG;
1768 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
1769 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1770 }
1771 ADAPTER_UNLOCK(sc);
1772}
1773
1774static void
1775check_link_status(adapter_t *sc)
1776{
1777 int i;
1778
1779 for (i = 0; i < (sc)->params.nports; ++i) {
1780 struct port_info *p = &sc->port[i];
1781
1782 if (!(p->port_type->caps & SUPPORTED_IRQ))
1783 t3_link_changed(sc, i);
1784 p->ifp->if_baudrate = p->link_config.speed * 1000000;
1785 }
1786}
1787
1788static void
1789check_t3b2_mac(struct adapter *adapter)
1790{
1791 int i;
1792
1793 for_each_port(adapter, i) {
1794 struct port_info *p = &adapter->port[i];
1795 struct ifnet *ifp = p->ifp;
1796 int status;
1797
1798 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1799 continue;
1800
1801 status = 0;
1802 PORT_LOCK(p);
1803 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
1804 status = t3b2_mac_watchdog_task(&p->mac);
1805 if (status == 1)
1806 p->mac.stats.num_toggled++;
1807 else if (status == 2) {
1808 struct cmac *mac = &p->mac;
1809
1810 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN
1811 + ETHER_VLAN_ENCAP_LEN);
1812 t3_mac_set_address(mac, 0, p->hw_addr);
1813 cxgb_set_rxmode(p);
1814 t3_link_start(&p->phy, mac, &p->link_config);
1815 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1816 t3_port_intr_enable(adapter, p->port);
1817 p->mac.stats.num_resets++;
1818 }
1819 PORT_UNLOCK(p);
1820 }
1821}
1822
1823static void
1824cxgb_tick(void *arg)
1825{
1826 adapter_t *sc = (adapter_t *)arg;
1827 const struct adapter_params *p = &sc->params;
1828
1829 if (p->linkpoll_period)
1830 check_link_status(sc);
1831 callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
1832 cxgb_tick, sc);
1833
1834 /*
1835 * adapter lock can currently only be acquire after the
1836 * port lock
1837 */
1838 ADAPTER_UNLOCK(sc);
1839
1840 if (p->rev == T3_REV_B2)
1841 check_t3b2_mac(sc);
1842}
1843
1844static int
1845in_range(int val, int lo, int hi)
1846{
1847 return val < 0 || (val <= hi && val >= lo);
1848}
1849
1850static int
1851cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td)
1852{
1853 return (0);
1854}
1855
1856static int
1857cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
1858{
1859 return (0);
1860}
1861
1862static int
1863cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
1864 int fflag, struct thread *td)
1865{
1866 int mmd, error = 0;
1867 struct port_info *pi = dev->si_drv1;
1868 adapter_t *sc = pi->adapter;
1869
1870#ifdef PRIV_SUPPORTED
1871 if (priv_check(td, PRIV_DRIVER)) {
1872 if (cxgb_debug)
1873 printf("user does not have access to privileged ioctls\n");
1874 return (EPERM);
1875 }
1876#else
1877 if (suser(td)) {
1878 if (cxgb_debug)
1879 printf("user does not have access to privileged ioctls\n");
1880 return (EPERM);
1881 }
1882#endif
1883
1884 switch (cmd) {
1885 case SIOCGMIIREG: {
1886 uint32_t val;
1887 struct cphy *phy = &pi->phy;
1888 struct mii_data *mid = (struct mii_data *)data;
1889
1890 if (!phy->mdio_read)
1891 return (EOPNOTSUPP);
1892 if (is_10G(sc)) {
1893 mmd = mid->phy_id >> 8;
1894 if (!mmd)
1895 mmd = MDIO_DEV_PCS;
1896 else if (mmd > MDIO_DEV_XGXS)
1897 return -EINVAL;
1898
1899 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
1900 mid->reg_num, &val);
1901 } else
1902 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
1903 mid->reg_num & 0x1f, &val);
1904 if (error == 0)
1905 mid->val_out = val;
1906 break;
1907 }
1908 case SIOCSMIIREG: {
1909 struct cphy *phy = &pi->phy;
1910 struct mii_data *mid = (struct mii_data *)data;
1911
1912 if (!phy->mdio_write)
1913 return (EOPNOTSUPP);
1914 if (is_10G(sc)) {
1915 mmd = mid->phy_id >> 8;
1916 if (!mmd)
1917 mmd = MDIO_DEV_PCS;
1918 else if (mmd > MDIO_DEV_XGXS)
1919 return (EINVAL);
1920
1921 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
1922 mmd, mid->reg_num, mid->val_in);
1923 } else
1924 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
1925 mid->reg_num & 0x1f,
1926 mid->val_in);
1927 break;
1928 }
1929 case CHELSIO_SETREG: {
1930 struct ch_reg *edata = (struct ch_reg *)data;
1931 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1932 return (EFAULT);
1933 t3_write_reg(sc, edata->addr, edata->val);
1934 break;
1935 }
1936 case CHELSIO_GETREG: {
1937 struct ch_reg *edata = (struct ch_reg *)data;
1938 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
1939 return (EFAULT);
1940 edata->val = t3_read_reg(sc, edata->addr);
1941 break;
1942 }
1943 case CHELSIO_GET_SGE_CONTEXT: {
1944 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
1945 mtx_lock(&sc->sge.reg_lock);
1946 switch (ecntxt->cntxt_type) {
1947 case CNTXT_TYPE_EGRESS:
1948 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
1949 ecntxt->data);
1950 break;
1951 case CNTXT_TYPE_FL:
1952 error = t3_sge_read_fl(sc, ecntxt->cntxt_id,
1953 ecntxt->data);
1954 break;
1955 case CNTXT_TYPE_RSP:
1956 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id,
1957 ecntxt->data);
1958 break;
1959 case CNTXT_TYPE_CQ:
1960 error = t3_sge_read_cq(sc, ecntxt->cntxt_id,
1961 ecntxt->data);
1962 break;
1963 default:
1964 error = EINVAL;
1965 break;
1966 }
1967 mtx_unlock(&sc->sge.reg_lock);
1968 break;
1969 }
1970 case CHELSIO_GET_SGE_DESC: {
1971 struct ch_desc *edesc = (struct ch_desc *)data;
1972 int ret;
1973 if (edesc->queue_num >= SGE_QSETS * 6)
1974 return (EINVAL);
1975 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
1976 edesc->queue_num % 6, edesc->idx, edesc->data);
1977 if (ret < 0)
1978 return (EINVAL);
1979 edesc->size = ret;
1980 break;
1981 }
1982 case CHELSIO_SET_QSET_PARAMS: {
1983 struct qset_params *q;
1984 struct ch_qset_params *t = (struct ch_qset_params *)data;
1985
1986 if (t->qset_idx >= SGE_QSETS)
1987 return -EINVAL;
1988 if (!in_range(t->intr_lat, 0, M_NEWTIMER) ||
1989 !in_range(t->cong_thres, 0, 255) ||
1990 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES,
1991 MAX_TXQ_ENTRIES) ||
1992 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES,
1993 MAX_TXQ_ENTRIES) ||
1994 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1995 MAX_CTRL_TXQ_ENTRIES) ||
1996 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) ||
1997 !in_range(t->fl_size[1], MIN_FL_ENTRIES,
1998 MAX_RX_JUMBO_BUFFERS) ||
1999 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
2000 return -EINVAL;
2001 if ((sc->flags & FULL_INIT_DONE) &&
2002 (t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
2003 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||
2004 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 ||
2005 t->polling >= 0 || t->cong_thres >= 0))
2006 return -EBUSY;
2007
2008 q = &sc->params.sge.qset[t->qset_idx];
2009
2010 if (t->rspq_size >= 0)
2011 q->rspq_size = t->rspq_size;
2012 if (t->fl_size[0] >= 0)
2013 q->fl_size = t->fl_size[0];
2014 if (t->fl_size[1] >= 0)
2015 q->jumbo_size = t->fl_size[1];
2016 if (t->txq_size[0] >= 0)
2017 q->txq_size[0] = t->txq_size[0];
2018 if (t->txq_size[1] >= 0)
2019 q->txq_size[1] = t->txq_size[1];
2020 if (t->txq_size[2] >= 0)
2021 q->txq_size[2] = t->txq_size[2];
2022 if (t->cong_thres >= 0)
2023 q->cong_thres = t->cong_thres;
2024 if (t->intr_lat >= 0) {
2025 struct sge_qset *qs = &sc->sge.qs[t->qset_idx];
2026
2027 q->coalesce_nsecs = t->intr_lat*1000;
2028 t3_update_qset_coalesce(qs, q);
2029 }
2030 break;
2031 }
2032 case CHELSIO_GET_QSET_PARAMS: {
2033 struct qset_params *q;
2034 struct ch_qset_params *t = (struct ch_qset_params *)data;
2035
2036 if (t->qset_idx >= SGE_QSETS)
2037 return (EINVAL);
2038
2039 q = &(sc)->params.sge.qset[t->qset_idx];
2040 t->rspq_size = q->rspq_size;
2041 t->txq_size[0] = q->txq_size[0];
2042 t->txq_size[1] = q->txq_size[1];
2043 t->txq_size[2] = q->txq_size[2];
2044 t->fl_size[0] = q->fl_size;
2045 t->fl_size[1] = q->jumbo_size;
2046 t->polling = q->polling;
2047 t->intr_lat = q->coalesce_nsecs / 1000;
2048 t->cong_thres = q->cong_thres;
2049 break;
2050 }
2051 case CHELSIO_SET_QSET_NUM: {
2052 struct ch_reg *edata = (struct ch_reg *)data;
2053 unsigned int port_idx = pi->port;
2054
2055 if (sc->flags & FULL_INIT_DONE)
2056 return (EBUSY);
2057 if (edata->val < 1 ||
2058 (edata->val > 1 && !(sc->flags & USING_MSIX)))
2059 return (EINVAL);
2060 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS)
2061 return (EINVAL);
2062 sc->port[port_idx].nqsets = edata->val;
2063 sc->port[0].first_qset = 0;
2064 /*
2065 * XXX hardcode ourselves to 2 ports just like LEEENUX
2066 */
2067 sc->port[1].first_qset = sc->port[0].nqsets;
2068 break;
2069 }
2070 case CHELSIO_GET_QSET_NUM: {
2071 struct ch_reg *edata = (struct ch_reg *)data;
2072 edata->val = pi->nqsets;
2073 break;
2074 }
2075#ifdef notyet
2076 case CHELSIO_LOAD_FW:
2077 case CHELSIO_GET_PM:
2078 case CHELSIO_SET_PM:
2079 return (EOPNOTSUPP);
2080 break;
2081#endif
2082 case CHELSIO_SETMTUTAB: {
2083 struct ch_mtus *m = (struct ch_mtus *)data;
2084 int i;
2085
2086 if (!is_offload(sc))
2087 return (EOPNOTSUPP);
2088 if (offload_running(sc))
2089 return (EBUSY);
2090 if (m->nmtus != NMTUS)
2091 return (EINVAL);
2092 if (m->mtus[0] < 81) /* accommodate SACK */
2093 return (EINVAL);
2094
2095 /*
2096 * MTUs must be in ascending order
2097 */
2098 for (i = 1; i < NMTUS; ++i)
2099 if (m->mtus[i] < m->mtus[i - 1])
2100 return (EINVAL);
2101
2102 memcpy(sc->params.mtus, m->mtus,
2103 sizeof(sc->params.mtus));
2104 break;
2105 }
2106 case CHELSIO_GETMTUTAB: {
2107 struct ch_mtus *m = (struct ch_mtus *)data;
2108
2109 if (!is_offload(sc))
2110 return (EOPNOTSUPP);
2111
2112 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2113 m->nmtus = NMTUS;
2114 break;
2115 }
2116 case CHELSIO_DEVUP:
2117 if (!is_offload(sc))
2118 return (EOPNOTSUPP);
2119 return offload_open(pi);
2120 break;
2121 case CHELSIO_GET_MEM: {
2122 struct ch_mem_range *t = (struct ch_mem_range *)data;
2123 struct mc7 *mem;
2124 uint8_t *useraddr;
2125 u64 buf[32];
2126
2127 if (!is_offload(sc))
2128 return (EOPNOTSUPP);
2129 if (!(sc->flags & FULL_INIT_DONE))
2130 return (EIO); /* need the memory controllers */
2131 if ((t->addr & 0x7) || (t->len & 0x7))
2132 return (EINVAL);
2133 if (t->mem_id == MEM_CM)
2134 mem = &sc->cm;
2135 else if (t->mem_id == MEM_PMRX)
2136 mem = &sc->pmrx;
2137 else if (t->mem_id == MEM_PMTX)
2138 mem = &sc->pmtx;
2139 else
2140 return (EINVAL);
2141
2142 /*
2143 * Version scheme:
2144 * bits 0..9: chip version
2145 * bits 10..15: chip revision
2146 */
2147 t->version = 3 | (sc->params.rev << 10);
2148
2149 /*
2150 * Read 256 bytes at a time as len can be large and we don't
2151 * want to use huge intermediate buffers.
2152 */
2153 useraddr = (uint8_t *)(t + 1); /* advance to start of buffer */
2154 while (t->len) {
2155 unsigned int chunk = min(t->len, sizeof(buf));
2156
2157 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf);
2158 if (error)
2159 return (-error);
2160 if (copyout(buf, useraddr, chunk))
2161 return (EFAULT);
2162 useraddr += chunk;
2163 t->addr += chunk;
2164 t->len -= chunk;
2165 }
2166 break;
2167 }
2168 case CHELSIO_READ_TCAM_WORD: {
2169 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2170
2171 if (!is_offload(sc))
2172 return (EOPNOTSUPP);
2173 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2174 break;
2175 }
2176 case CHELSIO_SET_TRACE_FILTER: {
2177 struct ch_trace *t = (struct ch_trace *)data;
2178 const struct trace_params *tp;
2179
2180 tp = (const struct trace_params *)&t->sip;
2181 if (t->config_tx)
2182 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2183 t->trace_tx);
2184 if (t->config_rx)
2185 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2186 t->trace_rx);
2187 break;
2188 }
2189 case CHELSIO_SET_PKTSCHED: {
2190 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2191 if (sc->open_device_map == 0)
2192 return (EAGAIN);
2193 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2194 p->binding);
2195 break;
2196 }
2197 case CHELSIO_IFCONF_GETREGS: {
2198 struct ifconf_regs *regs = (struct ifconf_regs *)data;
2199 int reglen = cxgb_get_regs_len();
2200 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT);
2201 if (buf == NULL) {
2202 return (ENOMEM);
2203 } if (regs->len > reglen)
2204 regs->len = reglen;
2205 else if (regs->len < reglen) {
2206 error = E2BIG;
2207 goto done;
2208 }
2209 cxgb_get_regs(sc, regs, buf);
2210 error = copyout(buf, regs->data, reglen);
2211
2212 done:
2213 free(buf, M_DEVBUF);
2214
2215 break;
2216 }
2217 case CHELSIO_SET_HW_SCHED: {
2218 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2219 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2220
2221 if ((sc->flags & FULL_INIT_DONE) == 0)
2222 return (EAGAIN); /* need TP to be initialized */
2223 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2224 !in_range(t->channel, 0, 1) ||
2225 !in_range(t->kbps, 0, 10000000) ||
2226 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2227 !in_range(t->flow_ipg, 0,
2228 dack_ticks_to_usec(sc, 0x7ff)))
2229 return (EINVAL);
2230
2231 if (t->kbps >= 0) {
2232 error = t3_config_sched(sc, t->kbps, t->sched);
2233 if (error < 0)
2234 return (-error);
2235 }
2236 if (t->class_ipg >= 0)
2237 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2238 if (t->flow_ipg >= 0) {
2239 t->flow_ipg *= 1000; /* us -> ns */
2240 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2241 }
2242 if (t->mode >= 0) {
2243 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2244
2245 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2246 bit, t->mode ? bit : 0);
2247 }
2248 if (t->channel >= 0)
2249 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2250 1 << t->sched, t->channel << t->sched);
2251 break;
2252 }
2253 default:
2254 return (EOPNOTSUPP);
2255 break;
2256 }
2257
2258 return (error);
2259}
2260
2261static __inline void
2262reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
2263 unsigned int end)
2264{
2265 uint32_t *p = (uint32_t *)buf + start;
2266
2267 for ( ; start <= end; start += sizeof(uint32_t))
2268 *p++ = t3_read_reg(ap, start);
2269}
2270
2271#define T3_REGMAP_SIZE (3 * 1024)
2272static int
2273cxgb_get_regs_len(void)
2274{
2275 return T3_REGMAP_SIZE;
2276}
2277#undef T3_REGMAP_SIZE
2278
2279static void
2280cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf)
2281{
2282
2283 /*
2284 * Version scheme:
2285 * bits 0..9: chip version
2286 * bits 10..15: chip revision
2287 * bit 31: set for PCIe cards
2288 */
2289 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
2290
2291 /*
2292 * We skip the MAC statistics registers because they are clear-on-read.
2293 * Also reading multi-register stats would need to synchronize with the
2294 * periodic mac stats accumulation. Hard to justify the complexity.
2295 */
2296 memset(buf, 0, REGDUMP_SIZE);
2297 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
2298 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
2299 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
2300 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
2301 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
2302 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
2303 XGM_REG(A_XGM_SERDES_STAT3, 1));
2304 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
2305 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
2306}