cxgb_main.c revision 183289
1/************************************************************************** 2 3Copyright (c) 2007-2008, Chelsio Inc. 4All rights reserved. 5 6Redistribution and use in source and binary forms, with or without 7modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26POSSIBILITY OF SUCH DAMAGE. 27 28***************************************************************************/ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_main.c 183289 2008-09-23 02:22:24Z kmacy $"); 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/kernel.h> 36#include <sys/bus.h> 37#include <sys/module.h> 38#include <sys/pciio.h> 39#include <sys/conf.h> 40#include <machine/bus.h> 41#include <machine/resource.h> 42#include <sys/bus_dma.h> 43#include <sys/ktr.h> 44#include <sys/rman.h> 45#include <sys/ioccom.h> 46#include <sys/mbuf.h> 47#include <sys/linker.h> 48#include <sys/firmware.h> 49#include <sys/socket.h> 50#include <sys/sockio.h> 51#include <sys/smp.h> 52#include <sys/sysctl.h> 53#include <sys/syslog.h> 54#include <sys/queue.h> 55#include <sys/taskqueue.h> 56#include <sys/proc.h> 57 58#include <net/bpf.h> 59#include <net/ethernet.h> 60#include <net/if.h> 61#include <net/if_arp.h> 62#include <net/if_dl.h> 63#include <net/if_media.h> 64#include <net/if_types.h> 65#include <net/if_vlan_var.h> 66 67#include <netinet/in_systm.h> 68#include <netinet/in.h> 69#include <netinet/if_ether.h> 70#include <netinet/ip.h> 71#include <netinet/ip.h> 72#include <netinet/tcp.h> 73#include <netinet/udp.h> 74 75#include <dev/pci/pcireg.h> 76#include <dev/pci/pcivar.h> 77#include <dev/pci/pci_private.h> 78 79#ifdef CONFIG_DEFINED 80#include <cxgb_include.h> 81#else 82#include <dev/cxgb/cxgb_include.h> 83#endif 84 85#ifdef PRIV_SUPPORTED 86#include <sys/priv.h> 87#endif 88 89#ifdef IFNET_MULTIQUEUE 90#include <machine/intr_machdep.h> 91#endif 92 93static int cxgb_setup_msix(adapter_t *, int); 94static void cxgb_teardown_msix(adapter_t *); 95static void cxgb_init(void *); 96static void cxgb_init_locked(struct port_info *); 97static void cxgb_stop_locked(struct port_info *); 98static void cxgb_set_rxmode(struct port_info *); 99static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 100static int cxgb_media_change(struct ifnet *); 101static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 102static int setup_sge_qsets(adapter_t *); 103static void cxgb_async_intr(void *); 104static void cxgb_ext_intr_handler(void *, int); 105static void cxgb_tick_handler(void *, int); 106static void cxgb_down_locked(struct adapter *sc); 107static void cxgb_tick(void *); 108static void setup_rss(adapter_t *sc); 109 110/* Attachment glue for the PCI controller end of the device. Each port of 111 * the device is attached separately, as defined later. 112 */ 113static int cxgb_controller_probe(device_t); 114static int cxgb_controller_attach(device_t); 115static int cxgb_controller_detach(device_t); 116static void cxgb_free(struct adapter *); 117static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 118 unsigned int end); 119static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 120static int cxgb_get_regs_len(void); 121static int offload_open(struct port_info *pi); 122static void touch_bars(device_t dev); 123static int offload_close(struct t3cdev *tdev); 124static void cxgb_link_start(struct port_info *p); 125 126static device_method_t cxgb_controller_methods[] = { 127 DEVMETHOD(device_probe, cxgb_controller_probe), 128 DEVMETHOD(device_attach, cxgb_controller_attach), 129 DEVMETHOD(device_detach, cxgb_controller_detach), 130 131 /* bus interface */ 132 DEVMETHOD(bus_print_child, bus_generic_print_child), 133 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 134 135 { 0, 0 } 136}; 137 138static driver_t cxgb_controller_driver = { 139 "cxgbc", 140 cxgb_controller_methods, 141 sizeof(struct adapter) 142}; 143 144static devclass_t cxgb_controller_devclass; 145DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 146 147/* 148 * Attachment glue for the ports. Attachment is done directly to the 149 * controller device. 150 */ 151static int cxgb_port_probe(device_t); 152static int cxgb_port_attach(device_t); 153static int cxgb_port_detach(device_t); 154 155static device_method_t cxgb_port_methods[] = { 156 DEVMETHOD(device_probe, cxgb_port_probe), 157 DEVMETHOD(device_attach, cxgb_port_attach), 158 DEVMETHOD(device_detach, cxgb_port_detach), 159 { 0, 0 } 160}; 161 162static driver_t cxgb_port_driver = { 163 "cxgb", 164 cxgb_port_methods, 165 0 166}; 167 168static d_ioctl_t cxgb_extension_ioctl; 169static d_open_t cxgb_extension_open; 170static d_close_t cxgb_extension_close; 171 172static struct cdevsw cxgb_cdevsw = { 173 .d_version = D_VERSION, 174 .d_flags = 0, 175 .d_open = cxgb_extension_open, 176 .d_close = cxgb_extension_close, 177 .d_ioctl = cxgb_extension_ioctl, 178 .d_name = "cxgb", 179}; 180 181static devclass_t cxgb_port_devclass; 182DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 183 184#define SGE_MSIX_COUNT (SGE_QSETS + 1) 185 186/* 187 * The driver uses the best interrupt scheme available on a platform in the 188 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 189 * of these schemes the driver may consider as follows: 190 * 191 * msi = 2: choose from among all three options 192 * msi = 1 : only consider MSI and pin interrupts 193 * msi = 0: force pin interrupts 194 */ 195static int msi_allowed = 2; 196 197TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 198SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 199SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 200 "MSI-X, MSI, INTx selector"); 201 202/* 203 * The driver enables offload as a default. 204 * To disable it, use ofld_disable = 1. 205 */ 206static int ofld_disable = 0; 207TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); 208SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, 209 "disable ULP offload"); 210 211/* 212 * The driver uses an auto-queue algorithm by default. 213 * To disable it and force a single queue-set per port, use singleq = 1. 214 */ 215static int singleq = 0; 216TUNABLE_INT("hw.cxgb.singleq", &singleq); 217SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0, 218 "use a single queue-set per port"); 219 220 221/* 222 * The driver uses an auto-queue algorithm by default. 223 * To disable it and force a single queue-set per port, use singleq = 1. 224 */ 225static int force_fw_update = 0; 226TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update); 227SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 228 "update firmware even if up to date"); 229 230int cxgb_use_16k_clusters = 1; 231TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters); 232SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 233 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 234 235enum { 236 MAX_TXQ_ENTRIES = 16384, 237 MAX_CTRL_TXQ_ENTRIES = 1024, 238 MAX_RSPQ_ENTRIES = 16384, 239 MAX_RX_BUFFERS = 16384, 240 MAX_RX_JUMBO_BUFFERS = 16384, 241 MIN_TXQ_ENTRIES = 4, 242 MIN_CTRL_TXQ_ENTRIES = 4, 243 MIN_RSPQ_ENTRIES = 32, 244 MIN_FL_ENTRIES = 32, 245 MIN_FL_JUMBO_ENTRIES = 32 246}; 247 248struct filter_info { 249 u32 sip; 250 u32 sip_mask; 251 u32 dip; 252 u16 sport; 253 u16 dport; 254 u32 vlan:12; 255 u32 vlan_prio:3; 256 u32 mac_hit:1; 257 u32 mac_idx:4; 258 u32 mac_vld:1; 259 u32 pkt_type:2; 260 u32 report_filter_id:1; 261 u32 pass:1; 262 u32 rss:1; 263 u32 qset:3; 264 u32 locked:1; 265 u32 valid:1; 266}; 267 268enum { FILTER_NO_VLAN_PRI = 7 }; 269 270#define EEPROM_MAGIC 0x38E2F10C 271 272#define PORT_MASK ((1 << MAX_NPORTS) - 1) 273 274/* Table for probing the cards. The desc field isn't actually used */ 275struct cxgb_ident { 276 uint16_t vendor; 277 uint16_t device; 278 int index; 279 char *desc; 280} cxgb_identifiers[] = { 281 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 282 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 283 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 284 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 285 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 286 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 287 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 288 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 289 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 290 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 291 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 292 {0, 0, 0, NULL} 293}; 294 295static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 296 297 298static __inline char 299t3rev2char(struct adapter *adapter) 300{ 301 char rev = 'z'; 302 303 switch(adapter->params.rev) { 304 case T3_REV_A: 305 rev = 'a'; 306 break; 307 case T3_REV_B: 308 case T3_REV_B2: 309 rev = 'b'; 310 break; 311 case T3_REV_C: 312 rev = 'c'; 313 break; 314 } 315 return rev; 316} 317 318static struct cxgb_ident * 319cxgb_get_ident(device_t dev) 320{ 321 struct cxgb_ident *id; 322 323 for (id = cxgb_identifiers; id->desc != NULL; id++) { 324 if ((id->vendor == pci_get_vendor(dev)) && 325 (id->device == pci_get_device(dev))) { 326 return (id); 327 } 328 } 329 return (NULL); 330} 331 332static const struct adapter_info * 333cxgb_get_adapter_info(device_t dev) 334{ 335 struct cxgb_ident *id; 336 const struct adapter_info *ai; 337 338 id = cxgb_get_ident(dev); 339 if (id == NULL) 340 return (NULL); 341 342 ai = t3_get_adapter_info(id->index); 343 344 return (ai); 345} 346 347static int 348cxgb_controller_probe(device_t dev) 349{ 350 const struct adapter_info *ai; 351 char *ports, buf[80]; 352 int nports; 353 struct adapter *sc = device_get_softc(dev); 354 355 ai = cxgb_get_adapter_info(dev); 356 if (ai == NULL) 357 return (ENXIO); 358 359 nports = ai->nports0 + ai->nports1; 360 if (nports == 1) 361 ports = "port"; 362 else 363 ports = "ports"; 364 365 snprintf(buf, sizeof(buf), "%s %sNIC, rev: %d nports: %d %s", 366 ai->desc, is_offload(sc) ? "R" : "", 367 sc->params.rev, nports, ports); 368 device_set_desc_copy(dev, buf); 369 return (BUS_PROBE_DEFAULT); 370} 371 372#define FW_FNAME "cxgb_t3fw" 373#define TPEEPROM_NAME "t3b_tp_eeprom" 374#define TPSRAM_NAME "t3b_protocol_sram" 375 376static int 377upgrade_fw(adapter_t *sc) 378{ 379#ifdef FIRMWARE_LATEST 380 const struct firmware *fw; 381#else 382 struct firmware *fw; 383#endif 384 int status; 385 386 if ((fw = firmware_get(FW_FNAME)) == NULL) { 387 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 388 return (ENOENT); 389 } else 390 device_printf(sc->dev, "updating firmware on card\n"); 391 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 392 393 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status); 394 395 firmware_put(fw, FIRMWARE_UNLOAD); 396 397 return (status); 398} 399 400static int 401cxgb_controller_attach(device_t dev) 402{ 403 device_t child; 404 const struct adapter_info *ai; 405 struct adapter *sc; 406 int i, error = 0; 407 uint32_t vers; 408 int port_qsets = 1; 409#ifdef MSI_SUPPORTED 410 int msi_needed, reg; 411#endif 412 int must_load = 0; 413 sc = device_get_softc(dev); 414 sc->dev = dev; 415 sc->msi_count = 0; 416 ai = cxgb_get_adapter_info(dev); 417 418 /* 419 * XXX not really related but a recent addition 420 */ 421#ifdef MSI_SUPPORTED 422 /* find the PCIe link width and set max read request to 4KB*/ 423 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 424 uint16_t lnk, pectl; 425 lnk = pci_read_config(dev, reg + 0x12, 2); 426 sc->link_width = (lnk >> 4) & 0x3f; 427 428 pectl = pci_read_config(dev, reg + 0x8, 2); 429 pectl = (pectl & ~0x7000) | (5 << 12); 430 pci_write_config(dev, reg + 0x8, pectl, 2); 431 } 432 433 if (sc->link_width != 0 && sc->link_width <= 4 && 434 (ai->nports0 + ai->nports1) <= 2) { 435 device_printf(sc->dev, 436 "PCIe x%d Link, expect reduced performance\n", 437 sc->link_width); 438 } 439#endif 440 touch_bars(dev); 441 pci_enable_busmaster(dev); 442 /* 443 * Allocate the registers and make them available to the driver. 444 * The registers that we care about for NIC mode are in BAR 0 445 */ 446 sc->regs_rid = PCIR_BAR(0); 447 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 448 &sc->regs_rid, RF_ACTIVE)) == NULL) { 449 device_printf(dev, "Cannot allocate BAR region 0\n"); 450 return (ENXIO); 451 } 452 sc->udbs_rid = PCIR_BAR(2); 453 if ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 454 &sc->udbs_rid, RF_ACTIVE)) == NULL) { 455 device_printf(dev, "Cannot allocate BAR region 1\n"); 456 error = ENXIO; 457 goto out; 458 } 459 460 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 461 device_get_unit(dev)); 462 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 463 464 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 465 device_get_unit(dev)); 466 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 467 device_get_unit(dev)); 468 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 469 device_get_unit(dev)); 470 471 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 472 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 473 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 474 475 sc->bt = rman_get_bustag(sc->regs_res); 476 sc->bh = rman_get_bushandle(sc->regs_res); 477 sc->mmio_len = rman_get_size(sc->regs_res); 478 479 if (t3_prep_adapter(sc, ai, 1) < 0) { 480 printf("prep adapter failed\n"); 481 error = ENODEV; 482 goto out; 483 } 484 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 485 * enough messages for the queue sets. If that fails, try falling 486 * back to MSI. If that fails, then try falling back to the legacy 487 * interrupt pin model. 488 */ 489#ifdef MSI_SUPPORTED 490 491 sc->msix_regs_rid = 0x20; 492 if ((msi_allowed >= 2) && 493 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 494 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 495 496 msi_needed = sc->msi_count = SGE_MSIX_COUNT; 497 498 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) || 499 (sc->msi_count != msi_needed)) { 500 device_printf(dev, "msix allocation failed - msi_count = %d" 501 " msi_needed=%d will try msi err=%d\n", sc->msi_count, 502 msi_needed, error); 503 sc->msi_count = 0; 504 pci_release_msi(dev); 505 bus_release_resource(dev, SYS_RES_MEMORY, 506 sc->msix_regs_rid, sc->msix_regs_res); 507 sc->msix_regs_res = NULL; 508 } else { 509 sc->flags |= USING_MSIX; 510 sc->cxgb_intr = t3_intr_msix; 511 } 512 } 513 514 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 515 sc->msi_count = 1; 516 if (pci_alloc_msi(dev, &sc->msi_count)) { 517 device_printf(dev, "alloc msi failed - will try INTx\n"); 518 sc->msi_count = 0; 519 pci_release_msi(dev); 520 } else { 521 sc->flags |= USING_MSI; 522 sc->irq_rid = 1; 523 sc->cxgb_intr = t3_intr_msi; 524 } 525 } 526#endif 527 if (sc->msi_count == 0) { 528 device_printf(dev, "using line interrupts\n"); 529 sc->irq_rid = 0; 530 sc->cxgb_intr = t3b_intr; 531 } 532 533 if ((sc->flags & USING_MSIX) && !singleq) 534 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus); 535 536 /* Create a private taskqueue thread for handling driver events */ 537#ifdef TASKQUEUE_CURRENT 538 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 539 taskqueue_thread_enqueue, &sc->tq); 540#else 541 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT, 542 taskqueue_thread_enqueue, &sc->tq); 543#endif 544 if (sc->tq == NULL) { 545 device_printf(dev, "failed to allocate controller task queue\n"); 546 goto out; 547 } 548 549 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 550 device_get_nameunit(dev)); 551 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 552 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 553 554 555 /* Create a periodic callout for checking adapter status */ 556 callout_init(&sc->cxgb_tick_ch, TRUE); 557 558 if ((t3_check_fw_version(sc, &must_load) != 0 && must_load) || force_fw_update) { 559 /* 560 * Warn user that a firmware update will be attempted in init. 561 */ 562 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 563 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 564 sc->flags &= ~FW_UPTODATE; 565 } else { 566 sc->flags |= FW_UPTODATE; 567 } 568 569 if (t3_check_tpsram_version(sc, &must_load) != 0 && must_load) { 570 /* 571 * Warn user that a firmware update will be attempted in init. 572 */ 573 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 574 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 575 sc->flags &= ~TPS_UPTODATE; 576 } else { 577 sc->flags |= TPS_UPTODATE; 578 } 579 580 /* 581 * Create a child device for each MAC. The ethernet attachment 582 * will be done in these children. 583 */ 584 for (i = 0; i < (sc)->params.nports; i++) { 585 struct port_info *pi; 586 587 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 588 device_printf(dev, "failed to add child port\n"); 589 error = EINVAL; 590 goto out; 591 } 592 pi = &sc->port[i]; 593 pi->adapter = sc; 594 pi->nqsets = port_qsets; 595 pi->first_qset = i*port_qsets; 596 pi->port_id = i; 597 pi->tx_chan = i >= ai->nports0; 598 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 599 sc->rxpkt_map[pi->txpkt_intf] = i; 600 sc->port[i].tx_chan = i >= ai->nports0; 601 sc->portdev[i] = child; 602 device_set_softc(child, pi); 603 } 604 if ((error = bus_generic_attach(dev)) != 0) 605 goto out; 606 607 /* initialize sge private state */ 608 t3_sge_init_adapter(sc); 609 610 t3_led_ready(sc); 611 612 cxgb_offload_init(); 613 if (is_offload(sc)) { 614 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 615 cxgb_adapter_ofld(sc); 616 } 617 error = t3_get_fw_version(sc, &vers); 618 if (error) 619 goto out; 620 621 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 622 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 623 G_FW_VERSION_MICRO(vers)); 624 625 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 626 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 627 t3_add_attach_sysctls(sc); 628out: 629 if (error) 630 cxgb_free(sc); 631 632 return (error); 633} 634 635static int 636cxgb_controller_detach(device_t dev) 637{ 638 struct adapter *sc; 639 640 sc = device_get_softc(dev); 641 642 cxgb_free(sc); 643 644 return (0); 645} 646 647static void 648cxgb_free(struct adapter *sc) 649{ 650 int i; 651 652 ADAPTER_LOCK(sc); 653 sc->flags |= CXGB_SHUTDOWN; 654 ADAPTER_UNLOCK(sc); 655 cxgb_pcpu_shutdown_threads(sc); 656 ADAPTER_LOCK(sc); 657 658/* 659 * drops the lock 660 */ 661 cxgb_down_locked(sc); 662 663#ifdef MSI_SUPPORTED 664 if (sc->flags & (USING_MSI | USING_MSIX)) { 665 device_printf(sc->dev, "releasing msi message(s)\n"); 666 pci_release_msi(sc->dev); 667 } else { 668 device_printf(sc->dev, "no msi message to release\n"); 669 } 670#endif 671 if (sc->msix_regs_res != NULL) { 672 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 673 sc->msix_regs_res); 674 } 675 676 t3_sge_deinit_sw(sc); 677 /* 678 * Wait for last callout 679 */ 680 681 DELAY(hz*100); 682 683 for (i = 0; i < (sc)->params.nports; ++i) { 684 if (sc->portdev[i] != NULL) 685 device_delete_child(sc->dev, sc->portdev[i]); 686 } 687 688 bus_generic_detach(sc->dev); 689 if (sc->tq != NULL) { 690 taskqueue_free(sc->tq); 691 sc->tq = NULL; 692 } 693 694 if (is_offload(sc)) { 695 cxgb_adapter_unofld(sc); 696 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) 697 offload_close(&sc->tdev); 698 else 699 printf("cxgb_free: DEVMAP_BIT not set\n"); 700 } else 701 printf("not offloading set\n"); 702#ifdef notyet 703 if (sc->flags & CXGB_OFLD_INIT) 704 cxgb_offload_deactivate(sc); 705#endif 706 free(sc->filters, M_DEVBUF); 707 t3_sge_free(sc); 708 709 cxgb_offload_exit(); 710 711 if (sc->udbs_res != NULL) 712 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 713 sc->udbs_res); 714 715 if (sc->regs_res != NULL) 716 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 717 sc->regs_res); 718 719 MTX_DESTROY(&sc->mdio_lock); 720 MTX_DESTROY(&sc->sge.reg_lock); 721 MTX_DESTROY(&sc->elmer_lock); 722 ADAPTER_LOCK_DEINIT(sc); 723} 724 725/** 726 * setup_sge_qsets - configure SGE Tx/Rx/response queues 727 * @sc: the controller softc 728 * 729 * Determines how many sets of SGE queues to use and initializes them. 730 * We support multiple queue sets per port if we have MSI-X, otherwise 731 * just one queue set per port. 732 */ 733static int 734setup_sge_qsets(adapter_t *sc) 735{ 736 int i, j, err, irq_idx = 0, qset_idx = 0; 737 u_int ntxq = SGE_TXQ_PER_SET; 738 739 if ((err = t3_sge_alloc(sc)) != 0) { 740 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 741 return (err); 742 } 743 744 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 745 irq_idx = -1; 746 747 for (i = 0; i < (sc)->params.nports; i++) { 748 struct port_info *pi = &sc->port[i]; 749 750 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 751 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 752 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 753 &sc->params.sge.qset[qset_idx], ntxq, pi); 754 if (err) { 755 t3_free_sge_resources(sc); 756 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", 757 err); 758 return (err); 759 } 760 } 761 } 762 763 return (0); 764} 765 766static void 767cxgb_teardown_msix(adapter_t *sc) 768{ 769 int i, nqsets; 770 771 for (nqsets = i = 0; i < (sc)->params.nports; i++) 772 nqsets += sc->port[i].nqsets; 773 774 for (i = 0; i < nqsets; i++) { 775 if (sc->msix_intr_tag[i] != NULL) { 776 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 777 sc->msix_intr_tag[i]); 778 sc->msix_intr_tag[i] = NULL; 779 } 780 if (sc->msix_irq_res[i] != NULL) { 781 bus_release_resource(sc->dev, SYS_RES_IRQ, 782 sc->msix_irq_rid[i], sc->msix_irq_res[i]); 783 sc->msix_irq_res[i] = NULL; 784 } 785 } 786} 787 788static int 789cxgb_setup_msix(adapter_t *sc, int msix_count) 790{ 791 int i, j, k, nqsets, rid; 792 793 /* The first message indicates link changes and error conditions */ 794 sc->irq_rid = 1; 795 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 796 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 797 device_printf(sc->dev, "Cannot allocate msix interrupt\n"); 798 return (EINVAL); 799 } 800 801 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 802#ifdef INTR_FILTERS 803 NULL, 804#endif 805 cxgb_async_intr, sc, &sc->intr_tag)) { 806 device_printf(sc->dev, "Cannot set up interrupt\n"); 807 return (EINVAL); 808 } 809 for (i = k = 0; i < (sc)->params.nports; i++) { 810 nqsets = sc->port[i].nqsets; 811 for (j = 0; j < nqsets; j++, k++) { 812 struct sge_qset *qs = &sc->sge.qs[k]; 813 814 rid = k + 2; 815 if (cxgb_debug) 816 printf("rid=%d ", rid); 817 if ((sc->msix_irq_res[k] = bus_alloc_resource_any( 818 sc->dev, SYS_RES_IRQ, &rid, 819 RF_SHAREABLE | RF_ACTIVE)) == NULL) { 820 device_printf(sc->dev, "Cannot allocate " 821 "interrupt for message %d\n", rid); 822 return (EINVAL); 823 } 824 sc->msix_irq_rid[k] = rid; 825 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k], 826 INTR_MPSAFE|INTR_TYPE_NET, 827#ifdef INTR_FILTERS 828 NULL, 829#endif 830 t3_intr_msix, qs, &sc->msix_intr_tag[k])) { 831 device_printf(sc->dev, "Cannot set up " 832 "interrupt for message %d\n", rid); 833 return (EINVAL); 834 } 835#ifdef IFNET_MULTIQUEUE 836 if (singleq == 0) { 837 int vector = rman_get_start(sc->msix_irq_res[k]); 838 if (bootverbose) 839 device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus); 840 intr_bind(vector, k % mp_ncpus); 841 } 842#endif 843 } 844 } 845 846 return (0); 847} 848 849static int 850cxgb_port_probe(device_t dev) 851{ 852 struct port_info *p; 853 char buf[80]; 854 const char *desc; 855 856 p = device_get_softc(dev); 857 desc = p->phy.desc; 858 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 859 device_set_desc_copy(dev, buf); 860 return (0); 861} 862 863 864static int 865cxgb_makedev(struct port_info *pi) 866{ 867 868 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 869 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp)); 870 871 if (pi->port_cdev == NULL) 872 return (ENOMEM); 873 874 pi->port_cdev->si_drv1 = (void *)pi; 875 876 return (0); 877} 878 879#ifndef LRO_SUPPORTED 880#ifdef IFCAP_LRO 881#undef IFCAP_LRO 882#endif 883#define IFCAP_LRO 0x0 884#endif 885 886#ifdef TSO_SUPPORTED 887#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO) 888/* Don't enable TSO6 yet */ 889#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO) 890#else 891#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 892/* Don't enable TSO6 yet */ 893#define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU) 894#define IFCAP_TSO4 0x0 895#define IFCAP_TSO6 0x0 896#define CSUM_TSO 0x0 897#endif 898 899 900static int 901cxgb_port_attach(device_t dev) 902{ 903 struct port_info *p; 904 struct ifnet *ifp; 905 int err, media_flags; 906 struct adapter *sc; 907 908 909 p = device_get_softc(dev); 910 sc = p->adapter; 911 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 912 device_get_unit(device_get_parent(dev)), p->port_id); 913 PORT_LOCK_INIT(p, p->lockbuf); 914 915 /* Allocate an ifnet object and set it up */ 916 ifp = p->ifp = if_alloc(IFT_ETHER); 917 if (ifp == NULL) { 918 device_printf(dev, "Cannot allocate ifnet\n"); 919 return (ENOMEM); 920 } 921 922 /* 923 * Note that there is currently no watchdog timer. 924 */ 925 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 926 ifp->if_init = cxgb_init; 927 ifp->if_softc = p; 928 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 929 ifp->if_ioctl = cxgb_ioctl; 930 ifp->if_start = cxgb_start; 931 932#if 0 933#ifdef IFNET_MULTIQUEUE 934 ifp->if_flags |= IFF_MULTIQ; 935 ifp->if_mq_start = cxgb_pcpu_start; 936#endif 937#endif 938 ifp->if_timer = 0; /* Disable ifnet watchdog */ 939 ifp->if_watchdog = NULL; 940 941 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 942 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 943 IFQ_SET_READY(&ifp->if_snd); 944 945 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0; 946 ifp->if_capabilities |= CXGB_CAP; 947 ifp->if_capenable |= CXGB_CAP_ENABLE; 948 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO); 949 /* 950 * disable TSO on 4-port - it isn't supported by the firmware yet 951 */ 952 if (p->adapter->params.nports > 2) { 953 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6); 954 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6); 955 ifp->if_hwassist &= ~CSUM_TSO; 956 } 957 958 ether_ifattach(ifp, p->hw_addr); 959 /* 960 * Only default to jumbo frames on 10GigE 961 */ 962 if (p->adapter->params.nports <= 2) 963 ifp->if_mtu = ETHERMTU_JUMBO; 964 if ((err = cxgb_makedev(p)) != 0) { 965 printf("makedev failed %d\n", err); 966 return (err); 967 } 968 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 969 cxgb_media_status); 970 971 if (!strcmp(p->phy.desc, "10GBASE-CX4")) { 972 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX; 973 } else if (!strcmp(p->phy.desc, "10GBASE-SR")) { 974 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX; 975 } else if (!strcmp(p->phy.desc, "10GBASE-R")) { 976 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX; 977 } else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) { 978 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL); 979 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX, 980 0, NULL); 981 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX, 982 0, NULL); 983 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 984 0, NULL); 985 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 986 0, NULL); 987 media_flags = 0; 988 } else { 989 printf("unsupported media type %s\n", p->phy.desc); 990 return (ENXIO); 991 } 992 if (media_flags) { 993 ifmedia_add(&p->media, media_flags, 0, NULL); 994 ifmedia_set(&p->media, media_flags); 995 } else { 996 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL); 997 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO); 998 } 999 1000 /* Get the latest mac address, User can use a LAA */ 1001 bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN); 1002 t3_sge_init_port(p); 1003#if defined(LINK_ATTACH) 1004 cxgb_link_start(p); 1005 t3_link_changed(sc, p->port_id); 1006#endif 1007 return (0); 1008} 1009 1010static int 1011cxgb_port_detach(device_t dev) 1012{ 1013 struct port_info *p; 1014 1015 p = device_get_softc(dev); 1016 1017 PORT_LOCK(p); 1018 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING) 1019 cxgb_stop_locked(p); 1020 PORT_UNLOCK(p); 1021 1022 ether_ifdetach(p->ifp); 1023 printf("waiting for callout to stop ..."); 1024 DELAY(1000000); 1025 printf("done\n"); 1026 /* 1027 * the lock may be acquired in ifdetach 1028 */ 1029 PORT_LOCK_DEINIT(p); 1030 if_free(p->ifp); 1031 1032 if (p->port_cdev != NULL) 1033 destroy_dev(p->port_cdev); 1034 1035 return (0); 1036} 1037 1038void 1039t3_fatal_err(struct adapter *sc) 1040{ 1041 u_int fw_status[4]; 1042 1043 if (sc->flags & FULL_INIT_DONE) { 1044 t3_sge_stop(sc); 1045 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1046 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1047 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1048 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1049 t3_intr_disable(sc); 1050 } 1051 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1052 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1053 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1054 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1055} 1056 1057int 1058t3_os_find_pci_capability(adapter_t *sc, int cap) 1059{ 1060 device_t dev; 1061 struct pci_devinfo *dinfo; 1062 pcicfgregs *cfg; 1063 uint32_t status; 1064 uint8_t ptr; 1065 1066 dev = sc->dev; 1067 dinfo = device_get_ivars(dev); 1068 cfg = &dinfo->cfg; 1069 1070 status = pci_read_config(dev, PCIR_STATUS, 2); 1071 if (!(status & PCIM_STATUS_CAPPRESENT)) 1072 return (0); 1073 1074 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1075 case 0: 1076 case 1: 1077 ptr = PCIR_CAP_PTR; 1078 break; 1079 case 2: 1080 ptr = PCIR_CAP_PTR_2; 1081 break; 1082 default: 1083 return (0); 1084 break; 1085 } 1086 ptr = pci_read_config(dev, ptr, 1); 1087 1088 while (ptr != 0) { 1089 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1090 return (ptr); 1091 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1092 } 1093 1094 return (0); 1095} 1096 1097int 1098t3_os_pci_save_state(struct adapter *sc) 1099{ 1100 device_t dev; 1101 struct pci_devinfo *dinfo; 1102 1103 dev = sc->dev; 1104 dinfo = device_get_ivars(dev); 1105 1106 pci_cfg_save(dev, dinfo, 0); 1107 return (0); 1108} 1109 1110int 1111t3_os_pci_restore_state(struct adapter *sc) 1112{ 1113 device_t dev; 1114 struct pci_devinfo *dinfo; 1115 1116 dev = sc->dev; 1117 dinfo = device_get_ivars(dev); 1118 1119 pci_cfg_restore(dev, dinfo); 1120 return (0); 1121} 1122 1123/** 1124 * t3_os_link_changed - handle link status changes 1125 * @adapter: the adapter associated with the link change 1126 * @port_id: the port index whose limk status has changed 1127 * @link_status: the new status of the link 1128 * @speed: the new speed setting 1129 * @duplex: the new duplex setting 1130 * @fc: the new flow-control setting 1131 * 1132 * This is the OS-dependent handler for link status changes. The OS 1133 * neutral handler takes care of most of the processing for these events, 1134 * then calls this handler for any OS-specific processing. 1135 */ 1136void 1137t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1138 int duplex, int fc) 1139{ 1140 struct port_info *pi = &adapter->port[port_id]; 1141 struct cmac *mac = &adapter->port[port_id].mac; 1142 1143 if (link_status) { 1144 DELAY(10); 1145 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1146 /* Clear errors created by MAC enable */ 1147 t3_set_reg_field(adapter, 1148 A_XGM_STAT_CTRL + pi->mac.offset, 1149 F_CLRSTATS, 1); 1150 if_link_state_change(pi->ifp, LINK_STATE_UP); 1151 1152 } else { 1153 pi->phy.ops->power_down(&pi->phy, 1); 1154 t3_mac_disable(mac, MAC_DIRECTION_RX); 1155 t3_link_start(&pi->phy, mac, &pi->link_config); 1156 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1157 if_link_state_change(pi->ifp, LINK_STATE_DOWN); 1158 } 1159} 1160 1161/** 1162 * t3_os_phymod_changed - handle PHY module changes 1163 * @phy: the PHY reporting the module change 1164 * @mod_type: new module type 1165 * 1166 * This is the OS-dependent handler for PHY module changes. It is 1167 * invoked when a PHY module is removed or inserted for any OS-specific 1168 * processing. 1169 */ 1170void t3_os_phymod_changed(struct adapter *adap, int port_id) 1171{ 1172 static const char *mod_str[] = { 1173 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown" 1174 }; 1175 1176 struct port_info *pi = &adap->port[port_id]; 1177 1178 if (pi->phy.modtype == phy_modtype_none) 1179 device_printf(adap->dev, "PHY module unplugged\n"); 1180 else { 1181 KASSERT(pi->phy.modtype < ARRAY_SIZE(mod_str), 1182 ("invalid PHY module type %d", pi->phy.modtype)); 1183 device_printf(adap->dev, "%s PHY module inserted\n", 1184 mod_str[pi->phy.modtype]); 1185 } 1186} 1187 1188/* 1189 * Interrupt-context handler for external (PHY) interrupts. 1190 */ 1191void 1192t3_os_ext_intr_handler(adapter_t *sc) 1193{ 1194 if (cxgb_debug) 1195 printf("t3_os_ext_intr_handler\n"); 1196 /* 1197 * Schedule a task to handle external interrupts as they may be slow 1198 * and we use a mutex to protect MDIO registers. We disable PHY 1199 * interrupts in the meantime and let the task reenable them when 1200 * it's done. 1201 */ 1202 ADAPTER_LOCK(sc); 1203 if (sc->slow_intr_mask) { 1204 sc->slow_intr_mask &= ~F_T3DBG; 1205 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1206 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 1207 } 1208 ADAPTER_UNLOCK(sc); 1209} 1210 1211void 1212t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1213{ 1214 1215 /* 1216 * The ifnet might not be allocated before this gets called, 1217 * as this is called early on in attach by t3_prep_adapter 1218 * save the address off in the port structure 1219 */ 1220 if (cxgb_debug) 1221 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1222 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1223} 1224 1225/** 1226 * link_start - enable a port 1227 * @p: the port to enable 1228 * 1229 * Performs the MAC and PHY actions needed to enable a port. 1230 */ 1231static void 1232cxgb_link_start(struct port_info *p) 1233{ 1234 struct ifnet *ifp; 1235 struct t3_rx_mode rm; 1236 struct cmac *mac = &p->mac; 1237 int mtu, hwtagging; 1238 1239 ifp = p->ifp; 1240 1241 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1242 1243 mtu = ifp->if_mtu; 1244 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1245 mtu += ETHER_VLAN_ENCAP_LEN; 1246 1247 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1248 1249 t3_init_rx_mode(&rm, p); 1250 if (!mac->multiport) 1251 t3_mac_reset(mac); 1252 t3_mac_set_mtu(mac, mtu); 1253 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1254 t3_mac_set_address(mac, 0, p->hw_addr); 1255 t3_mac_set_rx_mode(mac, &rm); 1256 t3_link_start(&p->phy, mac, &p->link_config); 1257 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1258} 1259 1260 1261static int 1262await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1263 unsigned long n) 1264{ 1265 int attempts = 5; 1266 1267 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1268 if (!--attempts) 1269 return (ETIMEDOUT); 1270 t3_os_sleep(10); 1271 } 1272 return 0; 1273} 1274 1275static int 1276init_tp_parity(struct adapter *adap) 1277{ 1278 int i; 1279 struct mbuf *m; 1280 struct cpl_set_tcb_field *greq; 1281 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1282 1283 t3_tp_set_offload_mode(adap, 1); 1284 1285 for (i = 0; i < 16; i++) { 1286 struct cpl_smt_write_req *req; 1287 1288 m = m_gethdr(M_WAITOK, MT_DATA); 1289 req = mtod(m, struct cpl_smt_write_req *); 1290 m->m_len = m->m_pkthdr.len = sizeof(*req); 1291 memset(req, 0, sizeof(*req)); 1292 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1293 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1294 req->iff = i; 1295 t3_mgmt_tx(adap, m); 1296 } 1297 1298 for (i = 0; i < 2048; i++) { 1299 struct cpl_l2t_write_req *req; 1300 1301 m = m_gethdr(M_WAITOK, MT_DATA); 1302 req = mtod(m, struct cpl_l2t_write_req *); 1303 m->m_len = m->m_pkthdr.len = sizeof(*req); 1304 memset(req, 0, sizeof(*req)); 1305 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1306 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1307 req->params = htonl(V_L2T_W_IDX(i)); 1308 t3_mgmt_tx(adap, m); 1309 } 1310 1311 for (i = 0; i < 2048; i++) { 1312 struct cpl_rte_write_req *req; 1313 1314 m = m_gethdr(M_WAITOK, MT_DATA); 1315 req = mtod(m, struct cpl_rte_write_req *); 1316 m->m_len = m->m_pkthdr.len = sizeof(*req); 1317 memset(req, 0, sizeof(*req)); 1318 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1319 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1320 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1321 t3_mgmt_tx(adap, m); 1322 } 1323 1324 m = m_gethdr(M_WAITOK, MT_DATA); 1325 greq = mtod(m, struct cpl_set_tcb_field *); 1326 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1327 memset(greq, 0, sizeof(*greq)); 1328 greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1329 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1330 greq->mask = htobe64(1); 1331 t3_mgmt_tx(adap, m); 1332 1333 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1334 t3_tp_set_offload_mode(adap, 0); 1335 return (i); 1336} 1337 1338/** 1339 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1340 * @adap: the adapter 1341 * 1342 * Sets up RSS to distribute packets to multiple receive queues. We 1343 * configure the RSS CPU lookup table to distribute to the number of HW 1344 * receive queues, and the response queue lookup table to narrow that 1345 * down to the response queues actually configured for each port. 1346 * We always configure the RSS mapping for two ports since the mapping 1347 * table has plenty of entries. 1348 */ 1349static void 1350setup_rss(adapter_t *adap) 1351{ 1352 int i; 1353 u_int nq[2]; 1354 uint8_t cpus[SGE_QSETS + 1]; 1355 uint16_t rspq_map[RSS_TABLE_SIZE]; 1356 1357 for (i = 0; i < SGE_QSETS; ++i) 1358 cpus[i] = i; 1359 cpus[SGE_QSETS] = 0xff; 1360 1361 nq[0] = nq[1] = 0; 1362 for_each_port(adap, i) { 1363 const struct port_info *pi = adap2pinfo(adap, i); 1364 1365 nq[pi->tx_chan] += pi->nqsets; 1366 } 1367 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1368 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1369 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1370 } 1371 /* Calculate the reverse RSS map table */ 1372 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1373 if (adap->rrss_map[rspq_map[i]] == 0xff) 1374 adap->rrss_map[rspq_map[i]] = i; 1375 1376 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1377 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1378 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1379 cpus, rspq_map); 1380 1381} 1382 1383/* 1384 * Sends an mbuf to an offload queue driver 1385 * after dealing with any active network taps. 1386 */ 1387static inline int 1388offload_tx(struct t3cdev *tdev, struct mbuf *m) 1389{ 1390 int ret; 1391 1392 ret = t3_offload_tx(tdev, m); 1393 return (ret); 1394} 1395 1396static int 1397write_smt_entry(struct adapter *adapter, int idx) 1398{ 1399 struct port_info *pi = &adapter->port[idx]; 1400 struct cpl_smt_write_req *req; 1401 struct mbuf *m; 1402 1403 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1404 return (ENOMEM); 1405 1406 req = mtod(m, struct cpl_smt_write_req *); 1407 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); 1408 1409 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1410 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 1411 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 1412 req->iff = idx; 1413 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 1414 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 1415 1416 m_set_priority(m, 1); 1417 1418 offload_tx(&adapter->tdev, m); 1419 1420 return (0); 1421} 1422 1423static int 1424init_smt(struct adapter *adapter) 1425{ 1426 int i; 1427 1428 for_each_port(adapter, i) 1429 write_smt_entry(adapter, i); 1430 return 0; 1431} 1432 1433static void 1434init_port_mtus(adapter_t *adapter) 1435{ 1436 unsigned int mtus = adapter->port[0].ifp->if_mtu; 1437 1438 if (adapter->port[1].ifp) 1439 mtus |= adapter->port[1].ifp->if_mtu << 16; 1440 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); 1441} 1442 1443static void 1444send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1445 int hi, int port) 1446{ 1447 struct mbuf *m; 1448 struct mngt_pktsched_wr *req; 1449 1450 m = m_gethdr(M_DONTWAIT, MT_DATA); 1451 if (m) { 1452 req = mtod(m, struct mngt_pktsched_wr *); 1453 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1454 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1455 req->sched = sched; 1456 req->idx = qidx; 1457 req->min = lo; 1458 req->max = hi; 1459 req->binding = port; 1460 m->m_len = m->m_pkthdr.len = sizeof(*req); 1461 t3_mgmt_tx(adap, m); 1462 } 1463} 1464 1465static void 1466bind_qsets(adapter_t *sc) 1467{ 1468 int i, j; 1469 1470 cxgb_pcpu_startup_threads(sc); 1471 for (i = 0; i < (sc)->params.nports; ++i) { 1472 const struct port_info *pi = adap2pinfo(sc, i); 1473 1474 for (j = 0; j < pi->nqsets; ++j) { 1475 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1476 -1, pi->tx_chan); 1477 1478 } 1479 } 1480} 1481 1482static void 1483update_tpeeprom(struct adapter *adap) 1484{ 1485#ifdef FIRMWARE_LATEST 1486 const struct firmware *tpeeprom; 1487#else 1488 struct firmware *tpeeprom; 1489#endif 1490 1491 uint32_t version; 1492 unsigned int major, minor; 1493 int ret, len; 1494 char rev; 1495 1496 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1497 1498 major = G_TP_VERSION_MAJOR(version); 1499 minor = G_TP_VERSION_MINOR(version); 1500 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1501 return; 1502 1503 rev = t3rev2char(adap); 1504 1505 tpeeprom = firmware_get(TPEEPROM_NAME); 1506 if (tpeeprom == NULL) { 1507 device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n", 1508 TPEEPROM_NAME); 1509 return; 1510 } 1511 1512 len = tpeeprom->datasize - 4; 1513 1514 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1515 if (ret) 1516 goto release_tpeeprom; 1517 1518 if (len != TP_SRAM_LEN) { 1519 device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", TPEEPROM_NAME, len, TP_SRAM_LEN); 1520 return; 1521 } 1522 1523 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1524 TP_SRAM_OFFSET); 1525 1526 if (!ret) { 1527 device_printf(adap->dev, 1528 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1529 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1530 } else 1531 device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n"); 1532 1533release_tpeeprom: 1534 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1535 1536 return; 1537} 1538 1539static int 1540update_tpsram(struct adapter *adap) 1541{ 1542#ifdef FIRMWARE_LATEST 1543 const struct firmware *tpsram; 1544#else 1545 struct firmware *tpsram; 1546#endif 1547 int ret; 1548 char rev; 1549 1550 rev = t3rev2char(adap); 1551 if (!rev) 1552 return 0; 1553 1554 update_tpeeprom(adap); 1555 1556 tpsram = firmware_get(TPSRAM_NAME); 1557 if (tpsram == NULL){ 1558 device_printf(adap->dev, "could not load TP SRAM\n"); 1559 return (EINVAL); 1560 } else 1561 device_printf(adap->dev, "updating TP SRAM\n"); 1562 1563 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1564 if (ret) 1565 goto release_tpsram; 1566 1567 ret = t3_set_proto_sram(adap, tpsram->data); 1568 if (ret) 1569 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1570 1571release_tpsram: 1572 firmware_put(tpsram, FIRMWARE_UNLOAD); 1573 1574 return ret; 1575} 1576 1577/** 1578 * cxgb_up - enable the adapter 1579 * @adap: adapter being enabled 1580 * 1581 * Called when the first port is enabled, this function performs the 1582 * actions necessary to make an adapter operational, such as completing 1583 * the initialization of HW modules, and enabling interrupts. 1584 * 1585 */ 1586static int 1587cxgb_up(struct adapter *sc) 1588{ 1589 int err = 0; 1590 1591 if ((sc->flags & FULL_INIT_DONE) == 0) { 1592 1593 if ((sc->flags & FW_UPTODATE) == 0) 1594 if ((err = upgrade_fw(sc))) 1595 goto out; 1596 if ((sc->flags & TPS_UPTODATE) == 0) 1597 if ((err = update_tpsram(sc))) 1598 goto out; 1599 err = t3_init_hw(sc, 0); 1600 if (err) 1601 goto out; 1602 1603 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1604 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1605 1606 err = setup_sge_qsets(sc); 1607 if (err) 1608 goto out; 1609 1610 setup_rss(sc); 1611 t3_add_configured_sysctls(sc); 1612 sc->flags |= FULL_INIT_DONE; 1613 } 1614 1615 t3_intr_clear(sc); 1616 1617 /* If it's MSI or INTx, allocate a single interrupt for everything */ 1618 if ((sc->flags & USING_MSIX) == 0) { 1619 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, 1620 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { 1621 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", 1622 sc->irq_rid); 1623 err = EINVAL; 1624 goto out; 1625 } 1626 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res); 1627 1628 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET, 1629#ifdef INTR_FILTERS 1630 NULL, 1631#endif 1632 sc->cxgb_intr, sc, &sc->intr_tag)) { 1633 device_printf(sc->dev, "Cannot set up interrupt\n"); 1634 err = EINVAL; 1635 goto irq_err; 1636 } 1637 } else { 1638 cxgb_setup_msix(sc, sc->msi_count); 1639 } 1640 1641 t3_sge_start(sc); 1642 t3_intr_enable(sc); 1643 1644 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1645 is_offload(sc) && init_tp_parity(sc) == 0) 1646 sc->flags |= TP_PARITY_INIT; 1647 1648 if (sc->flags & TP_PARITY_INIT) { 1649 t3_write_reg(sc, A_TP_INT_CAUSE, 1650 F_CMCACHEPERR | F_ARPLUTPERR); 1651 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1652 } 1653 1654 1655 if (!(sc->flags & QUEUES_BOUND)) { 1656 bind_qsets(sc); 1657 sc->flags |= QUEUES_BOUND; 1658 } 1659out: 1660 return (err); 1661irq_err: 1662 CH_ERR(sc, "request_irq failed, err %d\n", err); 1663 goto out; 1664} 1665 1666 1667/* 1668 * Release resources when all the ports and offloading have been stopped. 1669 */ 1670static void 1671cxgb_down_locked(struct adapter *sc) 1672{ 1673 1674 t3_sge_stop(sc); 1675 t3_intr_disable(sc); 1676 1677 if (sc->intr_tag != NULL) { 1678 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 1679 sc->intr_tag = NULL; 1680 } 1681 if (sc->irq_res != NULL) { 1682 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n", 1683 sc->irq_rid, sc->irq_res); 1684 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 1685 sc->irq_res); 1686 sc->irq_res = NULL; 1687 } 1688 1689 if (sc->flags & USING_MSIX) 1690 cxgb_teardown_msix(sc); 1691 1692 callout_stop(&sc->cxgb_tick_ch); 1693 callout_stop(&sc->sge_timer_ch); 1694 callout_drain(&sc->cxgb_tick_ch); 1695 callout_drain(&sc->sge_timer_ch); 1696 1697 if (sc->tq != NULL) { 1698 printf("draining slow intr\n"); 1699 1700 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1701 printf("draining ext intr\n"); 1702 taskqueue_drain(sc->tq, &sc->ext_intr_task); 1703 printf("draining tick task\n"); 1704 taskqueue_drain(sc->tq, &sc->tick_task); 1705 } 1706 ADAPTER_UNLOCK(sc); 1707} 1708 1709static int 1710offload_open(struct port_info *pi) 1711{ 1712 struct adapter *adapter = pi->adapter; 1713 struct t3cdev *tdev = &adapter->tdev; 1714 1715 int adap_up = adapter->open_device_map & PORT_MASK; 1716 int err = 0; 1717 1718 if (atomic_cmpset_int(&adapter->open_device_map, 1719 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)), 1720 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0) 1721 return (0); 1722 1723 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1724 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", 1725 adapter->open_device_map); 1726 ADAPTER_LOCK(pi->adapter); 1727 if (!adap_up) 1728 err = cxgb_up(adapter); 1729 ADAPTER_UNLOCK(pi->adapter); 1730 if (err) 1731 return (err); 1732 1733 t3_tp_set_offload_mode(adapter, 1); 1734 tdev->lldev = pi->ifp; 1735 1736 init_port_mtus(adapter); 1737 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, 1738 adapter->params.b_wnd, 1739 adapter->params.rev == 0 ? 1740 adapter->port[0].ifp->if_mtu : 0xffff); 1741 init_smt(adapter); 1742 /* Call back all registered clients */ 1743 cxgb_add_clients(tdev); 1744 1745 /* restore them in case the offload module has changed them */ 1746 if (err) { 1747 t3_tp_set_offload_mode(adapter, 0); 1748 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1749 cxgb_set_dummy_ops(tdev); 1750 } 1751 return (err); 1752} 1753 1754static int 1755offload_close(struct t3cdev *tdev) 1756{ 1757 struct adapter *adapter = tdev2adap(tdev); 1758 1759 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1760 return (0); 1761 1762 /* Call back all registered clients */ 1763 cxgb_remove_clients(tdev); 1764 1765 tdev->lldev = NULL; 1766 cxgb_set_dummy_ops(tdev); 1767 t3_tp_set_offload_mode(adapter, 0); 1768 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1769 1770 ADAPTER_LOCK(adapter); 1771 if (!adapter->open_device_map) 1772 cxgb_down_locked(adapter); 1773 else 1774 ADAPTER_UNLOCK(adapter); 1775 return (0); 1776} 1777 1778 1779static void 1780cxgb_init(void *arg) 1781{ 1782 struct port_info *p = arg; 1783 1784 PORT_LOCK(p); 1785 cxgb_init_locked(p); 1786 PORT_UNLOCK(p); 1787} 1788 1789static void 1790cxgb_init_locked(struct port_info *p) 1791{ 1792 struct ifnet *ifp; 1793 adapter_t *sc = p->adapter; 1794 int err; 1795 1796 PORT_LOCK_ASSERT_OWNED(p); 1797 ifp = p->ifp; 1798 1799 ADAPTER_LOCK(p->adapter); 1800 if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) { 1801 ADAPTER_UNLOCK(p->adapter); 1802 cxgb_stop_locked(p); 1803 return; 1804 } 1805 if (p->adapter->open_device_map == 0) { 1806 t3_intr_clear(sc); 1807 } 1808 setbit(&p->adapter->open_device_map, p->port_id); 1809 ADAPTER_UNLOCK(p->adapter); 1810 1811 if (is_offload(sc) && !ofld_disable) { 1812 err = offload_open(p); 1813 if (err) 1814 log(LOG_WARNING, 1815 "Could not initialize offload capabilities\n"); 1816 } 1817#if !defined(LINK_ATTACH) 1818 cxgb_link_start(p); 1819 t3_link_changed(sc, p->port_id); 1820#endif 1821 ifp->if_baudrate = p->link_config.speed * 1000000; 1822 1823 device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); 1824 t3_port_intr_enable(sc, p->port_id); 1825 1826 t3_sge_reset_adapter(sc); 1827 1828 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1829 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1830} 1831 1832static void 1833cxgb_set_rxmode(struct port_info *p) 1834{ 1835 struct t3_rx_mode rm; 1836 struct cmac *mac = &p->mac; 1837 1838 t3_init_rx_mode(&rm, p); 1839 mtx_lock(&p->adapter->mdio_lock); 1840 t3_mac_set_rx_mode(mac, &rm); 1841 mtx_unlock(&p->adapter->mdio_lock); 1842} 1843 1844static void 1845cxgb_stop_locked(struct port_info *pi) 1846{ 1847 struct ifnet *ifp; 1848 1849 PORT_LOCK_ASSERT_OWNED(pi); 1850 ADAPTER_LOCK_ASSERT_NOTOWNED(pi->adapter); 1851 1852 ifp = pi->ifp; 1853 t3_port_intr_disable(pi->adapter, pi->port_id); 1854 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1855 1856 /* disable pause frames */ 1857 t3_set_reg_field(pi->adapter, A_XGM_TX_CFG + pi->mac.offset, 1858 F_TXPAUSEEN, 0); 1859 1860 /* Reset RX FIFO HWM */ 1861 t3_set_reg_field(pi->adapter, A_XGM_RXFIFO_CFG + pi->mac.offset, 1862 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1863 1864 1865 ADAPTER_LOCK(pi->adapter); 1866 clrbit(&pi->adapter->open_device_map, pi->port_id); 1867 1868 if (pi->adapter->open_device_map == 0) { 1869 cxgb_down_locked(pi->adapter); 1870 } else 1871 ADAPTER_UNLOCK(pi->adapter); 1872 1873#if !defined(LINK_ATTACH) 1874 DELAY(100); 1875 1876 /* Wait for TXFIFO empty */ 1877 t3_wait_op_done(pi->adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 1878 F_TXFIFO_EMPTY, 1, 20, 5); 1879 1880 DELAY(100); 1881 t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); 1882 1883 pi->phy.ops->power_down(&pi->phy, 1); 1884#endif 1885 1886} 1887 1888static int 1889cxgb_set_mtu(struct port_info *p, int mtu) 1890{ 1891 struct ifnet *ifp = p->ifp; 1892 int error = 0; 1893 1894 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1895 error = EINVAL; 1896 else if (ifp->if_mtu != mtu) { 1897 PORT_LOCK(p); 1898 ifp->if_mtu = mtu; 1899 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1900 cxgb_stop_locked(p); 1901 cxgb_init_locked(p); 1902 } 1903 PORT_UNLOCK(p); 1904 } 1905 return (error); 1906} 1907 1908#ifdef LRO_SUPPORTED 1909/* 1910 * Mark lro enabled or disabled in all qsets for this port 1911 */ 1912static int 1913cxgb_set_lro(struct port_info *p, int enabled) 1914{ 1915 int i; 1916 struct adapter *adp = p->adapter; 1917 struct sge_qset *q; 1918 1919 PORT_LOCK_ASSERT_OWNED(p); 1920 for (i = 0; i < p->nqsets; i++) { 1921 q = &adp->sge.qs[p->first_qset + i]; 1922 q->lro.enabled = (enabled != 0); 1923 } 1924 return (0); 1925} 1926#endif 1927 1928static int 1929cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1930{ 1931 struct port_info *p = ifp->if_softc; 1932 struct ifaddr *ifa = (struct ifaddr *)data; 1933 struct ifreq *ifr = (struct ifreq *)data; 1934 int flags, error = 0, reinit = 0; 1935 uint32_t mask; 1936 1937 /* 1938 * XXX need to check that we aren't in the middle of an unload 1939 */ 1940 switch (command) { 1941 case SIOCSIFMTU: 1942 error = cxgb_set_mtu(p, ifr->ifr_mtu); 1943 break; 1944 case SIOCSIFADDR: 1945 if (ifa->ifa_addr->sa_family == AF_INET) { 1946 ifp->if_flags |= IFF_UP; 1947 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1948 PORT_LOCK(p); 1949 cxgb_init_locked(p); 1950 PORT_UNLOCK(p); 1951 } 1952 arp_ifinit(ifp, ifa); 1953 } else 1954 error = ether_ioctl(ifp, command, data); 1955 break; 1956 case SIOCSIFFLAGS: 1957 PORT_LOCK(p); 1958 if (ifp->if_flags & IFF_UP) { 1959 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1960 flags = p->if_flags; 1961 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1962 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) 1963 cxgb_set_rxmode(p); 1964 } else 1965 cxgb_init_locked(p); 1966 p->if_flags = ifp->if_flags; 1967 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1968 cxgb_stop_locked(p); 1969 1970 PORT_UNLOCK(p); 1971 break; 1972 case SIOCADDMULTI: 1973 case SIOCDELMULTI: 1974 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1975 cxgb_set_rxmode(p); 1976 } 1977 break; 1978 case SIOCSIFMEDIA: 1979 case SIOCGIFMEDIA: 1980 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 1981 break; 1982 case SIOCSIFCAP: 1983 PORT_LOCK(p); 1984 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1985 if (mask & IFCAP_TXCSUM) { 1986 if (IFCAP_TXCSUM & ifp->if_capenable) { 1987 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); 1988 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP 1989 | CSUM_IP | CSUM_TSO); 1990 } else { 1991 ifp->if_capenable |= IFCAP_TXCSUM; 1992 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP 1993 | CSUM_IP); 1994 } 1995 } 1996 if (mask & IFCAP_RXCSUM) { 1997 ifp->if_capenable ^= IFCAP_RXCSUM; 1998 } 1999 if (mask & IFCAP_TSO4) { 2000 if (IFCAP_TSO4 & ifp->if_capenable) { 2001 ifp->if_capenable &= ~IFCAP_TSO4; 2002 ifp->if_hwassist &= ~CSUM_TSO; 2003 } else if (IFCAP_TXCSUM & ifp->if_capenable) { 2004 ifp->if_capenable |= IFCAP_TSO4; 2005 ifp->if_hwassist |= CSUM_TSO; 2006 } else { 2007 if (cxgb_debug) 2008 printf("cxgb requires tx checksum offload" 2009 " be enabled to use TSO\n"); 2010 error = EINVAL; 2011 } 2012 } 2013#ifdef LRO_SUPPORTED 2014 if (mask & IFCAP_LRO) { 2015 ifp->if_capenable ^= IFCAP_LRO; 2016 2017 /* Safe to do this even if cxgb_up not called yet */ 2018 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2019 } 2020#endif 2021 if (mask & IFCAP_VLAN_HWTAGGING) { 2022 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2023 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2024 } 2025 if (mask & IFCAP_VLAN_MTU) { 2026 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2027 reinit = ifp->if_drv_flags & IFF_DRV_RUNNING; 2028 } 2029 if (mask & IFCAP_VLAN_HWCSUM) { 2030 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2031 } 2032 if (reinit) { 2033 cxgb_stop_locked(p); 2034 cxgb_init_locked(p); 2035 } 2036 PORT_UNLOCK(p); 2037 2038#ifdef VLAN_CAPABILITIES 2039 VLAN_CAPABILITIES(ifp); 2040#endif 2041 break; 2042 default: 2043 error = ether_ioctl(ifp, command, data); 2044 break; 2045 } 2046 return (error); 2047} 2048 2049static int 2050cxgb_media_change(struct ifnet *ifp) 2051{ 2052 if_printf(ifp, "media change not supported\n"); 2053 return (ENXIO); 2054} 2055 2056static void 2057cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2058{ 2059 struct port_info *p = ifp->if_softc; 2060 2061 ifmr->ifm_status = IFM_AVALID; 2062 ifmr->ifm_active = IFM_ETHER; 2063 2064 if (!p->link_config.link_ok) 2065 return; 2066 2067 ifmr->ifm_status |= IFM_ACTIVE; 2068 2069 switch (p->link_config.speed) { 2070 case 10: 2071 ifmr->ifm_active |= IFM_10_T; 2072 break; 2073 case 100: 2074 ifmr->ifm_active |= IFM_100_TX; 2075 break; 2076 case 1000: 2077 ifmr->ifm_active |= IFM_1000_T; 2078 break; 2079 } 2080 2081 if (p->link_config.duplex) 2082 ifmr->ifm_active |= IFM_FDX; 2083 else 2084 ifmr->ifm_active |= IFM_HDX; 2085} 2086 2087static void 2088cxgb_async_intr(void *data) 2089{ 2090 adapter_t *sc = data; 2091 2092 if (cxgb_debug) 2093 device_printf(sc->dev, "cxgb_async_intr\n"); 2094 /* 2095 * May need to sleep - defer to taskqueue 2096 */ 2097 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2098} 2099 2100static void 2101cxgb_ext_intr_handler(void *arg, int count) 2102{ 2103 adapter_t *sc = (adapter_t *)arg; 2104 2105 if (cxgb_debug) 2106 printf("cxgb_ext_intr_handler\n"); 2107 2108 t3_phy_intr_handler(sc); 2109 2110 /* Now reenable external interrupts */ 2111 ADAPTER_LOCK(sc); 2112 if (sc->slow_intr_mask) { 2113 sc->slow_intr_mask |= F_T3DBG; 2114 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 2115 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 2116 } 2117 ADAPTER_UNLOCK(sc); 2118} 2119 2120static void 2121check_link_status(adapter_t *sc) 2122{ 2123 int i; 2124 2125 for (i = 0; i < (sc)->params.nports; ++i) { 2126 struct port_info *p = &sc->port[i]; 2127 2128 if (!(p->phy.caps & SUPPORTED_IRQ)) 2129 t3_link_changed(sc, i); 2130 p->ifp->if_baudrate = p->link_config.speed * 1000000; 2131 } 2132} 2133 2134static void 2135check_t3b2_mac(struct adapter *adapter) 2136{ 2137 int i; 2138 2139 if(adapter->flags & CXGB_SHUTDOWN) 2140 return; 2141 2142 for_each_port(adapter, i) { 2143 struct port_info *p = &adapter->port[i]; 2144 struct ifnet *ifp = p->ifp; 2145 int status; 2146 2147 if(adapter->flags & CXGB_SHUTDOWN) 2148 return; 2149 2150 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2151 continue; 2152 2153 status = 0; 2154 PORT_LOCK(p); 2155 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) 2156 status = t3b2_mac_watchdog_task(&p->mac); 2157 if (status == 1) 2158 p->mac.stats.num_toggled++; 2159 else if (status == 2) { 2160 struct cmac *mac = &p->mac; 2161 int mtu = ifp->if_mtu; 2162 2163 if (ifp->if_capenable & IFCAP_VLAN_MTU) 2164 mtu += ETHER_VLAN_ENCAP_LEN; 2165 t3_mac_set_mtu(mac, mtu); 2166 t3_mac_set_address(mac, 0, p->hw_addr); 2167 cxgb_set_rxmode(p); 2168 t3_link_start(&p->phy, mac, &p->link_config); 2169 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2170 t3_port_intr_enable(adapter, p->port_id); 2171 p->mac.stats.num_resets++; 2172 } 2173 PORT_UNLOCK(p); 2174 } 2175} 2176 2177static void 2178cxgb_tick(void *arg) 2179{ 2180 adapter_t *sc = (adapter_t *)arg; 2181 2182 if(sc->flags & CXGB_SHUTDOWN) 2183 return; 2184 2185 taskqueue_enqueue(sc->tq, &sc->tick_task); 2186 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 2187} 2188 2189static void 2190cxgb_tick_handler(void *arg, int count) 2191{ 2192 adapter_t *sc = (adapter_t *)arg; 2193 const struct adapter_params *p = &sc->params; 2194 int i; 2195 2196 if(sc->flags & CXGB_SHUTDOWN) 2197 return; 2198 2199 ADAPTER_LOCK(sc); 2200 if (p->linkpoll_period) 2201 check_link_status(sc); 2202 2203 sc->check_task_cnt++; 2204 2205 /* 2206 * adapter lock can currently only be acquired after the 2207 * port lock 2208 */ 2209 ADAPTER_UNLOCK(sc); 2210 2211 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2212 check_t3b2_mac(sc); 2213 2214 /* Update MAC stats if it's time to do so */ 2215 if (!p->linkpoll_period || 2216 (sc->check_task_cnt * p->linkpoll_period) / 10 >= 2217 p->stats_update_period) { 2218 for_each_port(sc, i) { 2219 struct port_info *port = &sc->port[i]; 2220 PORT_LOCK(port); 2221 t3_mac_update_stats(&port->mac); 2222 PORT_UNLOCK(port); 2223 } 2224 sc->check_task_cnt = 0; 2225 } 2226} 2227 2228static void 2229touch_bars(device_t dev) 2230{ 2231 /* 2232 * Don't enable yet 2233 */ 2234#if !defined(__LP64__) && 0 2235 u32 v; 2236 2237 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2238 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2239 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2240 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2241 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2242 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2243#endif 2244} 2245 2246static int 2247set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2248{ 2249 uint8_t *buf; 2250 int err = 0; 2251 u32 aligned_offset, aligned_len, *p; 2252 struct adapter *adapter = pi->adapter; 2253 2254 2255 aligned_offset = offset & ~3; 2256 aligned_len = (len + (offset & 3) + 3) & ~3; 2257 2258 if (aligned_offset != offset || aligned_len != len) { 2259 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2260 if (!buf) 2261 return (ENOMEM); 2262 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2263 if (!err && aligned_len > 4) 2264 err = t3_seeprom_read(adapter, 2265 aligned_offset + aligned_len - 4, 2266 (u32 *)&buf[aligned_len - 4]); 2267 if (err) 2268 goto out; 2269 memcpy(buf + (offset & 3), data, len); 2270 } else 2271 buf = (uint8_t *)(uintptr_t)data; 2272 2273 err = t3_seeprom_wp(adapter, 0); 2274 if (err) 2275 goto out; 2276 2277 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2278 err = t3_seeprom_write(adapter, aligned_offset, *p); 2279 aligned_offset += 4; 2280 } 2281 2282 if (!err) 2283 err = t3_seeprom_wp(adapter, 1); 2284out: 2285 if (buf != data) 2286 free(buf, M_DEVBUF); 2287 return err; 2288} 2289 2290 2291static int 2292in_range(int val, int lo, int hi) 2293{ 2294 return val < 0 || (val <= hi && val >= lo); 2295} 2296 2297static int 2298cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td) 2299{ 2300 return (0); 2301} 2302 2303static int 2304cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td) 2305{ 2306 return (0); 2307} 2308 2309static int 2310cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2311 int fflag, struct thread *td) 2312{ 2313 int mmd, error = 0; 2314 struct port_info *pi = dev->si_drv1; 2315 adapter_t *sc = pi->adapter; 2316 2317#ifdef PRIV_SUPPORTED 2318 if (priv_check(td, PRIV_DRIVER)) { 2319 if (cxgb_debug) 2320 printf("user does not have access to privileged ioctls\n"); 2321 return (EPERM); 2322 } 2323#else 2324 if (suser(td)) { 2325 if (cxgb_debug) 2326 printf("user does not have access to privileged ioctls\n"); 2327 return (EPERM); 2328 } 2329#endif 2330 2331 switch (cmd) { 2332 case CHELSIO_GET_MIIREG: { 2333 uint32_t val; 2334 struct cphy *phy = &pi->phy; 2335 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2336 2337 if (!phy->mdio_read) 2338 return (EOPNOTSUPP); 2339 if (is_10G(sc)) { 2340 mmd = mid->phy_id >> 8; 2341 if (!mmd) 2342 mmd = MDIO_DEV_PCS; 2343 else if (mmd > MDIO_DEV_XGXS) 2344 return (EINVAL); 2345 2346 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2347 mid->reg_num, &val); 2348 } else 2349 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2350 mid->reg_num & 0x1f, &val); 2351 if (error == 0) 2352 mid->val_out = val; 2353 break; 2354 } 2355 case CHELSIO_SET_MIIREG: { 2356 struct cphy *phy = &pi->phy; 2357 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2358 2359 if (!phy->mdio_write) 2360 return (EOPNOTSUPP); 2361 if (is_10G(sc)) { 2362 mmd = mid->phy_id >> 8; 2363 if (!mmd) 2364 mmd = MDIO_DEV_PCS; 2365 else if (mmd > MDIO_DEV_XGXS) 2366 return (EINVAL); 2367 2368 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2369 mmd, mid->reg_num, mid->val_in); 2370 } else 2371 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2372 mid->reg_num & 0x1f, 2373 mid->val_in); 2374 break; 2375 } 2376 case CHELSIO_SETREG: { 2377 struct ch_reg *edata = (struct ch_reg *)data; 2378 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2379 return (EFAULT); 2380 t3_write_reg(sc, edata->addr, edata->val); 2381 break; 2382 } 2383 case CHELSIO_GETREG: { 2384 struct ch_reg *edata = (struct ch_reg *)data; 2385 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2386 return (EFAULT); 2387 edata->val = t3_read_reg(sc, edata->addr); 2388 break; 2389 } 2390 case CHELSIO_GET_SGE_CONTEXT: { 2391 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2392 mtx_lock_spin(&sc->sge.reg_lock); 2393 switch (ecntxt->cntxt_type) { 2394 case CNTXT_TYPE_EGRESS: 2395 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2396 ecntxt->data); 2397 break; 2398 case CNTXT_TYPE_FL: 2399 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2400 ecntxt->data); 2401 break; 2402 case CNTXT_TYPE_RSP: 2403 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2404 ecntxt->data); 2405 break; 2406 case CNTXT_TYPE_CQ: 2407 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2408 ecntxt->data); 2409 break; 2410 default: 2411 error = EINVAL; 2412 break; 2413 } 2414 mtx_unlock_spin(&sc->sge.reg_lock); 2415 break; 2416 } 2417 case CHELSIO_GET_SGE_DESC: { 2418 struct ch_desc *edesc = (struct ch_desc *)data; 2419 int ret; 2420 if (edesc->queue_num >= SGE_QSETS * 6) 2421 return (EINVAL); 2422 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2423 edesc->queue_num % 6, edesc->idx, edesc->data); 2424 if (ret < 0) 2425 return (EINVAL); 2426 edesc->size = ret; 2427 break; 2428 } 2429 case CHELSIO_GET_QSET_PARAMS: { 2430 struct qset_params *q; 2431 struct ch_qset_params *t = (struct ch_qset_params *)data; 2432 int q1 = pi->first_qset; 2433 int nqsets = pi->nqsets; 2434 int i; 2435 2436 if (t->qset_idx >= nqsets) 2437 return EINVAL; 2438 2439 i = q1 + t->qset_idx; 2440 q = &sc->params.sge.qset[i]; 2441 t->rspq_size = q->rspq_size; 2442 t->txq_size[0] = q->txq_size[0]; 2443 t->txq_size[1] = q->txq_size[1]; 2444 t->txq_size[2] = q->txq_size[2]; 2445 t->fl_size[0] = q->fl_size; 2446 t->fl_size[1] = q->jumbo_size; 2447 t->polling = q->polling; 2448 t->lro = q->lro; 2449 t->intr_lat = q->coalesce_usecs; 2450 t->cong_thres = q->cong_thres; 2451 t->qnum = i; 2452 2453 if (sc->flags & USING_MSIX) 2454 t->vector = rman_get_start(sc->msix_irq_res[i]); 2455 else 2456 t->vector = rman_get_start(sc->irq_res); 2457 2458 break; 2459 } 2460 case CHELSIO_GET_QSET_NUM: { 2461 struct ch_reg *edata = (struct ch_reg *)data; 2462 edata->val = pi->nqsets; 2463 break; 2464 } 2465 case CHELSIO_LOAD_FW: { 2466 uint8_t *fw_data; 2467 uint32_t vers; 2468 struct ch_mem_range *t = (struct ch_mem_range *)data; 2469 2470 /* 2471 * You're allowed to load a firmware only before FULL_INIT_DONE 2472 * 2473 * FW_UPTODATE is also set so the rest of the initialization 2474 * will not overwrite what was loaded here. This gives you the 2475 * flexibility to load any firmware (and maybe shoot yourself in 2476 * the foot). 2477 */ 2478 2479 ADAPTER_LOCK(sc); 2480 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2481 ADAPTER_UNLOCK(sc); 2482 return (EBUSY); 2483 } 2484 2485 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2486 if (!fw_data) 2487 error = ENOMEM; 2488 else 2489 error = copyin(t->buf, fw_data, t->len); 2490 2491 if (!error) 2492 error = -t3_load_fw(sc, fw_data, t->len); 2493 2494 if (t3_get_fw_version(sc, &vers) == 0) { 2495 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2496 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2497 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2498 } 2499 2500 if (!error) 2501 sc->flags |= FW_UPTODATE; 2502 2503 free(fw_data, M_DEVBUF); 2504 ADAPTER_UNLOCK(sc); 2505 break; 2506 } 2507 case CHELSIO_LOAD_BOOT: { 2508 uint8_t *boot_data; 2509 struct ch_mem_range *t = (struct ch_mem_range *)data; 2510 2511 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2512 if (!boot_data) 2513 return ENOMEM; 2514 2515 error = copyin(t->buf, boot_data, t->len); 2516 if (!error) 2517 error = -t3_load_boot(sc, boot_data, t->len); 2518 2519 free(boot_data, M_DEVBUF); 2520 break; 2521 } 2522 case CHELSIO_GET_PM: { 2523 struct ch_pm *m = (struct ch_pm *)data; 2524 struct tp_params *p = &sc->params.tp; 2525 2526 if (!is_offload(sc)) 2527 return (EOPNOTSUPP); 2528 2529 m->tx_pg_sz = p->tx_pg_size; 2530 m->tx_num_pg = p->tx_num_pgs; 2531 m->rx_pg_sz = p->rx_pg_size; 2532 m->rx_num_pg = p->rx_num_pgs; 2533 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2534 2535 break; 2536 } 2537 case CHELSIO_SET_PM: { 2538 struct ch_pm *m = (struct ch_pm *)data; 2539 struct tp_params *p = &sc->params.tp; 2540 2541 if (!is_offload(sc)) 2542 return (EOPNOTSUPP); 2543 if (sc->flags & FULL_INIT_DONE) 2544 return (EBUSY); 2545 2546 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2547 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2548 return (EINVAL); /* not power of 2 */ 2549 if (!(m->rx_pg_sz & 0x14000)) 2550 return (EINVAL); /* not 16KB or 64KB */ 2551 if (!(m->tx_pg_sz & 0x1554000)) 2552 return (EINVAL); 2553 if (m->tx_num_pg == -1) 2554 m->tx_num_pg = p->tx_num_pgs; 2555 if (m->rx_num_pg == -1) 2556 m->rx_num_pg = p->rx_num_pgs; 2557 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2558 return (EINVAL); 2559 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2560 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2561 return (EINVAL); 2562 2563 p->rx_pg_size = m->rx_pg_sz; 2564 p->tx_pg_size = m->tx_pg_sz; 2565 p->rx_num_pgs = m->rx_num_pg; 2566 p->tx_num_pgs = m->tx_num_pg; 2567 break; 2568 } 2569 case CHELSIO_SETMTUTAB: { 2570 struct ch_mtus *m = (struct ch_mtus *)data; 2571 int i; 2572 2573 if (!is_offload(sc)) 2574 return (EOPNOTSUPP); 2575 if (offload_running(sc)) 2576 return (EBUSY); 2577 if (m->nmtus != NMTUS) 2578 return (EINVAL); 2579 if (m->mtus[0] < 81) /* accommodate SACK */ 2580 return (EINVAL); 2581 2582 /* 2583 * MTUs must be in ascending order 2584 */ 2585 for (i = 1; i < NMTUS; ++i) 2586 if (m->mtus[i] < m->mtus[i - 1]) 2587 return (EINVAL); 2588 2589 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2590 break; 2591 } 2592 case CHELSIO_GETMTUTAB: { 2593 struct ch_mtus *m = (struct ch_mtus *)data; 2594 2595 if (!is_offload(sc)) 2596 return (EOPNOTSUPP); 2597 2598 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2599 m->nmtus = NMTUS; 2600 break; 2601 } 2602 case CHELSIO_GET_MEM: { 2603 struct ch_mem_range *t = (struct ch_mem_range *)data; 2604 struct mc7 *mem; 2605 uint8_t *useraddr; 2606 u64 buf[32]; 2607 2608 /* 2609 * Use these to avoid modifying len/addr in the the return 2610 * struct 2611 */ 2612 uint32_t len = t->len, addr = t->addr; 2613 2614 if (!is_offload(sc)) 2615 return (EOPNOTSUPP); 2616 if (!(sc->flags & FULL_INIT_DONE)) 2617 return (EIO); /* need the memory controllers */ 2618 if ((addr & 0x7) || (len & 0x7)) 2619 return (EINVAL); 2620 if (t->mem_id == MEM_CM) 2621 mem = &sc->cm; 2622 else if (t->mem_id == MEM_PMRX) 2623 mem = &sc->pmrx; 2624 else if (t->mem_id == MEM_PMTX) 2625 mem = &sc->pmtx; 2626 else 2627 return (EINVAL); 2628 2629 /* 2630 * Version scheme: 2631 * bits 0..9: chip version 2632 * bits 10..15: chip revision 2633 */ 2634 t->version = 3 | (sc->params.rev << 10); 2635 2636 /* 2637 * Read 256 bytes at a time as len can be large and we don't 2638 * want to use huge intermediate buffers. 2639 */ 2640 useraddr = (uint8_t *)t->buf; 2641 while (len) { 2642 unsigned int chunk = min(len, sizeof(buf)); 2643 2644 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2645 if (error) 2646 return (-error); 2647 if (copyout(buf, useraddr, chunk)) 2648 return (EFAULT); 2649 useraddr += chunk; 2650 addr += chunk; 2651 len -= chunk; 2652 } 2653 break; 2654 } 2655 case CHELSIO_READ_TCAM_WORD: { 2656 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2657 2658 if (!is_offload(sc)) 2659 return (EOPNOTSUPP); 2660 if (!(sc->flags & FULL_INIT_DONE)) 2661 return (EIO); /* need MC5 */ 2662 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2663 break; 2664 } 2665 case CHELSIO_SET_TRACE_FILTER: { 2666 struct ch_trace *t = (struct ch_trace *)data; 2667 const struct trace_params *tp; 2668 2669 tp = (const struct trace_params *)&t->sip; 2670 if (t->config_tx) 2671 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2672 t->trace_tx); 2673 if (t->config_rx) 2674 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2675 t->trace_rx); 2676 break; 2677 } 2678 case CHELSIO_SET_PKTSCHED: { 2679 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2680 if (sc->open_device_map == 0) 2681 return (EAGAIN); 2682 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2683 p->binding); 2684 break; 2685 } 2686 case CHELSIO_IFCONF_GETREGS: { 2687 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2688 int reglen = cxgb_get_regs_len(); 2689 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2690 if (buf == NULL) { 2691 return (ENOMEM); 2692 } 2693 if (regs->len > reglen) 2694 regs->len = reglen; 2695 else if (regs->len < reglen) 2696 error = E2BIG; 2697 2698 if (!error) { 2699 cxgb_get_regs(sc, regs, buf); 2700 error = copyout(buf, regs->data, reglen); 2701 } 2702 free(buf, M_DEVBUF); 2703 2704 break; 2705 } 2706 case CHELSIO_SET_HW_SCHED: { 2707 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2708 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2709 2710 if ((sc->flags & FULL_INIT_DONE) == 0) 2711 return (EAGAIN); /* need TP to be initialized */ 2712 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2713 !in_range(t->channel, 0, 1) || 2714 !in_range(t->kbps, 0, 10000000) || 2715 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2716 !in_range(t->flow_ipg, 0, 2717 dack_ticks_to_usec(sc, 0x7ff))) 2718 return (EINVAL); 2719 2720 if (t->kbps >= 0) { 2721 error = t3_config_sched(sc, t->kbps, t->sched); 2722 if (error < 0) 2723 return (-error); 2724 } 2725 if (t->class_ipg >= 0) 2726 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2727 if (t->flow_ipg >= 0) { 2728 t->flow_ipg *= 1000; /* us -> ns */ 2729 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2730 } 2731 if (t->mode >= 0) { 2732 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2733 2734 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2735 bit, t->mode ? bit : 0); 2736 } 2737 if (t->channel >= 0) 2738 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2739 1 << t->sched, t->channel << t->sched); 2740 break; 2741 } 2742 case CHELSIO_GET_EEPROM: { 2743 int i; 2744 struct ch_eeprom *e = (struct ch_eeprom *)data; 2745 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2746 2747 if (buf == NULL) { 2748 return (ENOMEM); 2749 } 2750 e->magic = EEPROM_MAGIC; 2751 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2752 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2753 2754 if (!error) 2755 error = copyout(buf + e->offset, e->data, e->len); 2756 2757 free(buf, M_DEVBUF); 2758 break; 2759 } 2760 case CHELSIO_CLEAR_STATS: { 2761 if (!(sc->flags & FULL_INIT_DONE)) 2762 return EAGAIN; 2763 2764 PORT_LOCK(pi); 2765 t3_mac_update_stats(&pi->mac); 2766 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 2767 PORT_UNLOCK(pi); 2768 break; 2769 } 2770 default: 2771 return (EOPNOTSUPP); 2772 break; 2773 } 2774 2775 return (error); 2776} 2777 2778static __inline void 2779reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 2780 unsigned int end) 2781{ 2782 uint32_t *p = (uint32_t *)(buf + start); 2783 2784 for ( ; start <= end; start += sizeof(uint32_t)) 2785 *p++ = t3_read_reg(ap, start); 2786} 2787 2788#define T3_REGMAP_SIZE (3 * 1024) 2789static int 2790cxgb_get_regs_len(void) 2791{ 2792 return T3_REGMAP_SIZE; 2793} 2794 2795static void 2796cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 2797{ 2798 2799 /* 2800 * Version scheme: 2801 * bits 0..9: chip version 2802 * bits 10..15: chip revision 2803 * bit 31: set for PCIe cards 2804 */ 2805 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 2806 2807 /* 2808 * We skip the MAC statistics registers because they are clear-on-read. 2809 * Also reading multi-register stats would need to synchronize with the 2810 * periodic mac stats accumulation. Hard to justify the complexity. 2811 */ 2812 memset(buf, 0, cxgb_get_regs_len()); 2813 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 2814 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 2815 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 2816 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 2817 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 2818 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 2819 XGM_REG(A_XGM_SERDES_STAT3, 1)); 2820 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 2821 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 2822} 2823 2824 2825MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1); 2826