cxgb_main.c revision 207643
1/************************************************************************** 2 3Copyright (c) 2007-2009, Chelsio Inc. 4All rights reserved. 5 6Redistribution and use in source and binary forms, with or without 7modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26POSSIBILITY OF SUCH DAMAGE. 27 28***************************************************************************/ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: head/sys/dev/cxgb/cxgb_main.c 207643 2010-05-05 00:41:40Z np $"); 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/kernel.h> 36#include <sys/bus.h> 37#include <sys/module.h> 38#include <sys/pciio.h> 39#include <sys/conf.h> 40#include <machine/bus.h> 41#include <machine/resource.h> 42#include <sys/bus_dma.h> 43#include <sys/ktr.h> 44#include <sys/rman.h> 45#include <sys/ioccom.h> 46#include <sys/mbuf.h> 47#include <sys/linker.h> 48#include <sys/firmware.h> 49#include <sys/socket.h> 50#include <sys/sockio.h> 51#include <sys/smp.h> 52#include <sys/sysctl.h> 53#include <sys/syslog.h> 54#include <sys/queue.h> 55#include <sys/taskqueue.h> 56#include <sys/proc.h> 57 58#include <net/bpf.h> 59#include <net/ethernet.h> 60#include <net/if.h> 61#include <net/if_arp.h> 62#include <net/if_dl.h> 63#include <net/if_media.h> 64#include <net/if_types.h> 65#include <net/if_vlan_var.h> 66 67#include <netinet/in_systm.h> 68#include <netinet/in.h> 69#include <netinet/if_ether.h> 70#include <netinet/ip.h> 71#include <netinet/ip.h> 72#include <netinet/tcp.h> 73#include <netinet/udp.h> 74 75#include <dev/pci/pcireg.h> 76#include <dev/pci/pcivar.h> 77#include <dev/pci/pci_private.h> 78 79#include <cxgb_include.h> 80 81#ifdef PRIV_SUPPORTED 82#include <sys/priv.h> 83#endif 84 85static int cxgb_setup_interrupts(adapter_t *); 86static void cxgb_teardown_interrupts(adapter_t *); 87static void cxgb_init(void *); 88static int cxgb_init_locked(struct port_info *); 89static int cxgb_uninit_locked(struct port_info *); 90static int cxgb_uninit_synchronized(struct port_info *); 91static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 92static int cxgb_media_change(struct ifnet *); 93static int cxgb_ifm_type(int); 94static void cxgb_build_medialist(struct port_info *); 95static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 96static int setup_sge_qsets(adapter_t *); 97static void cxgb_async_intr(void *); 98static void cxgb_ext_intr_handler(void *, int); 99static void cxgb_tick_handler(void *, int); 100static void cxgb_tick(void *); 101static void setup_rss(adapter_t *sc); 102static int alloc_filters(struct adapter *); 103static int setup_hw_filters(struct adapter *); 104static int set_filter(struct adapter *, int, const struct filter_info *); 105static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int, 106 unsigned int, u64, u64); 107static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int, 108 unsigned int, u64, u64); 109 110/* Attachment glue for the PCI controller end of the device. Each port of 111 * the device is attached separately, as defined later. 112 */ 113static int cxgb_controller_probe(device_t); 114static int cxgb_controller_attach(device_t); 115static int cxgb_controller_detach(device_t); 116static void cxgb_free(struct adapter *); 117static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 118 unsigned int end); 119static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 120static int cxgb_get_regs_len(void); 121static int offload_open(struct port_info *pi); 122static void touch_bars(device_t dev); 123static int offload_close(struct t3cdev *tdev); 124static void cxgb_update_mac_settings(struct port_info *p); 125 126static device_method_t cxgb_controller_methods[] = { 127 DEVMETHOD(device_probe, cxgb_controller_probe), 128 DEVMETHOD(device_attach, cxgb_controller_attach), 129 DEVMETHOD(device_detach, cxgb_controller_detach), 130 131 /* bus interface */ 132 DEVMETHOD(bus_print_child, bus_generic_print_child), 133 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 134 135 { 0, 0 } 136}; 137 138static driver_t cxgb_controller_driver = { 139 "cxgbc", 140 cxgb_controller_methods, 141 sizeof(struct adapter) 142}; 143 144static devclass_t cxgb_controller_devclass; 145DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0); 146 147/* 148 * Attachment glue for the ports. Attachment is done directly to the 149 * controller device. 150 */ 151static int cxgb_port_probe(device_t); 152static int cxgb_port_attach(device_t); 153static int cxgb_port_detach(device_t); 154 155static device_method_t cxgb_port_methods[] = { 156 DEVMETHOD(device_probe, cxgb_port_probe), 157 DEVMETHOD(device_attach, cxgb_port_attach), 158 DEVMETHOD(device_detach, cxgb_port_detach), 159 { 0, 0 } 160}; 161 162static driver_t cxgb_port_driver = { 163 "cxgb", 164 cxgb_port_methods, 165 0 166}; 167 168static d_ioctl_t cxgb_extension_ioctl; 169static d_open_t cxgb_extension_open; 170static d_close_t cxgb_extension_close; 171 172static struct cdevsw cxgb_cdevsw = { 173 .d_version = D_VERSION, 174 .d_flags = 0, 175 .d_open = cxgb_extension_open, 176 .d_close = cxgb_extension_close, 177 .d_ioctl = cxgb_extension_ioctl, 178 .d_name = "cxgb", 179}; 180 181static devclass_t cxgb_port_devclass; 182DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 183 184/* 185 * The driver uses the best interrupt scheme available on a platform in the 186 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 187 * of these schemes the driver may consider as follows: 188 * 189 * msi = 2: choose from among all three options 190 * msi = 1 : only consider MSI and pin interrupts 191 * msi = 0: force pin interrupts 192 */ 193static int msi_allowed = 2; 194 195TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 196SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 197SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 198 "MSI-X, MSI, INTx selector"); 199 200/* 201 * The driver enables offload as a default. 202 * To disable it, use ofld_disable = 1. 203 */ 204static int ofld_disable = 0; 205TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable); 206SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0, 207 "disable ULP offload"); 208 209/* 210 * The driver uses an auto-queue algorithm by default. 211 * To disable it and force a single queue-set per port, use multiq = 0 212 */ 213static int multiq = 1; 214TUNABLE_INT("hw.cxgb.multiq", &multiq); 215SYSCTL_UINT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 216 "use min(ncpus/ports, 8) queue-sets per port"); 217 218/* 219 * By default the driver will not update the firmware unless 220 * it was compiled against a newer version 221 * 222 */ 223static int force_fw_update = 0; 224TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update); 225SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 226 "update firmware even if up to date"); 227 228int cxgb_use_16k_clusters = -1; 229TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters); 230SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 231 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 232 233/* 234 * Tune the size of the output queue. 235 */ 236int cxgb_snd_queue_len = IFQ_MAXLEN; 237TUNABLE_INT("hw.cxgb.snd_queue_len", &cxgb_snd_queue_len); 238SYSCTL_UINT(_hw_cxgb, OID_AUTO, snd_queue_len, CTLFLAG_RDTUN, 239 &cxgb_snd_queue_len, 0, "send queue size "); 240 241 242enum { 243 MAX_TXQ_ENTRIES = 16384, 244 MAX_CTRL_TXQ_ENTRIES = 1024, 245 MAX_RSPQ_ENTRIES = 16384, 246 MAX_RX_BUFFERS = 16384, 247 MAX_RX_JUMBO_BUFFERS = 16384, 248 MIN_TXQ_ENTRIES = 4, 249 MIN_CTRL_TXQ_ENTRIES = 4, 250 MIN_RSPQ_ENTRIES = 32, 251 MIN_FL_ENTRIES = 32, 252 MIN_FL_JUMBO_ENTRIES = 32 253}; 254 255struct filter_info { 256 u32 sip; 257 u32 sip_mask; 258 u32 dip; 259 u16 sport; 260 u16 dport; 261 u32 vlan:12; 262 u32 vlan_prio:3; 263 u32 mac_hit:1; 264 u32 mac_idx:4; 265 u32 mac_vld:1; 266 u32 pkt_type:2; 267 u32 report_filter_id:1; 268 u32 pass:1; 269 u32 rss:1; 270 u32 qset:3; 271 u32 locked:1; 272 u32 valid:1; 273}; 274 275enum { FILTER_NO_VLAN_PRI = 7 }; 276 277#define EEPROM_MAGIC 0x38E2F10C 278 279#define PORT_MASK ((1 << MAX_NPORTS) - 1) 280 281/* Table for probing the cards. The desc field isn't actually used */ 282struct cxgb_ident { 283 uint16_t vendor; 284 uint16_t device; 285 int index; 286 char *desc; 287} cxgb_identifiers[] = { 288 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 289 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 290 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 291 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 292 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 293 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 294 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 295 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 296 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 297 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 298 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 299 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"}, 300 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"}, 301 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"}, 302 {0, 0, 0, NULL} 303}; 304 305static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 306 307 308static __inline char 309t3rev2char(struct adapter *adapter) 310{ 311 char rev = 'z'; 312 313 switch(adapter->params.rev) { 314 case T3_REV_A: 315 rev = 'a'; 316 break; 317 case T3_REV_B: 318 case T3_REV_B2: 319 rev = 'b'; 320 break; 321 case T3_REV_C: 322 rev = 'c'; 323 break; 324 } 325 return rev; 326} 327 328static struct cxgb_ident * 329cxgb_get_ident(device_t dev) 330{ 331 struct cxgb_ident *id; 332 333 for (id = cxgb_identifiers; id->desc != NULL; id++) { 334 if ((id->vendor == pci_get_vendor(dev)) && 335 (id->device == pci_get_device(dev))) { 336 return (id); 337 } 338 } 339 return (NULL); 340} 341 342static const struct adapter_info * 343cxgb_get_adapter_info(device_t dev) 344{ 345 struct cxgb_ident *id; 346 const struct adapter_info *ai; 347 348 id = cxgb_get_ident(dev); 349 if (id == NULL) 350 return (NULL); 351 352 ai = t3_get_adapter_info(id->index); 353 354 return (ai); 355} 356 357static int 358cxgb_controller_probe(device_t dev) 359{ 360 const struct adapter_info *ai; 361 char *ports, buf[80]; 362 int nports; 363 364 ai = cxgb_get_adapter_info(dev); 365 if (ai == NULL) 366 return (ENXIO); 367 368 nports = ai->nports0 + ai->nports1; 369 if (nports == 1) 370 ports = "port"; 371 else 372 ports = "ports"; 373 374 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports); 375 device_set_desc_copy(dev, buf); 376 return (BUS_PROBE_DEFAULT); 377} 378 379#define FW_FNAME "cxgb_t3fw" 380#define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 381#define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 382 383static int 384upgrade_fw(adapter_t *sc) 385{ 386 const struct firmware *fw; 387 int status; 388 u32 vers; 389 390 if ((fw = firmware_get(FW_FNAME)) == NULL) { 391 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 392 return (ENOENT); 393 } else 394 device_printf(sc->dev, "installing firmware on card\n"); 395 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 396 397 if (status != 0) { 398 device_printf(sc->dev, "failed to install firmware: %d\n", 399 status); 400 } else { 401 t3_get_fw_version(sc, &vers); 402 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 403 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 404 G_FW_VERSION_MICRO(vers)); 405 } 406 407 firmware_put(fw, FIRMWARE_UNLOAD); 408 409 return (status); 410} 411 412/* 413 * The cxgb_controller_attach function is responsible for the initial 414 * bringup of the device. Its responsibilities include: 415 * 416 * 1. Determine if the device supports MSI or MSI-X. 417 * 2. Allocate bus resources so that we can access the Base Address Register 418 * 3. Create and initialize mutexes for the controller and its control 419 * logic such as SGE and MDIO. 420 * 4. Call hardware specific setup routine for the adapter as a whole. 421 * 5. Allocate the BAR for doing MSI-X. 422 * 6. Setup the line interrupt iff MSI-X is not supported. 423 * 7. Create the driver's taskq. 424 * 8. Start one task queue service thread. 425 * 9. Check if the firmware and SRAM are up-to-date. They will be 426 * auto-updated later (before FULL_INIT_DONE), if required. 427 * 10. Create a child device for each MAC (port) 428 * 11. Initialize T3 private state. 429 * 12. Trigger the LED 430 * 13. Setup offload iff supported. 431 * 14. Reset/restart the tick callout. 432 * 15. Attach sysctls 433 * 434 * NOTE: Any modification or deviation from this list MUST be reflected in 435 * the above comment. Failure to do so will result in problems on various 436 * error conditions including link flapping. 437 */ 438static int 439cxgb_controller_attach(device_t dev) 440{ 441 device_t child; 442 const struct adapter_info *ai; 443 struct adapter *sc; 444 int i, error = 0; 445 uint32_t vers; 446 int port_qsets = 1; 447 int msi_needed, reg; 448 char buf[80]; 449 450 sc = device_get_softc(dev); 451 sc->dev = dev; 452 sc->msi_count = 0; 453 ai = cxgb_get_adapter_info(dev); 454 455 /* find the PCIe link width and set max read request to 4KB*/ 456 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { 457 uint16_t lnk, pectl; 458 lnk = pci_read_config(dev, reg + 0x12, 2); 459 sc->link_width = (lnk >> 4) & 0x3f; 460 461 pectl = pci_read_config(dev, reg + 0x8, 2); 462 pectl = (pectl & ~0x7000) | (5 << 12); 463 pci_write_config(dev, reg + 0x8, pectl, 2); 464 } 465 466 if (sc->link_width != 0 && sc->link_width <= 4 && 467 (ai->nports0 + ai->nports1) <= 2) { 468 device_printf(sc->dev, 469 "PCIe x%d Link, expect reduced performance\n", 470 sc->link_width); 471 } 472 473 touch_bars(dev); 474 pci_enable_busmaster(dev); 475 /* 476 * Allocate the registers and make them available to the driver. 477 * The registers that we care about for NIC mode are in BAR 0 478 */ 479 sc->regs_rid = PCIR_BAR(0); 480 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 481 &sc->regs_rid, RF_ACTIVE)) == NULL) { 482 device_printf(dev, "Cannot allocate BAR region 0\n"); 483 return (ENXIO); 484 } 485 sc->udbs_rid = PCIR_BAR(2); 486 sc->udbs_res = NULL; 487 if (is_offload(sc) && 488 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 489 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 490 device_printf(dev, "Cannot allocate BAR region 1\n"); 491 error = ENXIO; 492 goto out; 493 } 494 495 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 496 device_get_unit(dev)); 497 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 498 499 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 500 device_get_unit(dev)); 501 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 502 device_get_unit(dev)); 503 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 504 device_get_unit(dev)); 505 506 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 507 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 508 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 509 510 sc->bt = rman_get_bustag(sc->regs_res); 511 sc->bh = rman_get_bushandle(sc->regs_res); 512 sc->mmio_len = rman_get_size(sc->regs_res); 513 514 for (i = 0; i < MAX_NPORTS; i++) 515 sc->port[i].adapter = sc; 516 517 if (t3_prep_adapter(sc, ai, 1) < 0) { 518 printf("prep adapter failed\n"); 519 error = ENODEV; 520 goto out; 521 } 522 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 523 * enough messages for the queue sets. If that fails, try falling 524 * back to MSI. If that fails, then try falling back to the legacy 525 * interrupt pin model. 526 */ 527 sc->msix_regs_rid = 0x20; 528 if ((msi_allowed >= 2) && 529 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 530 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 531 532 if (multiq) 533 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus); 534 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1; 535 536 if (pci_msix_count(dev) == 0 || 537 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 || 538 sc->msi_count != msi_needed) { 539 device_printf(dev, "alloc msix failed - " 540 "msi_count=%d, msi_needed=%d, err=%d; " 541 "will try MSI\n", sc->msi_count, 542 msi_needed, error); 543 sc->msi_count = 0; 544 port_qsets = 1; 545 pci_release_msi(dev); 546 bus_release_resource(dev, SYS_RES_MEMORY, 547 sc->msix_regs_rid, sc->msix_regs_res); 548 sc->msix_regs_res = NULL; 549 } else { 550 sc->flags |= USING_MSIX; 551 sc->cxgb_intr = cxgb_async_intr; 552 device_printf(dev, 553 "using MSI-X interrupts (%u vectors)\n", 554 sc->msi_count); 555 } 556 } 557 558 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 559 sc->msi_count = 1; 560 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) { 561 device_printf(dev, "alloc msi failed - " 562 "err=%d; will try INTx\n", error); 563 sc->msi_count = 0; 564 port_qsets = 1; 565 pci_release_msi(dev); 566 } else { 567 sc->flags |= USING_MSI; 568 sc->cxgb_intr = t3_intr_msi; 569 device_printf(dev, "using MSI interrupts\n"); 570 } 571 } 572 if (sc->msi_count == 0) { 573 device_printf(dev, "using line interrupts\n"); 574 sc->cxgb_intr = t3b_intr; 575 } 576 577 /* Create a private taskqueue thread for handling driver events */ 578 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 579 taskqueue_thread_enqueue, &sc->tq); 580 if (sc->tq == NULL) { 581 device_printf(dev, "failed to allocate controller task queue\n"); 582 goto out; 583 } 584 585 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 586 device_get_nameunit(dev)); 587 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc); 588 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 589 590 591 /* Create a periodic callout for checking adapter status */ 592 callout_init(&sc->cxgb_tick_ch, TRUE); 593 594 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 595 /* 596 * Warn user that a firmware update will be attempted in init. 597 */ 598 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 599 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 600 sc->flags &= ~FW_UPTODATE; 601 } else { 602 sc->flags |= FW_UPTODATE; 603 } 604 605 if (t3_check_tpsram_version(sc) < 0) { 606 /* 607 * Warn user that a firmware update will be attempted in init. 608 */ 609 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 610 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 611 sc->flags &= ~TPS_UPTODATE; 612 } else { 613 sc->flags |= TPS_UPTODATE; 614 } 615 616 /* 617 * Create a child device for each MAC. The ethernet attachment 618 * will be done in these children. 619 */ 620 for (i = 0; i < (sc)->params.nports; i++) { 621 struct port_info *pi; 622 623 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 624 device_printf(dev, "failed to add child port\n"); 625 error = EINVAL; 626 goto out; 627 } 628 pi = &sc->port[i]; 629 pi->adapter = sc; 630 pi->nqsets = port_qsets; 631 pi->first_qset = i*port_qsets; 632 pi->port_id = i; 633 pi->tx_chan = i >= ai->nports0; 634 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 635 sc->rxpkt_map[pi->txpkt_intf] = i; 636 sc->port[i].tx_chan = i >= ai->nports0; 637 sc->portdev[i] = child; 638 device_set_softc(child, pi); 639 } 640 if ((error = bus_generic_attach(dev)) != 0) 641 goto out; 642 643 /* initialize sge private state */ 644 t3_sge_init_adapter(sc); 645 646 t3_led_ready(sc); 647 648 cxgb_offload_init(); 649 if (is_offload(sc)) { 650 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 651 cxgb_adapter_ofld(sc); 652 } 653 error = t3_get_fw_version(sc, &vers); 654 if (error) 655 goto out; 656 657 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 658 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 659 G_FW_VERSION_MICRO(vers)); 660 661 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s", 662 ai->desc, is_offload(sc) ? "R" : "", 663 sc->params.vpd.ec, sc->params.vpd.sn); 664 device_set_desc_copy(dev, buf); 665 666 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x", 667 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1], 668 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]); 669 670 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 671 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 672 t3_add_attach_sysctls(sc); 673out: 674 if (error) 675 cxgb_free(sc); 676 677 return (error); 678} 679 680/* 681 * The cxgb_controller_detach routine is called with the device is 682 * unloaded from the system. 683 */ 684 685static int 686cxgb_controller_detach(device_t dev) 687{ 688 struct adapter *sc; 689 690 sc = device_get_softc(dev); 691 692 cxgb_free(sc); 693 694 return (0); 695} 696 697/* 698 * The cxgb_free() is called by the cxgb_controller_detach() routine 699 * to tear down the structures that were built up in 700 * cxgb_controller_attach(), and should be the final piece of work 701 * done when fully unloading the driver. 702 * 703 * 704 * 1. Shutting down the threads started by the cxgb_controller_attach() 705 * routine. 706 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()). 707 * 3. Detaching all of the port devices created during the 708 * cxgb_controller_attach() routine. 709 * 4. Removing the device children created via cxgb_controller_attach(). 710 * 5. Releasing PCI resources associated with the device. 711 * 6. Turning off the offload support, iff it was turned on. 712 * 7. Destroying the mutexes created in cxgb_controller_attach(). 713 * 714 */ 715static void 716cxgb_free(struct adapter *sc) 717{ 718 int i; 719 720 ADAPTER_LOCK(sc); 721 sc->flags |= CXGB_SHUTDOWN; 722 ADAPTER_UNLOCK(sc); 723 724 /* 725 * Make sure all child devices are gone. 726 */ 727 bus_generic_detach(sc->dev); 728 for (i = 0; i < (sc)->params.nports; i++) { 729 if (sc->portdev[i] && 730 device_delete_child(sc->dev, sc->portdev[i]) != 0) 731 device_printf(sc->dev, "failed to delete child port\n"); 732 } 733 734 /* 735 * At this point, it is as if cxgb_port_detach has run on all ports, and 736 * cxgb_down has run on the adapter. All interrupts have been silenced, 737 * all open devices have been closed. 738 */ 739 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)", 740 __func__, sc->open_device_map)); 741 for (i = 0; i < sc->params.nports; i++) { 742 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!", 743 __func__, i)); 744 } 745 746 /* 747 * Finish off the adapter's callouts. 748 */ 749 callout_drain(&sc->cxgb_tick_ch); 750 callout_drain(&sc->sge_timer_ch); 751 752 /* 753 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The 754 * sysctls are cleaned up by the kernel linker. 755 */ 756 if (sc->flags & FULL_INIT_DONE) { 757 t3_free_sge_resources(sc); 758 sc->flags &= ~FULL_INIT_DONE; 759 } 760 761 /* 762 * Release all interrupt resources. 763 */ 764 cxgb_teardown_interrupts(sc); 765 if (sc->flags & (USING_MSI | USING_MSIX)) { 766 device_printf(sc->dev, "releasing msi message(s)\n"); 767 pci_release_msi(sc->dev); 768 } else { 769 device_printf(sc->dev, "no msi message to release\n"); 770 } 771 772 if (sc->msix_regs_res != NULL) { 773 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 774 sc->msix_regs_res); 775 } 776 777 /* 778 * Free the adapter's taskqueue. 779 */ 780 if (sc->tq != NULL) { 781 taskqueue_free(sc->tq); 782 sc->tq = NULL; 783 } 784 785 if (is_offload(sc)) { 786 clrbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); 787 cxgb_adapter_unofld(sc); 788 } 789 790#ifdef notyet 791 if (sc->flags & CXGB_OFLD_INIT) 792 cxgb_offload_deactivate(sc); 793#endif 794 free(sc->filters, M_DEVBUF); 795 t3_sge_free(sc); 796 797 cxgb_offload_exit(); 798 799 if (sc->udbs_res != NULL) 800 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 801 sc->udbs_res); 802 803 if (sc->regs_res != NULL) 804 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 805 sc->regs_res); 806 807 MTX_DESTROY(&sc->mdio_lock); 808 MTX_DESTROY(&sc->sge.reg_lock); 809 MTX_DESTROY(&sc->elmer_lock); 810 ADAPTER_LOCK_DEINIT(sc); 811} 812 813/** 814 * setup_sge_qsets - configure SGE Tx/Rx/response queues 815 * @sc: the controller softc 816 * 817 * Determines how many sets of SGE queues to use and initializes them. 818 * We support multiple queue sets per port if we have MSI-X, otherwise 819 * just one queue set per port. 820 */ 821static int 822setup_sge_qsets(adapter_t *sc) 823{ 824 int i, j, err, irq_idx = 0, qset_idx = 0; 825 u_int ntxq = SGE_TXQ_PER_SET; 826 827 if ((err = t3_sge_alloc(sc)) != 0) { 828 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 829 return (err); 830 } 831 832 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 833 irq_idx = -1; 834 835 for (i = 0; i < (sc)->params.nports; i++) { 836 struct port_info *pi = &sc->port[i]; 837 838 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 839 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 840 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 841 &sc->params.sge.qset[qset_idx], ntxq, pi); 842 if (err) { 843 t3_free_sge_resources(sc); 844 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n", 845 err); 846 return (err); 847 } 848 } 849 } 850 851 return (0); 852} 853 854static void 855cxgb_teardown_interrupts(adapter_t *sc) 856{ 857 int i; 858 859 for (i = 0; i < SGE_QSETS; i++) { 860 if (sc->msix_intr_tag[i] == NULL) { 861 862 /* Should have been setup fully or not at all */ 863 KASSERT(sc->msix_irq_res[i] == NULL && 864 sc->msix_irq_rid[i] == 0, 865 ("%s: half-done interrupt (%d).", __func__, i)); 866 867 continue; 868 } 869 870 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 871 sc->msix_intr_tag[i]); 872 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i], 873 sc->msix_irq_res[i]); 874 875 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL; 876 sc->msix_irq_rid[i] = 0; 877 } 878 879 if (sc->intr_tag) { 880 KASSERT(sc->irq_res != NULL, 881 ("%s: half-done interrupt.", __func__)); 882 883 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 884 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 885 sc->irq_res); 886 887 sc->irq_res = sc->intr_tag = NULL; 888 sc->irq_rid = 0; 889 } 890} 891 892static int 893cxgb_setup_interrupts(adapter_t *sc) 894{ 895 struct resource *res; 896 void *tag; 897 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX); 898 899 sc->irq_rid = intr_flag ? 1 : 0; 900 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, 901 RF_SHAREABLE | RF_ACTIVE); 902 if (sc->irq_res == NULL) { 903 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n", 904 intr_flag, sc->irq_rid); 905 err = EINVAL; 906 sc->irq_rid = 0; 907 } else { 908 err = bus_setup_intr(sc->dev, sc->irq_res, 909 INTR_MPSAFE | INTR_TYPE_NET, NULL, 910 sc->cxgb_intr, sc, &sc->intr_tag); 911 912 if (err) { 913 device_printf(sc->dev, 914 "Cannot set up interrupt (%x, %u, %d)\n", 915 intr_flag, sc->irq_rid, err); 916 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 917 sc->irq_res); 918 sc->irq_res = sc->intr_tag = NULL; 919 sc->irq_rid = 0; 920 } 921 } 922 923 /* That's all for INTx or MSI */ 924 if (!(intr_flag & USING_MSIX) || err) 925 return (err); 926 927 for (i = 0; i < sc->msi_count - 1; i++) { 928 rid = i + 2; 929 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 930 RF_SHAREABLE | RF_ACTIVE); 931 if (res == NULL) { 932 device_printf(sc->dev, "Cannot allocate interrupt " 933 "for message %d\n", rid); 934 err = EINVAL; 935 break; 936 } 937 938 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET, 939 NULL, t3_intr_msix, &sc->sge.qs[i], &tag); 940 if (err) { 941 device_printf(sc->dev, "Cannot set up interrupt " 942 "for message %d (%d)\n", rid, err); 943 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); 944 break; 945 } 946 947 sc->msix_irq_rid[i] = rid; 948 sc->msix_irq_res[i] = res; 949 sc->msix_intr_tag[i] = tag; 950 } 951 952 if (err) 953 cxgb_teardown_interrupts(sc); 954 955 return (err); 956} 957 958 959static int 960cxgb_port_probe(device_t dev) 961{ 962 struct port_info *p; 963 char buf[80]; 964 const char *desc; 965 966 p = device_get_softc(dev); 967 desc = p->phy.desc; 968 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 969 device_set_desc_copy(dev, buf); 970 return (0); 971} 972 973 974static int 975cxgb_makedev(struct port_info *pi) 976{ 977 978 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 979 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp)); 980 981 if (pi->port_cdev == NULL) 982 return (ENOMEM); 983 984 pi->port_cdev->si_drv1 = (void *)pi; 985 986 return (0); 987} 988 989#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 990 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 991 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE) 992#define CXGB_CAP_ENABLE (CXGB_CAP & ~IFCAP_TSO6) 993 994static int 995cxgb_port_attach(device_t dev) 996{ 997 struct port_info *p; 998 struct ifnet *ifp; 999 int err; 1000 struct adapter *sc; 1001 1002 p = device_get_softc(dev); 1003 sc = p->adapter; 1004 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 1005 device_get_unit(device_get_parent(dev)), p->port_id); 1006 PORT_LOCK_INIT(p, p->lockbuf); 1007 1008 /* Allocate an ifnet object and set it up */ 1009 ifp = p->ifp = if_alloc(IFT_ETHER); 1010 if (ifp == NULL) { 1011 device_printf(dev, "Cannot allocate ifnet\n"); 1012 return (ENOMEM); 1013 } 1014 1015 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1016 ifp->if_init = cxgb_init; 1017 ifp->if_softc = p; 1018 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1019 ifp->if_ioctl = cxgb_ioctl; 1020 ifp->if_start = cxgb_start; 1021 1022 ifp->if_snd.ifq_drv_maxlen = max(cxgb_snd_queue_len, ifqmaxlen); 1023 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 1024 IFQ_SET_READY(&ifp->if_snd); 1025 1026 ifp->if_capabilities = CXGB_CAP; 1027 ifp->if_capenable = CXGB_CAP_ENABLE; 1028 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; 1029 1030 /* 1031 * Disable TSO on 4-port - it isn't supported by the firmware. 1032 */ 1033 if (sc->params.nports > 2) { 1034 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1035 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1036 ifp->if_hwassist &= ~CSUM_TSO; 1037 } 1038 1039 ether_ifattach(ifp, p->hw_addr); 1040 ifp->if_transmit = cxgb_transmit; 1041 ifp->if_qflush = cxgb_qflush; 1042 1043#ifdef DEFAULT_JUMBO 1044 if (sc->params.nports <= 2) 1045 ifp->if_mtu = ETHERMTU_JUMBO; 1046#endif 1047 if ((err = cxgb_makedev(p)) != 0) { 1048 printf("makedev failed %d\n", err); 1049 return (err); 1050 } 1051 1052 /* Create a list of media supported by this port */ 1053 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 1054 cxgb_media_status); 1055 cxgb_build_medialist(p); 1056 1057 t3_sge_init_port(p); 1058 1059 return (err); 1060} 1061 1062/* 1063 * cxgb_port_detach() is called via the device_detach methods when 1064 * cxgb_free() calls the bus_generic_detach. It is responsible for 1065 * removing the device from the view of the kernel, i.e. from all 1066 * interfaces lists etc. This routine is only called when the driver is 1067 * being unloaded, not when the link goes down. 1068 */ 1069static int 1070cxgb_port_detach(device_t dev) 1071{ 1072 struct port_info *p; 1073 struct adapter *sc; 1074 int i; 1075 1076 p = device_get_softc(dev); 1077 sc = p->adapter; 1078 1079 /* Tell cxgb_ioctl and if_init that the port is going away */ 1080 ADAPTER_LOCK(sc); 1081 SET_DOOMED(p); 1082 wakeup(&sc->flags); 1083 while (IS_BUSY(sc)) 1084 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0); 1085 SET_BUSY(sc); 1086 ADAPTER_UNLOCK(sc); 1087 1088 if (p->port_cdev != NULL) 1089 destroy_dev(p->port_cdev); 1090 1091 cxgb_uninit_synchronized(p); 1092 ether_ifdetach(p->ifp); 1093 1094 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1095 struct sge_qset *qs = &sc->sge.qs[i]; 1096 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1097 1098 callout_drain(&txq->txq_watchdog); 1099 callout_drain(&txq->txq_timer); 1100 } 1101 1102 PORT_LOCK_DEINIT(p); 1103 if_free(p->ifp); 1104 p->ifp = NULL; 1105 1106 ADAPTER_LOCK(sc); 1107 CLR_BUSY(sc); 1108 wakeup_one(&sc->flags); 1109 ADAPTER_UNLOCK(sc); 1110 return (0); 1111} 1112 1113void 1114t3_fatal_err(struct adapter *sc) 1115{ 1116 u_int fw_status[4]; 1117 1118 if (sc->flags & FULL_INIT_DONE) { 1119 t3_sge_stop(sc); 1120 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1121 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1122 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1123 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1124 t3_intr_disable(sc); 1125 } 1126 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1127 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1128 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1129 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1130} 1131 1132int 1133t3_os_find_pci_capability(adapter_t *sc, int cap) 1134{ 1135 device_t dev; 1136 struct pci_devinfo *dinfo; 1137 pcicfgregs *cfg; 1138 uint32_t status; 1139 uint8_t ptr; 1140 1141 dev = sc->dev; 1142 dinfo = device_get_ivars(dev); 1143 cfg = &dinfo->cfg; 1144 1145 status = pci_read_config(dev, PCIR_STATUS, 2); 1146 if (!(status & PCIM_STATUS_CAPPRESENT)) 1147 return (0); 1148 1149 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1150 case 0: 1151 case 1: 1152 ptr = PCIR_CAP_PTR; 1153 break; 1154 case 2: 1155 ptr = PCIR_CAP_PTR_2; 1156 break; 1157 default: 1158 return (0); 1159 break; 1160 } 1161 ptr = pci_read_config(dev, ptr, 1); 1162 1163 while (ptr != 0) { 1164 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1165 return (ptr); 1166 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1167 } 1168 1169 return (0); 1170} 1171 1172int 1173t3_os_pci_save_state(struct adapter *sc) 1174{ 1175 device_t dev; 1176 struct pci_devinfo *dinfo; 1177 1178 dev = sc->dev; 1179 dinfo = device_get_ivars(dev); 1180 1181 pci_cfg_save(dev, dinfo, 0); 1182 return (0); 1183} 1184 1185int 1186t3_os_pci_restore_state(struct adapter *sc) 1187{ 1188 device_t dev; 1189 struct pci_devinfo *dinfo; 1190 1191 dev = sc->dev; 1192 dinfo = device_get_ivars(dev); 1193 1194 pci_cfg_restore(dev, dinfo); 1195 return (0); 1196} 1197 1198/** 1199 * t3_os_link_changed - handle link status changes 1200 * @sc: the adapter associated with the link change 1201 * @port_id: the port index whose link status has changed 1202 * @link_status: the new status of the link 1203 * @speed: the new speed setting 1204 * @duplex: the new duplex setting 1205 * @fc: the new flow-control setting 1206 * 1207 * This is the OS-dependent handler for link status changes. The OS 1208 * neutral handler takes care of most of the processing for these events, 1209 * then calls this handler for any OS-specific processing. 1210 */ 1211void 1212t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1213 int duplex, int fc, int mac_was_reset) 1214{ 1215 struct port_info *pi = &adapter->port[port_id]; 1216 struct ifnet *ifp = pi->ifp; 1217 1218 /* no race with detach, so ifp should always be good */ 1219 KASSERT(ifp, ("%s: if detached.", __func__)); 1220 1221 /* Reapply mac settings if they were lost due to a reset */ 1222 if (mac_was_reset) { 1223 PORT_LOCK(pi); 1224 cxgb_update_mac_settings(pi); 1225 PORT_UNLOCK(pi); 1226 } 1227 1228 if (link_status) { 1229 ifp->if_baudrate = IF_Mbps(speed); 1230 if_link_state_change(ifp, LINK_STATE_UP); 1231 } else 1232 if_link_state_change(ifp, LINK_STATE_DOWN); 1233} 1234 1235/** 1236 * t3_os_phymod_changed - handle PHY module changes 1237 * @phy: the PHY reporting the module change 1238 * @mod_type: new module type 1239 * 1240 * This is the OS-dependent handler for PHY module changes. It is 1241 * invoked when a PHY module is removed or inserted for any OS-specific 1242 * processing. 1243 */ 1244void t3_os_phymod_changed(struct adapter *adap, int port_id) 1245{ 1246 static const char *mod_str[] = { 1247 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown" 1248 }; 1249 struct port_info *pi = &adap->port[port_id]; 1250 int mod = pi->phy.modtype; 1251 1252 if (mod != pi->media.ifm_cur->ifm_data) 1253 cxgb_build_medialist(pi); 1254 1255 if (mod == phy_modtype_none) 1256 if_printf(pi->ifp, "PHY module unplugged\n"); 1257 else { 1258 KASSERT(mod < ARRAY_SIZE(mod_str), 1259 ("invalid PHY module type %d", mod)); 1260 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]); 1261 } 1262} 1263 1264/* 1265 * Interrupt-context handler for external (PHY) interrupts. 1266 */ 1267void 1268t3_os_ext_intr_handler(adapter_t *sc) 1269{ 1270 if (cxgb_debug) 1271 printf("t3_os_ext_intr_handler\n"); 1272 /* 1273 * Schedule a task to handle external interrupts as they may be slow 1274 * and we use a mutex to protect MDIO registers. We disable PHY 1275 * interrupts in the meantime and let the task reenable them when 1276 * it's done. 1277 */ 1278 if (sc->slow_intr_mask) { 1279 ADAPTER_LOCK(sc); 1280 sc->slow_intr_mask &= ~F_T3DBG; 1281 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 1282 taskqueue_enqueue(sc->tq, &sc->ext_intr_task); 1283 ADAPTER_UNLOCK(sc); 1284 } 1285} 1286 1287void 1288t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1289{ 1290 1291 /* 1292 * The ifnet might not be allocated before this gets called, 1293 * as this is called early on in attach by t3_prep_adapter 1294 * save the address off in the port structure 1295 */ 1296 if (cxgb_debug) 1297 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1298 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1299} 1300 1301/* 1302 * Programs the XGMAC based on the settings in the ifnet. These settings 1303 * include MTU, MAC address, mcast addresses, etc. 1304 */ 1305static void 1306cxgb_update_mac_settings(struct port_info *p) 1307{ 1308 struct ifnet *ifp = p->ifp; 1309 struct t3_rx_mode rm; 1310 struct cmac *mac = &p->mac; 1311 int mtu, hwtagging; 1312 1313 PORT_LOCK_ASSERT_OWNED(p); 1314 1315 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1316 1317 mtu = ifp->if_mtu; 1318 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1319 mtu += ETHER_VLAN_ENCAP_LEN; 1320 1321 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1322 1323 t3_mac_set_mtu(mac, mtu); 1324 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1325 t3_mac_set_address(mac, 0, p->hw_addr); 1326 t3_init_rx_mode(&rm, p); 1327 t3_mac_set_rx_mode(mac, &rm); 1328} 1329 1330 1331static int 1332await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1333 unsigned long n) 1334{ 1335 int attempts = 5; 1336 1337 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1338 if (!--attempts) 1339 return (ETIMEDOUT); 1340 t3_os_sleep(10); 1341 } 1342 return 0; 1343} 1344 1345static int 1346init_tp_parity(struct adapter *adap) 1347{ 1348 int i; 1349 struct mbuf *m; 1350 struct cpl_set_tcb_field *greq; 1351 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1352 1353 t3_tp_set_offload_mode(adap, 1); 1354 1355 for (i = 0; i < 16; i++) { 1356 struct cpl_smt_write_req *req; 1357 1358 m = m_gethdr(M_WAITOK, MT_DATA); 1359 req = mtod(m, struct cpl_smt_write_req *); 1360 m->m_len = m->m_pkthdr.len = sizeof(*req); 1361 memset(req, 0, sizeof(*req)); 1362 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1363 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1364 req->iff = i; 1365 t3_mgmt_tx(adap, m); 1366 } 1367 1368 for (i = 0; i < 2048; i++) { 1369 struct cpl_l2t_write_req *req; 1370 1371 m = m_gethdr(M_WAITOK, MT_DATA); 1372 req = mtod(m, struct cpl_l2t_write_req *); 1373 m->m_len = m->m_pkthdr.len = sizeof(*req); 1374 memset(req, 0, sizeof(*req)); 1375 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1376 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1377 req->params = htonl(V_L2T_W_IDX(i)); 1378 t3_mgmt_tx(adap, m); 1379 } 1380 1381 for (i = 0; i < 2048; i++) { 1382 struct cpl_rte_write_req *req; 1383 1384 m = m_gethdr(M_WAITOK, MT_DATA); 1385 req = mtod(m, struct cpl_rte_write_req *); 1386 m->m_len = m->m_pkthdr.len = sizeof(*req); 1387 memset(req, 0, sizeof(*req)); 1388 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1389 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1390 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1391 t3_mgmt_tx(adap, m); 1392 } 1393 1394 m = m_gethdr(M_WAITOK, MT_DATA); 1395 greq = mtod(m, struct cpl_set_tcb_field *); 1396 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1397 memset(greq, 0, sizeof(*greq)); 1398 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1399 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1400 greq->mask = htobe64(1); 1401 t3_mgmt_tx(adap, m); 1402 1403 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1404 t3_tp_set_offload_mode(adap, 0); 1405 return (i); 1406} 1407 1408/** 1409 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1410 * @adap: the adapter 1411 * 1412 * Sets up RSS to distribute packets to multiple receive queues. We 1413 * configure the RSS CPU lookup table to distribute to the number of HW 1414 * receive queues, and the response queue lookup table to narrow that 1415 * down to the response queues actually configured for each port. 1416 * We always configure the RSS mapping for two ports since the mapping 1417 * table has plenty of entries. 1418 */ 1419static void 1420setup_rss(adapter_t *adap) 1421{ 1422 int i; 1423 u_int nq[2]; 1424 uint8_t cpus[SGE_QSETS + 1]; 1425 uint16_t rspq_map[RSS_TABLE_SIZE]; 1426 1427 for (i = 0; i < SGE_QSETS; ++i) 1428 cpus[i] = i; 1429 cpus[SGE_QSETS] = 0xff; 1430 1431 nq[0] = nq[1] = 0; 1432 for_each_port(adap, i) { 1433 const struct port_info *pi = adap2pinfo(adap, i); 1434 1435 nq[pi->tx_chan] += pi->nqsets; 1436 } 1437 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1438 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1439 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1440 } 1441 1442 /* Calculate the reverse RSS map table */ 1443 for (i = 0; i < SGE_QSETS; ++i) 1444 adap->rrss_map[i] = 0xff; 1445 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1446 if (adap->rrss_map[rspq_map[i]] == 0xff) 1447 adap->rrss_map[rspq_map[i]] = i; 1448 1449 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1450 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1451 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1452 cpus, rspq_map); 1453 1454} 1455 1456/* 1457 * Sends an mbuf to an offload queue driver 1458 * after dealing with any active network taps. 1459 */ 1460static inline int 1461offload_tx(struct t3cdev *tdev, struct mbuf *m) 1462{ 1463 int ret; 1464 1465 ret = t3_offload_tx(tdev, m); 1466 return (ret); 1467} 1468 1469static int 1470write_smt_entry(struct adapter *adapter, int idx) 1471{ 1472 struct port_info *pi = &adapter->port[idx]; 1473 struct cpl_smt_write_req *req; 1474 struct mbuf *m; 1475 1476 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1477 return (ENOMEM); 1478 1479 req = mtod(m, struct cpl_smt_write_req *); 1480 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); 1481 1482 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1483 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 1484 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 1485 req->iff = idx; 1486 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 1487 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 1488 1489 m_set_priority(m, 1); 1490 1491 offload_tx(&adapter->tdev, m); 1492 1493 return (0); 1494} 1495 1496static int 1497init_smt(struct adapter *adapter) 1498{ 1499 int i; 1500 1501 for_each_port(adapter, i) 1502 write_smt_entry(adapter, i); 1503 return 0; 1504} 1505 1506static void 1507init_port_mtus(adapter_t *adapter) 1508{ 1509 unsigned int mtus = ETHERMTU | (ETHERMTU << 16); 1510 1511 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); 1512} 1513 1514static void 1515send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1516 int hi, int port) 1517{ 1518 struct mbuf *m; 1519 struct mngt_pktsched_wr *req; 1520 1521 m = m_gethdr(M_DONTWAIT, MT_DATA); 1522 if (m) { 1523 req = mtod(m, struct mngt_pktsched_wr *); 1524 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1525 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1526 req->sched = sched; 1527 req->idx = qidx; 1528 req->min = lo; 1529 req->max = hi; 1530 req->binding = port; 1531 m->m_len = m->m_pkthdr.len = sizeof(*req); 1532 t3_mgmt_tx(adap, m); 1533 } 1534} 1535 1536static void 1537bind_qsets(adapter_t *sc) 1538{ 1539 int i, j; 1540 1541 for (i = 0; i < (sc)->params.nports; ++i) { 1542 const struct port_info *pi = adap2pinfo(sc, i); 1543 1544 for (j = 0; j < pi->nqsets; ++j) { 1545 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1546 -1, pi->tx_chan); 1547 1548 } 1549 } 1550} 1551 1552static void 1553update_tpeeprom(struct adapter *adap) 1554{ 1555 const struct firmware *tpeeprom; 1556 1557 uint32_t version; 1558 unsigned int major, minor; 1559 int ret, len; 1560 char rev, name[32]; 1561 1562 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1563 1564 major = G_TP_VERSION_MAJOR(version); 1565 minor = G_TP_VERSION_MINOR(version); 1566 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1567 return; 1568 1569 rev = t3rev2char(adap); 1570 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1571 1572 tpeeprom = firmware_get(name); 1573 if (tpeeprom == NULL) { 1574 device_printf(adap->dev, 1575 "could not load TP EEPROM: unable to load %s\n", 1576 name); 1577 return; 1578 } 1579 1580 len = tpeeprom->datasize - 4; 1581 1582 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1583 if (ret) 1584 goto release_tpeeprom; 1585 1586 if (len != TP_SRAM_LEN) { 1587 device_printf(adap->dev, 1588 "%s length is wrong len=%d expected=%d\n", name, 1589 len, TP_SRAM_LEN); 1590 return; 1591 } 1592 1593 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1594 TP_SRAM_OFFSET); 1595 1596 if (!ret) { 1597 device_printf(adap->dev, 1598 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1599 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1600 } else 1601 device_printf(adap->dev, 1602 "Protocol SRAM image update in EEPROM failed\n"); 1603 1604release_tpeeprom: 1605 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1606 1607 return; 1608} 1609 1610static int 1611update_tpsram(struct adapter *adap) 1612{ 1613 const struct firmware *tpsram; 1614 int ret; 1615 char rev, name[32]; 1616 1617 rev = t3rev2char(adap); 1618 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1619 1620 update_tpeeprom(adap); 1621 1622 tpsram = firmware_get(name); 1623 if (tpsram == NULL){ 1624 device_printf(adap->dev, "could not load TP SRAM\n"); 1625 return (EINVAL); 1626 } else 1627 device_printf(adap->dev, "updating TP SRAM\n"); 1628 1629 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1630 if (ret) 1631 goto release_tpsram; 1632 1633 ret = t3_set_proto_sram(adap, tpsram->data); 1634 if (ret) 1635 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1636 1637release_tpsram: 1638 firmware_put(tpsram, FIRMWARE_UNLOAD); 1639 1640 return ret; 1641} 1642 1643/** 1644 * cxgb_up - enable the adapter 1645 * @adap: adapter being enabled 1646 * 1647 * Called when the first port is enabled, this function performs the 1648 * actions necessary to make an adapter operational, such as completing 1649 * the initialization of HW modules, and enabling interrupts. 1650 */ 1651static int 1652cxgb_up(struct adapter *sc) 1653{ 1654 int err = 0; 1655 1656 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)", 1657 __func__, sc->open_device_map)); 1658 1659 if ((sc->flags & FULL_INIT_DONE) == 0) { 1660 1661 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1662 1663 if ((sc->flags & FW_UPTODATE) == 0) 1664 if ((err = upgrade_fw(sc))) 1665 goto out; 1666 1667 if ((sc->flags & TPS_UPTODATE) == 0) 1668 if ((err = update_tpsram(sc))) 1669 goto out; 1670 1671 if (is_offload(sc)) { 1672 sc->params.mc5.nservers = 0; 1673 sc->params.mc5.nroutes = 0; 1674 sc->params.mc5.nfilters = t3_mc5_size(&sc->mc5) - 1675 MC5_MIN_TIDS; 1676 } 1677 1678 err = t3_init_hw(sc, 0); 1679 if (err) 1680 goto out; 1681 1682 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1683 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1684 1685 err = setup_sge_qsets(sc); 1686 if (err) 1687 goto out; 1688 1689 alloc_filters(sc); 1690 setup_rss(sc); 1691 1692 t3_intr_clear(sc); 1693 err = cxgb_setup_interrupts(sc); 1694 if (err) 1695 goto out; 1696 1697 t3_add_configured_sysctls(sc); 1698 sc->flags |= FULL_INIT_DONE; 1699 } 1700 1701 t3_intr_clear(sc); 1702 t3_sge_start(sc); 1703 t3_intr_enable(sc); 1704 1705 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1706 is_offload(sc) && init_tp_parity(sc) == 0) 1707 sc->flags |= TP_PARITY_INIT; 1708 1709 if (sc->flags & TP_PARITY_INIT) { 1710 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR); 1711 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1712 } 1713 1714 if (!(sc->flags & QUEUES_BOUND)) { 1715 bind_qsets(sc); 1716 setup_hw_filters(sc); 1717 sc->flags |= QUEUES_BOUND; 1718 } 1719 1720 t3_sge_reset_adapter(sc); 1721out: 1722 return (err); 1723} 1724 1725/* 1726 * Called when the last open device is closed. Does NOT undo all of cxgb_up's 1727 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released 1728 * during controller_detach, not here. 1729 */ 1730static void 1731cxgb_down(struct adapter *sc) 1732{ 1733 t3_sge_stop(sc); 1734 t3_intr_disable(sc); 1735} 1736 1737static int 1738offload_open(struct port_info *pi) 1739{ 1740 struct adapter *sc = pi->adapter; 1741 struct t3cdev *tdev = &sc->tdev; 1742 1743 setbit(&sc->open_device_map, OFFLOAD_DEVMAP_BIT); 1744 1745 t3_tp_set_offload_mode(sc, 1); 1746 tdev->lldev = pi->ifp; 1747 init_port_mtus(sc); 1748 t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd, 1749 sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff); 1750 init_smt(sc); 1751 cxgb_add_clients(tdev); 1752 1753 return (0); 1754} 1755 1756static int 1757offload_close(struct t3cdev *tdev) 1758{ 1759 struct adapter *adapter = tdev2adap(tdev); 1760 1761 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) 1762 return (0); 1763 1764 /* Call back all registered clients */ 1765 cxgb_remove_clients(tdev); 1766 1767 tdev->lldev = NULL; 1768 cxgb_set_dummy_ops(tdev); 1769 t3_tp_set_offload_mode(adapter, 0); 1770 1771 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); 1772 1773 return (0); 1774} 1775 1776/* 1777 * if_init for cxgb ports. 1778 */ 1779static void 1780cxgb_init(void *arg) 1781{ 1782 struct port_info *p = arg; 1783 struct adapter *sc = p->adapter; 1784 1785 ADAPTER_LOCK(sc); 1786 cxgb_init_locked(p); /* releases adapter lock */ 1787 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1788} 1789 1790static int 1791cxgb_init_locked(struct port_info *p) 1792{ 1793 struct adapter *sc = p->adapter; 1794 struct ifnet *ifp = p->ifp; 1795 struct cmac *mac = &p->mac; 1796 int i, rc = 0, may_sleep = 0; 1797 1798 ADAPTER_LOCK_ASSERT_OWNED(sc); 1799 1800 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1801 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) { 1802 rc = EINTR; 1803 goto done; 1804 } 1805 } 1806 if (IS_DOOMED(p)) { 1807 rc = ENXIO; 1808 goto done; 1809 } 1810 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1811 1812 /* 1813 * The code that runs during one-time adapter initialization can sleep 1814 * so it's important not to hold any locks across it. 1815 */ 1816 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1; 1817 1818 if (may_sleep) { 1819 SET_BUSY(sc); 1820 ADAPTER_UNLOCK(sc); 1821 } 1822 1823 if (sc->open_device_map == 0) { 1824 if ((rc = cxgb_up(sc)) != 0) 1825 goto done; 1826 1827 if (is_offload(sc) && !ofld_disable && offload_open(p)) 1828 log(LOG_WARNING, 1829 "Could not initialize offload capabilities\n"); 1830 } 1831 1832 PORT_LOCK(p); 1833 if (isset(&sc->open_device_map, p->port_id) && 1834 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1835 PORT_UNLOCK(p); 1836 goto done; 1837 } 1838 t3_port_intr_enable(sc, p->port_id); 1839 if (!mac->multiport) 1840 t3_mac_init(mac); 1841 cxgb_update_mac_settings(p); 1842 t3_link_start(&p->phy, mac, &p->link_config); 1843 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1844 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1845 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1846 PORT_UNLOCK(p); 1847 1848 t3_link_changed(sc, p->port_id); 1849 1850 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1851 struct sge_qset *qs = &sc->sge.qs[i]; 1852 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1853 1854 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs, 1855 txq->txq_watchdog.c_cpu); 1856 } 1857 1858 /* all ok */ 1859 setbit(&sc->open_device_map, p->port_id); 1860 1861done: 1862 if (may_sleep) { 1863 ADAPTER_LOCK(sc); 1864 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1865 CLR_BUSY(sc); 1866 wakeup_one(&sc->flags); 1867 } 1868 ADAPTER_UNLOCK(sc); 1869 return (rc); 1870} 1871 1872static int 1873cxgb_uninit_locked(struct port_info *p) 1874{ 1875 struct adapter *sc = p->adapter; 1876 int rc; 1877 1878 ADAPTER_LOCK_ASSERT_OWNED(sc); 1879 1880 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1881 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) { 1882 rc = EINTR; 1883 goto done; 1884 } 1885 } 1886 if (IS_DOOMED(p)) { 1887 rc = ENXIO; 1888 goto done; 1889 } 1890 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1891 SET_BUSY(sc); 1892 ADAPTER_UNLOCK(sc); 1893 1894 rc = cxgb_uninit_synchronized(p); 1895 1896 ADAPTER_LOCK(sc); 1897 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1898 CLR_BUSY(sc); 1899 wakeup_one(&sc->flags); 1900done: 1901 ADAPTER_UNLOCK(sc); 1902 return (rc); 1903} 1904 1905/* 1906 * Called on "ifconfig down", and from port_detach 1907 */ 1908static int 1909cxgb_uninit_synchronized(struct port_info *pi) 1910{ 1911 struct adapter *sc = pi->adapter; 1912 struct ifnet *ifp = pi->ifp; 1913 1914 /* 1915 * taskqueue_drain may cause a deadlock if the adapter lock is held. 1916 */ 1917 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1918 1919 /* 1920 * Clear this port's bit from the open device map, and then drain all 1921 * the tasks that can access/manipulate this port's port_info or ifp. 1922 * We disable this port's interrupts here and so the the slow/ext 1923 * interrupt tasks won't be enqueued. The tick task will continue to 1924 * be enqueued every second but the runs after this drain will not see 1925 * this port in the open device map. 1926 * 1927 * A well behaved task must take open_device_map into account and ignore 1928 * ports that are not open. 1929 */ 1930 clrbit(&sc->open_device_map, pi->port_id); 1931 t3_port_intr_disable(sc, pi->port_id); 1932 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1933 taskqueue_drain(sc->tq, &sc->ext_intr_task); 1934 taskqueue_drain(sc->tq, &sc->tick_task); 1935 1936 PORT_LOCK(pi); 1937 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1938 1939 /* disable pause frames */ 1940 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0); 1941 1942 /* Reset RX FIFO HWM */ 1943 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset, 1944 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1945 1946 DELAY(100 * 1000); 1947 1948 /* Wait for TXFIFO empty */ 1949 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset, 1950 F_TXFIFO_EMPTY, 1, 20, 5); 1951 1952 DELAY(100 * 1000); 1953 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX); 1954 1955 1956 pi->phy.ops->power_down(&pi->phy, 1); 1957 1958 PORT_UNLOCK(pi); 1959 1960 pi->link_config.link_ok = 0; 1961 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0); 1962 1963 if ((sc->open_device_map & PORT_MASK) == 0) 1964 offload_close(&sc->tdev); 1965 1966 if (sc->open_device_map == 0) 1967 cxgb_down(pi->adapter); 1968 1969 return (0); 1970} 1971 1972/* 1973 * Mark lro enabled or disabled in all qsets for this port 1974 */ 1975static int 1976cxgb_set_lro(struct port_info *p, int enabled) 1977{ 1978 int i; 1979 struct adapter *adp = p->adapter; 1980 struct sge_qset *q; 1981 1982 PORT_LOCK_ASSERT_OWNED(p); 1983 for (i = 0; i < p->nqsets; i++) { 1984 q = &adp->sge.qs[p->first_qset + i]; 1985 q->lro.enabled = (enabled != 0); 1986 } 1987 return (0); 1988} 1989 1990static int 1991cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1992{ 1993 struct port_info *p = ifp->if_softc; 1994 struct adapter *sc = p->adapter; 1995 struct ifreq *ifr = (struct ifreq *)data; 1996 int flags, error = 0, mtu; 1997 uint32_t mask; 1998 1999 switch (command) { 2000 case SIOCSIFMTU: 2001 ADAPTER_LOCK(sc); 2002 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 2003 if (error) { 2004fail: 2005 ADAPTER_UNLOCK(sc); 2006 return (error); 2007 } 2008 2009 mtu = ifr->ifr_mtu; 2010 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 2011 error = EINVAL; 2012 } else { 2013 ifp->if_mtu = mtu; 2014 PORT_LOCK(p); 2015 cxgb_update_mac_settings(p); 2016 PORT_UNLOCK(p); 2017 } 2018 ADAPTER_UNLOCK(sc); 2019 break; 2020 case SIOCSIFFLAGS: 2021 ADAPTER_LOCK(sc); 2022 if (IS_DOOMED(p)) { 2023 error = ENXIO; 2024 goto fail; 2025 } 2026 if (ifp->if_flags & IFF_UP) { 2027 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2028 flags = p->if_flags; 2029 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 2030 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) { 2031 if (IS_BUSY(sc)) { 2032 error = EBUSY; 2033 goto fail; 2034 } 2035 PORT_LOCK(p); 2036 cxgb_update_mac_settings(p); 2037 PORT_UNLOCK(p); 2038 } 2039 ADAPTER_UNLOCK(sc); 2040 } else 2041 error = cxgb_init_locked(p); 2042 p->if_flags = ifp->if_flags; 2043 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2044 error = cxgb_uninit_locked(p); 2045 else 2046 ADAPTER_UNLOCK(sc); 2047 2048 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2049 break; 2050 case SIOCADDMULTI: 2051 case SIOCDELMULTI: 2052 ADAPTER_LOCK(sc); 2053 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 2054 if (error) 2055 goto fail; 2056 2057 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2058 PORT_LOCK(p); 2059 cxgb_update_mac_settings(p); 2060 PORT_UNLOCK(p); 2061 } 2062 ADAPTER_UNLOCK(sc); 2063 2064 break; 2065 case SIOCSIFCAP: 2066 ADAPTER_LOCK(sc); 2067 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 2068 if (error) 2069 goto fail; 2070 2071 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 2072 if (mask & IFCAP_TXCSUM) { 2073 ifp->if_capenable ^= IFCAP_TXCSUM; 2074 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 2075 2076 if (IFCAP_TSO & ifp->if_capenable && 2077 !(IFCAP_TXCSUM & ifp->if_capenable)) { 2078 ifp->if_capenable &= ~IFCAP_TSO; 2079 ifp->if_hwassist &= ~CSUM_TSO; 2080 if_printf(ifp, 2081 "tso disabled due to -txcsum.\n"); 2082 } 2083 } 2084 if (mask & IFCAP_RXCSUM) 2085 ifp->if_capenable ^= IFCAP_RXCSUM; 2086 if (mask & IFCAP_TSO4) { 2087 ifp->if_capenable ^= IFCAP_TSO4; 2088 2089 if (IFCAP_TSO & ifp->if_capenable) { 2090 if (IFCAP_TXCSUM & ifp->if_capenable) 2091 ifp->if_hwassist |= CSUM_TSO; 2092 else { 2093 ifp->if_capenable &= ~IFCAP_TSO; 2094 ifp->if_hwassist &= ~CSUM_TSO; 2095 if_printf(ifp, 2096 "enable txcsum first.\n"); 2097 error = EAGAIN; 2098 } 2099 } else 2100 ifp->if_hwassist &= ~CSUM_TSO; 2101 } 2102 if (mask & IFCAP_LRO) { 2103 ifp->if_capenable ^= IFCAP_LRO; 2104 2105 /* Safe to do this even if cxgb_up not called yet */ 2106 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2107 } 2108 if (mask & IFCAP_VLAN_HWTAGGING) { 2109 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2110 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2111 PORT_LOCK(p); 2112 cxgb_update_mac_settings(p); 2113 PORT_UNLOCK(p); 2114 } 2115 } 2116 if (mask & IFCAP_VLAN_MTU) { 2117 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2118 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2119 PORT_LOCK(p); 2120 cxgb_update_mac_settings(p); 2121 PORT_UNLOCK(p); 2122 } 2123 } 2124 if (mask & IFCAP_VLAN_HWTSO) 2125 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 2126 if (mask & IFCAP_VLAN_HWCSUM) 2127 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2128 2129#ifdef VLAN_CAPABILITIES 2130 VLAN_CAPABILITIES(ifp); 2131#endif 2132 ADAPTER_UNLOCK(sc); 2133 break; 2134 case SIOCSIFMEDIA: 2135 case SIOCGIFMEDIA: 2136 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2137 break; 2138 default: 2139 error = ether_ioctl(ifp, command, data); 2140 } 2141 2142 return (error); 2143} 2144 2145static int 2146cxgb_media_change(struct ifnet *ifp) 2147{ 2148 return (EOPNOTSUPP); 2149} 2150 2151/* 2152 * Translates phy->modtype to the correct Ethernet media subtype. 2153 */ 2154static int 2155cxgb_ifm_type(int mod) 2156{ 2157 switch (mod) { 2158 case phy_modtype_sr: 2159 return (IFM_10G_SR); 2160 case phy_modtype_lr: 2161 return (IFM_10G_LR); 2162 case phy_modtype_lrm: 2163 return (IFM_10G_LRM); 2164 case phy_modtype_twinax: 2165 return (IFM_10G_TWINAX); 2166 case phy_modtype_twinax_long: 2167 return (IFM_10G_TWINAX_LONG); 2168 case phy_modtype_none: 2169 return (IFM_NONE); 2170 case phy_modtype_unknown: 2171 return (IFM_UNKNOWN); 2172 } 2173 2174 KASSERT(0, ("%s: modtype %d unknown", __func__, mod)); 2175 return (IFM_UNKNOWN); 2176} 2177 2178/* 2179 * Rebuilds the ifmedia list for this port, and sets the current media. 2180 */ 2181static void 2182cxgb_build_medialist(struct port_info *p) 2183{ 2184 struct cphy *phy = &p->phy; 2185 struct ifmedia *media = &p->media; 2186 int mod = phy->modtype; 2187 int m = IFM_ETHER | IFM_FDX; 2188 2189 PORT_LOCK(p); 2190 2191 ifmedia_removeall(media); 2192 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) { 2193 /* Copper (RJ45) */ 2194 2195 if (phy->caps & SUPPORTED_10000baseT_Full) 2196 ifmedia_add(media, m | IFM_10G_T, mod, NULL); 2197 2198 if (phy->caps & SUPPORTED_1000baseT_Full) 2199 ifmedia_add(media, m | IFM_1000_T, mod, NULL); 2200 2201 if (phy->caps & SUPPORTED_100baseT_Full) 2202 ifmedia_add(media, m | IFM_100_TX, mod, NULL); 2203 2204 if (phy->caps & SUPPORTED_10baseT_Full) 2205 ifmedia_add(media, m | IFM_10_T, mod, NULL); 2206 2207 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL); 2208 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2209 2210 } else if (phy->caps & SUPPORTED_TP) { 2211 /* Copper (CX4) */ 2212 2213 KASSERT(phy->caps & SUPPORTED_10000baseT_Full, 2214 ("%s: unexpected cap 0x%x", __func__, phy->caps)); 2215 2216 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL); 2217 ifmedia_set(media, m | IFM_10G_CX4); 2218 2219 } else if (phy->caps & SUPPORTED_FIBRE && 2220 phy->caps & SUPPORTED_10000baseT_Full) { 2221 /* 10G optical (but includes SFP+ twinax) */ 2222 2223 m |= cxgb_ifm_type(mod); 2224 if (IFM_SUBTYPE(m) == IFM_NONE) 2225 m &= ~IFM_FDX; 2226 2227 ifmedia_add(media, m, mod, NULL); 2228 ifmedia_set(media, m); 2229 2230 } else if (phy->caps & SUPPORTED_FIBRE && 2231 phy->caps & SUPPORTED_1000baseT_Full) { 2232 /* 1G optical */ 2233 2234 /* XXX: Lie and claim to be SX, could actually be any 1G-X */ 2235 ifmedia_add(media, m | IFM_1000_SX, mod, NULL); 2236 ifmedia_set(media, m | IFM_1000_SX); 2237 2238 } else { 2239 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__, 2240 phy->caps)); 2241 } 2242 2243 PORT_UNLOCK(p); 2244} 2245 2246static void 2247cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2248{ 2249 struct port_info *p = ifp->if_softc; 2250 struct ifmedia_entry *cur = p->media.ifm_cur; 2251 int speed = p->link_config.speed; 2252 2253 if (cur->ifm_data != p->phy.modtype) { 2254 cxgb_build_medialist(p); 2255 cur = p->media.ifm_cur; 2256 } 2257 2258 ifmr->ifm_status = IFM_AVALID; 2259 if (!p->link_config.link_ok) 2260 return; 2261 2262 ifmr->ifm_status |= IFM_ACTIVE; 2263 2264 /* 2265 * active and current will differ iff current media is autoselect. That 2266 * can happen only for copper RJ45. 2267 */ 2268 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 2269 return; 2270 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg, 2271 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps)); 2272 2273 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 2274 if (speed == SPEED_10000) 2275 ifmr->ifm_active |= IFM_10G_T; 2276 else if (speed == SPEED_1000) 2277 ifmr->ifm_active |= IFM_1000_T; 2278 else if (speed == SPEED_100) 2279 ifmr->ifm_active |= IFM_100_TX; 2280 else if (speed == SPEED_10) 2281 ifmr->ifm_active |= IFM_10_T; 2282 else 2283 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 2284 speed)); 2285} 2286 2287static void 2288cxgb_async_intr(void *data) 2289{ 2290 adapter_t *sc = data; 2291 2292 if (cxgb_debug) 2293 device_printf(sc->dev, "cxgb_async_intr\n"); 2294 /* 2295 * May need to sleep - defer to taskqueue 2296 */ 2297 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2298} 2299 2300static void 2301cxgb_ext_intr_handler(void *arg, int count) 2302{ 2303 adapter_t *sc = (adapter_t *)arg; 2304 2305 if (cxgb_debug) 2306 printf("cxgb_ext_intr_handler\n"); 2307 2308 t3_phy_intr_handler(sc); 2309 2310 /* Now reenable external interrupts */ 2311 ADAPTER_LOCK(sc); 2312 if (sc->slow_intr_mask) { 2313 sc->slow_intr_mask |= F_T3DBG; 2314 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG); 2315 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); 2316 } 2317 ADAPTER_UNLOCK(sc); 2318} 2319 2320static inline int 2321link_poll_needed(struct port_info *p) 2322{ 2323 struct cphy *phy = &p->phy; 2324 2325 if (phy->caps & POLL_LINK_1ST_TIME) { 2326 p->phy.caps &= ~POLL_LINK_1ST_TIME; 2327 return (1); 2328 } 2329 2330 return (p->link_fault || !(phy->caps & SUPPORTED_LINK_IRQ)); 2331} 2332 2333static void 2334check_link_status(adapter_t *sc) 2335{ 2336 int i; 2337 2338 for (i = 0; i < (sc)->params.nports; ++i) { 2339 struct port_info *p = &sc->port[i]; 2340 2341 if (!isset(&sc->open_device_map, p->port_id)) 2342 continue; 2343 2344 if (link_poll_needed(p)) 2345 t3_link_changed(sc, i); 2346 } 2347} 2348 2349static void 2350check_t3b2_mac(struct adapter *sc) 2351{ 2352 int i; 2353 2354 if (sc->flags & CXGB_SHUTDOWN) 2355 return; 2356 2357 for_each_port(sc, i) { 2358 struct port_info *p = &sc->port[i]; 2359 int status; 2360#ifdef INVARIANTS 2361 struct ifnet *ifp = p->ifp; 2362#endif 2363 2364 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault || 2365 !p->link_config.link_ok) 2366 continue; 2367 2368 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2369 ("%s: state mismatch (drv_flags %x, device_map %x)", 2370 __func__, ifp->if_drv_flags, sc->open_device_map)); 2371 2372 PORT_LOCK(p); 2373 status = t3b2_mac_watchdog_task(&p->mac); 2374 if (status == 1) 2375 p->mac.stats.num_toggled++; 2376 else if (status == 2) { 2377 struct cmac *mac = &p->mac; 2378 2379 cxgb_update_mac_settings(p); 2380 t3_link_start(&p->phy, mac, &p->link_config); 2381 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2382 t3_port_intr_enable(sc, p->port_id); 2383 p->mac.stats.num_resets++; 2384 } 2385 PORT_UNLOCK(p); 2386 } 2387} 2388 2389static void 2390cxgb_tick(void *arg) 2391{ 2392 adapter_t *sc = (adapter_t *)arg; 2393 2394 if (sc->flags & CXGB_SHUTDOWN) 2395 return; 2396 2397 taskqueue_enqueue(sc->tq, &sc->tick_task); 2398 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc); 2399} 2400 2401static void 2402cxgb_tick_handler(void *arg, int count) 2403{ 2404 adapter_t *sc = (adapter_t *)arg; 2405 const struct adapter_params *p = &sc->params; 2406 int i; 2407 uint32_t cause, reset; 2408 2409 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE)) 2410 return; 2411 2412 check_link_status(sc); 2413 2414 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2415 check_t3b2_mac(sc); 2416 2417 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY); 2418 if (cause) { 2419 struct sge_qset *qs = &sc->sge.qs[0]; 2420 uint32_t mask, v; 2421 2422 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00; 2423 2424 mask = 1; 2425 for (i = 0; i < SGE_QSETS; i++) { 2426 if (v & mask) 2427 qs[i].rspq.starved++; 2428 mask <<= 1; 2429 } 2430 2431 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */ 2432 2433 for (i = 0; i < SGE_QSETS * 2; i++) { 2434 if (v & mask) { 2435 qs[i / 2].fl[i % 2].empty++; 2436 } 2437 mask <<= 1; 2438 } 2439 2440 /* clear */ 2441 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v); 2442 t3_write_reg(sc, A_SG_INT_CAUSE, cause); 2443 } 2444 2445 for (i = 0; i < sc->params.nports; i++) { 2446 struct port_info *pi = &sc->port[i]; 2447 struct ifnet *ifp = pi->ifp; 2448 struct cmac *mac = &pi->mac; 2449 struct mac_stats *mstats = &mac->stats; 2450 int drops, j; 2451 2452 if (!isset(&sc->open_device_map, pi->port_id)) 2453 continue; 2454 2455 PORT_LOCK(pi); 2456 t3_mac_update_stats(mac); 2457 PORT_UNLOCK(pi); 2458 2459 ifp->if_opackets = mstats->tx_frames; 2460 ifp->if_ipackets = mstats->rx_frames; 2461 ifp->if_obytes = mstats->tx_octets; 2462 ifp->if_ibytes = mstats->rx_octets; 2463 ifp->if_omcasts = mstats->tx_mcast_frames; 2464 ifp->if_imcasts = mstats->rx_mcast_frames; 2465 ifp->if_collisions = mstats->tx_total_collisions; 2466 ifp->if_iqdrops = mstats->rx_cong_drops; 2467 2468 drops = 0; 2469 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets; j++) 2470 drops += sc->sge.qs[j].txq[TXQ_ETH].txq_mr->br_drops; 2471 ifp->if_snd.ifq_drops = drops; 2472 2473 ifp->if_oerrors = 2474 mstats->tx_excess_collisions + 2475 mstats->tx_underrun + 2476 mstats->tx_len_errs + 2477 mstats->tx_mac_internal_errs + 2478 mstats->tx_excess_deferral + 2479 mstats->tx_fcs_errs; 2480 ifp->if_ierrors = 2481 mstats->rx_jabber + 2482 mstats->rx_data_errs + 2483 mstats->rx_sequence_errs + 2484 mstats->rx_runt + 2485 mstats->rx_too_long + 2486 mstats->rx_mac_internal_errs + 2487 mstats->rx_short + 2488 mstats->rx_fcs_errs; 2489 2490 if (mac->multiport) 2491 continue; 2492 2493 /* Count rx fifo overflows, once per second */ 2494 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2495 reset = 0; 2496 if (cause & F_RXFIFO_OVERFLOW) { 2497 mac->stats.rx_fifo_ovfl++; 2498 reset |= F_RXFIFO_OVERFLOW; 2499 } 2500 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2501 } 2502} 2503 2504static void 2505touch_bars(device_t dev) 2506{ 2507 /* 2508 * Don't enable yet 2509 */ 2510#if !defined(__LP64__) && 0 2511 u32 v; 2512 2513 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2514 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2515 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2516 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2517 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2518 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2519#endif 2520} 2521 2522static int 2523set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2524{ 2525 uint8_t *buf; 2526 int err = 0; 2527 u32 aligned_offset, aligned_len, *p; 2528 struct adapter *adapter = pi->adapter; 2529 2530 2531 aligned_offset = offset & ~3; 2532 aligned_len = (len + (offset & 3) + 3) & ~3; 2533 2534 if (aligned_offset != offset || aligned_len != len) { 2535 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2536 if (!buf) 2537 return (ENOMEM); 2538 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2539 if (!err && aligned_len > 4) 2540 err = t3_seeprom_read(adapter, 2541 aligned_offset + aligned_len - 4, 2542 (u32 *)&buf[aligned_len - 4]); 2543 if (err) 2544 goto out; 2545 memcpy(buf + (offset & 3), data, len); 2546 } else 2547 buf = (uint8_t *)(uintptr_t)data; 2548 2549 err = t3_seeprom_wp(adapter, 0); 2550 if (err) 2551 goto out; 2552 2553 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2554 err = t3_seeprom_write(adapter, aligned_offset, *p); 2555 aligned_offset += 4; 2556 } 2557 2558 if (!err) 2559 err = t3_seeprom_wp(adapter, 1); 2560out: 2561 if (buf != data) 2562 free(buf, M_DEVBUF); 2563 return err; 2564} 2565 2566 2567static int 2568in_range(int val, int lo, int hi) 2569{ 2570 return val < 0 || (val <= hi && val >= lo); 2571} 2572 2573static int 2574cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td) 2575{ 2576 return (0); 2577} 2578 2579static int 2580cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td) 2581{ 2582 return (0); 2583} 2584 2585static int 2586cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2587 int fflag, struct thread *td) 2588{ 2589 int mmd, error = 0; 2590 struct port_info *pi = dev->si_drv1; 2591 adapter_t *sc = pi->adapter; 2592 2593#ifdef PRIV_SUPPORTED 2594 if (priv_check(td, PRIV_DRIVER)) { 2595 if (cxgb_debug) 2596 printf("user does not have access to privileged ioctls\n"); 2597 return (EPERM); 2598 } 2599#else 2600 if (suser(td)) { 2601 if (cxgb_debug) 2602 printf("user does not have access to privileged ioctls\n"); 2603 return (EPERM); 2604 } 2605#endif 2606 2607 switch (cmd) { 2608 case CHELSIO_GET_MIIREG: { 2609 uint32_t val; 2610 struct cphy *phy = &pi->phy; 2611 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2612 2613 if (!phy->mdio_read) 2614 return (EOPNOTSUPP); 2615 if (is_10G(sc)) { 2616 mmd = mid->phy_id >> 8; 2617 if (!mmd) 2618 mmd = MDIO_DEV_PCS; 2619 else if (mmd > MDIO_DEV_VEND2) 2620 return (EINVAL); 2621 2622 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2623 mid->reg_num, &val); 2624 } else 2625 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2626 mid->reg_num & 0x1f, &val); 2627 if (error == 0) 2628 mid->val_out = val; 2629 break; 2630 } 2631 case CHELSIO_SET_MIIREG: { 2632 struct cphy *phy = &pi->phy; 2633 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2634 2635 if (!phy->mdio_write) 2636 return (EOPNOTSUPP); 2637 if (is_10G(sc)) { 2638 mmd = mid->phy_id >> 8; 2639 if (!mmd) 2640 mmd = MDIO_DEV_PCS; 2641 else if (mmd > MDIO_DEV_VEND2) 2642 return (EINVAL); 2643 2644 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2645 mmd, mid->reg_num, mid->val_in); 2646 } else 2647 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2648 mid->reg_num & 0x1f, 2649 mid->val_in); 2650 break; 2651 } 2652 case CHELSIO_SETREG: { 2653 struct ch_reg *edata = (struct ch_reg *)data; 2654 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2655 return (EFAULT); 2656 t3_write_reg(sc, edata->addr, edata->val); 2657 break; 2658 } 2659 case CHELSIO_GETREG: { 2660 struct ch_reg *edata = (struct ch_reg *)data; 2661 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2662 return (EFAULT); 2663 edata->val = t3_read_reg(sc, edata->addr); 2664 break; 2665 } 2666 case CHELSIO_GET_SGE_CONTEXT: { 2667 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2668 mtx_lock_spin(&sc->sge.reg_lock); 2669 switch (ecntxt->cntxt_type) { 2670 case CNTXT_TYPE_EGRESS: 2671 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2672 ecntxt->data); 2673 break; 2674 case CNTXT_TYPE_FL: 2675 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2676 ecntxt->data); 2677 break; 2678 case CNTXT_TYPE_RSP: 2679 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2680 ecntxt->data); 2681 break; 2682 case CNTXT_TYPE_CQ: 2683 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2684 ecntxt->data); 2685 break; 2686 default: 2687 error = EINVAL; 2688 break; 2689 } 2690 mtx_unlock_spin(&sc->sge.reg_lock); 2691 break; 2692 } 2693 case CHELSIO_GET_SGE_DESC: { 2694 struct ch_desc *edesc = (struct ch_desc *)data; 2695 int ret; 2696 if (edesc->queue_num >= SGE_QSETS * 6) 2697 return (EINVAL); 2698 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2699 edesc->queue_num % 6, edesc->idx, edesc->data); 2700 if (ret < 0) 2701 return (EINVAL); 2702 edesc->size = ret; 2703 break; 2704 } 2705 case CHELSIO_GET_QSET_PARAMS: { 2706 struct qset_params *q; 2707 struct ch_qset_params *t = (struct ch_qset_params *)data; 2708 int q1 = pi->first_qset; 2709 int nqsets = pi->nqsets; 2710 int i; 2711 2712 if (t->qset_idx >= nqsets) 2713 return EINVAL; 2714 2715 i = q1 + t->qset_idx; 2716 q = &sc->params.sge.qset[i]; 2717 t->rspq_size = q->rspq_size; 2718 t->txq_size[0] = q->txq_size[0]; 2719 t->txq_size[1] = q->txq_size[1]; 2720 t->txq_size[2] = q->txq_size[2]; 2721 t->fl_size[0] = q->fl_size; 2722 t->fl_size[1] = q->jumbo_size; 2723 t->polling = q->polling; 2724 t->lro = q->lro; 2725 t->intr_lat = q->coalesce_usecs; 2726 t->cong_thres = q->cong_thres; 2727 t->qnum = i; 2728 2729 if ((sc->flags & FULL_INIT_DONE) == 0) 2730 t->vector = 0; 2731 else if (sc->flags & USING_MSIX) 2732 t->vector = rman_get_start(sc->msix_irq_res[i]); 2733 else 2734 t->vector = rman_get_start(sc->irq_res); 2735 2736 break; 2737 } 2738 case CHELSIO_GET_QSET_NUM: { 2739 struct ch_reg *edata = (struct ch_reg *)data; 2740 edata->val = pi->nqsets; 2741 break; 2742 } 2743 case CHELSIO_LOAD_FW: { 2744 uint8_t *fw_data; 2745 uint32_t vers; 2746 struct ch_mem_range *t = (struct ch_mem_range *)data; 2747 2748 /* 2749 * You're allowed to load a firmware only before FULL_INIT_DONE 2750 * 2751 * FW_UPTODATE is also set so the rest of the initialization 2752 * will not overwrite what was loaded here. This gives you the 2753 * flexibility to load any firmware (and maybe shoot yourself in 2754 * the foot). 2755 */ 2756 2757 ADAPTER_LOCK(sc); 2758 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2759 ADAPTER_UNLOCK(sc); 2760 return (EBUSY); 2761 } 2762 2763 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2764 if (!fw_data) 2765 error = ENOMEM; 2766 else 2767 error = copyin(t->buf, fw_data, t->len); 2768 2769 if (!error) 2770 error = -t3_load_fw(sc, fw_data, t->len); 2771 2772 if (t3_get_fw_version(sc, &vers) == 0) { 2773 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2774 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2775 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2776 } 2777 2778 if (!error) 2779 sc->flags |= FW_UPTODATE; 2780 2781 free(fw_data, M_DEVBUF); 2782 ADAPTER_UNLOCK(sc); 2783 break; 2784 } 2785 case CHELSIO_LOAD_BOOT: { 2786 uint8_t *boot_data; 2787 struct ch_mem_range *t = (struct ch_mem_range *)data; 2788 2789 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2790 if (!boot_data) 2791 return ENOMEM; 2792 2793 error = copyin(t->buf, boot_data, t->len); 2794 if (!error) 2795 error = -t3_load_boot(sc, boot_data, t->len); 2796 2797 free(boot_data, M_DEVBUF); 2798 break; 2799 } 2800 case CHELSIO_GET_PM: { 2801 struct ch_pm *m = (struct ch_pm *)data; 2802 struct tp_params *p = &sc->params.tp; 2803 2804 if (!is_offload(sc)) 2805 return (EOPNOTSUPP); 2806 2807 m->tx_pg_sz = p->tx_pg_size; 2808 m->tx_num_pg = p->tx_num_pgs; 2809 m->rx_pg_sz = p->rx_pg_size; 2810 m->rx_num_pg = p->rx_num_pgs; 2811 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2812 2813 break; 2814 } 2815 case CHELSIO_SET_PM: { 2816 struct ch_pm *m = (struct ch_pm *)data; 2817 struct tp_params *p = &sc->params.tp; 2818 2819 if (!is_offload(sc)) 2820 return (EOPNOTSUPP); 2821 if (sc->flags & FULL_INIT_DONE) 2822 return (EBUSY); 2823 2824 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2825 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2826 return (EINVAL); /* not power of 2 */ 2827 if (!(m->rx_pg_sz & 0x14000)) 2828 return (EINVAL); /* not 16KB or 64KB */ 2829 if (!(m->tx_pg_sz & 0x1554000)) 2830 return (EINVAL); 2831 if (m->tx_num_pg == -1) 2832 m->tx_num_pg = p->tx_num_pgs; 2833 if (m->rx_num_pg == -1) 2834 m->rx_num_pg = p->rx_num_pgs; 2835 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2836 return (EINVAL); 2837 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2838 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2839 return (EINVAL); 2840 2841 p->rx_pg_size = m->rx_pg_sz; 2842 p->tx_pg_size = m->tx_pg_sz; 2843 p->rx_num_pgs = m->rx_num_pg; 2844 p->tx_num_pgs = m->tx_num_pg; 2845 break; 2846 } 2847 case CHELSIO_SETMTUTAB: { 2848 struct ch_mtus *m = (struct ch_mtus *)data; 2849 int i; 2850 2851 if (!is_offload(sc)) 2852 return (EOPNOTSUPP); 2853 if (offload_running(sc)) 2854 return (EBUSY); 2855 if (m->nmtus != NMTUS) 2856 return (EINVAL); 2857 if (m->mtus[0] < 81) /* accommodate SACK */ 2858 return (EINVAL); 2859 2860 /* 2861 * MTUs must be in ascending order 2862 */ 2863 for (i = 1; i < NMTUS; ++i) 2864 if (m->mtus[i] < m->mtus[i - 1]) 2865 return (EINVAL); 2866 2867 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2868 break; 2869 } 2870 case CHELSIO_GETMTUTAB: { 2871 struct ch_mtus *m = (struct ch_mtus *)data; 2872 2873 if (!is_offload(sc)) 2874 return (EOPNOTSUPP); 2875 2876 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2877 m->nmtus = NMTUS; 2878 break; 2879 } 2880 case CHELSIO_GET_MEM: { 2881 struct ch_mem_range *t = (struct ch_mem_range *)data; 2882 struct mc7 *mem; 2883 uint8_t *useraddr; 2884 u64 buf[32]; 2885 2886 /* 2887 * Use these to avoid modifying len/addr in the the return 2888 * struct 2889 */ 2890 uint32_t len = t->len, addr = t->addr; 2891 2892 if (!is_offload(sc)) 2893 return (EOPNOTSUPP); 2894 if (!(sc->flags & FULL_INIT_DONE)) 2895 return (EIO); /* need the memory controllers */ 2896 if ((addr & 0x7) || (len & 0x7)) 2897 return (EINVAL); 2898 if (t->mem_id == MEM_CM) 2899 mem = &sc->cm; 2900 else if (t->mem_id == MEM_PMRX) 2901 mem = &sc->pmrx; 2902 else if (t->mem_id == MEM_PMTX) 2903 mem = &sc->pmtx; 2904 else 2905 return (EINVAL); 2906 2907 /* 2908 * Version scheme: 2909 * bits 0..9: chip version 2910 * bits 10..15: chip revision 2911 */ 2912 t->version = 3 | (sc->params.rev << 10); 2913 2914 /* 2915 * Read 256 bytes at a time as len can be large and we don't 2916 * want to use huge intermediate buffers. 2917 */ 2918 useraddr = (uint8_t *)t->buf; 2919 while (len) { 2920 unsigned int chunk = min(len, sizeof(buf)); 2921 2922 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2923 if (error) 2924 return (-error); 2925 if (copyout(buf, useraddr, chunk)) 2926 return (EFAULT); 2927 useraddr += chunk; 2928 addr += chunk; 2929 len -= chunk; 2930 } 2931 break; 2932 } 2933 case CHELSIO_READ_TCAM_WORD: { 2934 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2935 2936 if (!is_offload(sc)) 2937 return (EOPNOTSUPP); 2938 if (!(sc->flags & FULL_INIT_DONE)) 2939 return (EIO); /* need MC5 */ 2940 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2941 break; 2942 } 2943 case CHELSIO_SET_TRACE_FILTER: { 2944 struct ch_trace *t = (struct ch_trace *)data; 2945 const struct trace_params *tp; 2946 2947 tp = (const struct trace_params *)&t->sip; 2948 if (t->config_tx) 2949 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2950 t->trace_tx); 2951 if (t->config_rx) 2952 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2953 t->trace_rx); 2954 break; 2955 } 2956 case CHELSIO_SET_PKTSCHED: { 2957 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2958 if (sc->open_device_map == 0) 2959 return (EAGAIN); 2960 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2961 p->binding); 2962 break; 2963 } 2964 case CHELSIO_IFCONF_GETREGS: { 2965 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2966 int reglen = cxgb_get_regs_len(); 2967 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2968 if (buf == NULL) { 2969 return (ENOMEM); 2970 } 2971 if (regs->len > reglen) 2972 regs->len = reglen; 2973 else if (regs->len < reglen) 2974 error = ENOBUFS; 2975 2976 if (!error) { 2977 cxgb_get_regs(sc, regs, buf); 2978 error = copyout(buf, regs->data, reglen); 2979 } 2980 free(buf, M_DEVBUF); 2981 2982 break; 2983 } 2984 case CHELSIO_SET_HW_SCHED: { 2985 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2986 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2987 2988 if ((sc->flags & FULL_INIT_DONE) == 0) 2989 return (EAGAIN); /* need TP to be initialized */ 2990 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2991 !in_range(t->channel, 0, 1) || 2992 !in_range(t->kbps, 0, 10000000) || 2993 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2994 !in_range(t->flow_ipg, 0, 2995 dack_ticks_to_usec(sc, 0x7ff))) 2996 return (EINVAL); 2997 2998 if (t->kbps >= 0) { 2999 error = t3_config_sched(sc, t->kbps, t->sched); 3000 if (error < 0) 3001 return (-error); 3002 } 3003 if (t->class_ipg >= 0) 3004 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 3005 if (t->flow_ipg >= 0) { 3006 t->flow_ipg *= 1000; /* us -> ns */ 3007 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 3008 } 3009 if (t->mode >= 0) { 3010 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 3011 3012 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 3013 bit, t->mode ? bit : 0); 3014 } 3015 if (t->channel >= 0) 3016 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 3017 1 << t->sched, t->channel << t->sched); 3018 break; 3019 } 3020 case CHELSIO_GET_EEPROM: { 3021 int i; 3022 struct ch_eeprom *e = (struct ch_eeprom *)data; 3023 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 3024 3025 if (buf == NULL) { 3026 return (ENOMEM); 3027 } 3028 e->magic = EEPROM_MAGIC; 3029 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 3030 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 3031 3032 if (!error) 3033 error = copyout(buf + e->offset, e->data, e->len); 3034 3035 free(buf, M_DEVBUF); 3036 break; 3037 } 3038 case CHELSIO_CLEAR_STATS: { 3039 if (!(sc->flags & FULL_INIT_DONE)) 3040 return EAGAIN; 3041 3042 PORT_LOCK(pi); 3043 t3_mac_update_stats(&pi->mac); 3044 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 3045 PORT_UNLOCK(pi); 3046 break; 3047 } 3048 case CHELSIO_GET_UP_LA: { 3049 struct ch_up_la *la = (struct ch_up_la *)data; 3050 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 3051 if (buf == NULL) { 3052 return (ENOMEM); 3053 } 3054 if (la->bufsize < LA_BUFSIZE) 3055 error = ENOBUFS; 3056 3057 if (!error) 3058 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 3059 &la->bufsize, buf); 3060 if (!error) 3061 error = copyout(buf, la->data, la->bufsize); 3062 3063 free(buf, M_DEVBUF); 3064 break; 3065 } 3066 case CHELSIO_GET_UP_IOQS: { 3067 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 3068 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 3069 uint32_t *v; 3070 3071 if (buf == NULL) { 3072 return (ENOMEM); 3073 } 3074 if (ioqs->bufsize < IOQS_BUFSIZE) 3075 error = ENOBUFS; 3076 3077 if (!error) 3078 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 3079 3080 if (!error) { 3081 v = (uint32_t *)buf; 3082 3083 ioqs->bufsize -= 4 * sizeof(uint32_t); 3084 ioqs->ioq_rx_enable = *v++; 3085 ioqs->ioq_tx_enable = *v++; 3086 ioqs->ioq_rx_status = *v++; 3087 ioqs->ioq_tx_status = *v++; 3088 3089 error = copyout(v, ioqs->data, ioqs->bufsize); 3090 } 3091 3092 free(buf, M_DEVBUF); 3093 break; 3094 } 3095 case CHELSIO_SET_FILTER: { 3096 struct ch_filter *f = (struct ch_filter *)data;; 3097 struct filter_info *p; 3098 unsigned int nfilters = sc->params.mc5.nfilters; 3099 3100 if (!is_offload(sc)) 3101 return (EOPNOTSUPP); /* No TCAM */ 3102 if (!(sc->flags & FULL_INIT_DONE)) 3103 return (EAGAIN); /* mc5 not setup yet */ 3104 if (nfilters == 0) 3105 return (EBUSY); /* TOE will use TCAM */ 3106 3107 /* sanity checks */ 3108 if (f->filter_id >= nfilters || 3109 (f->val.dip && f->mask.dip != 0xffffffff) || 3110 (f->val.sport && f->mask.sport != 0xffff) || 3111 (f->val.dport && f->mask.dport != 0xffff) || 3112 (f->val.vlan && f->mask.vlan != 0xfff) || 3113 (f->val.vlan_prio && 3114 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) || 3115 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) || 3116 f->qset >= SGE_QSETS || 3117 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE) 3118 return (EINVAL); 3119 3120 /* Was allocated with M_WAITOK */ 3121 KASSERT(sc->filters, ("filter table NULL\n")); 3122 3123 p = &sc->filters[f->filter_id]; 3124 if (p->locked) 3125 return (EPERM); 3126 3127 bzero(p, sizeof(*p)); 3128 p->sip = f->val.sip; 3129 p->sip_mask = f->mask.sip; 3130 p->dip = f->val.dip; 3131 p->sport = f->val.sport; 3132 p->dport = f->val.dport; 3133 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff; 3134 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) : 3135 FILTER_NO_VLAN_PRI; 3136 p->mac_hit = f->mac_hit; 3137 p->mac_vld = f->mac_addr_idx != 0xffff; 3138 p->mac_idx = f->mac_addr_idx; 3139 p->pkt_type = f->proto; 3140 p->report_filter_id = f->want_filter_id; 3141 p->pass = f->pass; 3142 p->rss = f->rss; 3143 p->qset = f->qset; 3144 3145 error = set_filter(sc, f->filter_id, p); 3146 if (error == 0) 3147 p->valid = 1; 3148 break; 3149 } 3150 case CHELSIO_DEL_FILTER: { 3151 struct ch_filter *f = (struct ch_filter *)data; 3152 struct filter_info *p; 3153 unsigned int nfilters = sc->params.mc5.nfilters; 3154 3155 if (!is_offload(sc)) 3156 return (EOPNOTSUPP); 3157 if (!(sc->flags & FULL_INIT_DONE)) 3158 return (EAGAIN); 3159 if (nfilters == 0 || sc->filters == NULL) 3160 return (EINVAL); 3161 if (f->filter_id >= nfilters) 3162 return (EINVAL); 3163 3164 p = &sc->filters[f->filter_id]; 3165 if (p->locked) 3166 return (EPERM); 3167 if (!p->valid) 3168 return (EFAULT); /* Read "Bad address" as "Bad index" */ 3169 3170 bzero(p, sizeof(*p)); 3171 p->sip = p->sip_mask = 0xffffffff; 3172 p->vlan = 0xfff; 3173 p->vlan_prio = FILTER_NO_VLAN_PRI; 3174 p->pkt_type = 1; 3175 error = set_filter(sc, f->filter_id, p); 3176 break; 3177 } 3178 case CHELSIO_GET_FILTER: { 3179 struct ch_filter *f = (struct ch_filter *)data; 3180 struct filter_info *p; 3181 unsigned int i, nfilters = sc->params.mc5.nfilters; 3182 3183 if (!is_offload(sc)) 3184 return (EOPNOTSUPP); 3185 if (!(sc->flags & FULL_INIT_DONE)) 3186 return (EAGAIN); 3187 if (nfilters == 0 || sc->filters == NULL) 3188 return (EINVAL); 3189 3190 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1; 3191 for (; i < nfilters; i++) { 3192 p = &sc->filters[i]; 3193 if (!p->valid) 3194 continue; 3195 3196 bzero(f, sizeof(*f)); 3197 3198 f->filter_id = i; 3199 f->val.sip = p->sip; 3200 f->mask.sip = p->sip_mask; 3201 f->val.dip = p->dip; 3202 f->mask.dip = p->dip ? 0xffffffff : 0; 3203 f->val.sport = p->sport; 3204 f->mask.sport = p->sport ? 0xffff : 0; 3205 f->val.dport = p->dport; 3206 f->mask.dport = p->dport ? 0xffff : 0; 3207 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan; 3208 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff; 3209 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3210 0 : p->vlan_prio; 3211 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3212 0 : FILTER_NO_VLAN_PRI; 3213 f->mac_hit = p->mac_hit; 3214 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff; 3215 f->proto = p->pkt_type; 3216 f->want_filter_id = p->report_filter_id; 3217 f->pass = p->pass; 3218 f->rss = p->rss; 3219 f->qset = p->qset; 3220 3221 break; 3222 } 3223 3224 if (i == nfilters) 3225 f->filter_id = 0xffffffff; 3226 break; 3227 } 3228 default: 3229 return (EOPNOTSUPP); 3230 break; 3231 } 3232 3233 return (error); 3234} 3235 3236static __inline void 3237reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3238 unsigned int end) 3239{ 3240 uint32_t *p = (uint32_t *)(buf + start); 3241 3242 for ( ; start <= end; start += sizeof(uint32_t)) 3243 *p++ = t3_read_reg(ap, start); 3244} 3245 3246#define T3_REGMAP_SIZE (3 * 1024) 3247static int 3248cxgb_get_regs_len(void) 3249{ 3250 return T3_REGMAP_SIZE; 3251} 3252 3253static void 3254cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3255{ 3256 3257 /* 3258 * Version scheme: 3259 * bits 0..9: chip version 3260 * bits 10..15: chip revision 3261 * bit 31: set for PCIe cards 3262 */ 3263 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3264 3265 /* 3266 * We skip the MAC statistics registers because they are clear-on-read. 3267 * Also reading multi-register stats would need to synchronize with the 3268 * periodic mac stats accumulation. Hard to justify the complexity. 3269 */ 3270 memset(buf, 0, cxgb_get_regs_len()); 3271 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3272 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3273 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3274 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3275 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3276 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3277 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3278 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3279 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3280} 3281 3282static int 3283alloc_filters(struct adapter *sc) 3284{ 3285 struct filter_info *p; 3286 unsigned int nfilters = sc->params.mc5.nfilters; 3287 3288 if (nfilters == 0) 3289 return (0); 3290 3291 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO); 3292 sc->filters = p; 3293 3294 p = &sc->filters[nfilters - 1]; 3295 p->vlan = 0xfff; 3296 p->vlan_prio = FILTER_NO_VLAN_PRI; 3297 p->pass = p->rss = p->valid = p->locked = 1; 3298 3299 return (0); 3300} 3301 3302static int 3303setup_hw_filters(struct adapter *sc) 3304{ 3305 int i, rc; 3306 unsigned int nfilters = sc->params.mc5.nfilters; 3307 3308 if (!sc->filters) 3309 return (0); 3310 3311 t3_enable_filters(sc); 3312 3313 for (i = rc = 0; i < nfilters && !rc; i++) { 3314 if (sc->filters[i].locked) 3315 rc = set_filter(sc, i, &sc->filters[i]); 3316 } 3317 3318 return (rc); 3319} 3320 3321static int 3322set_filter(struct adapter *sc, int id, const struct filter_info *f) 3323{ 3324 int len; 3325 struct mbuf *m; 3326 struct ulp_txpkt *txpkt; 3327 struct work_request_hdr *wr; 3328 struct cpl_pass_open_req *oreq; 3329 struct cpl_set_tcb_field *sreq; 3330 3331 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq); 3332 KASSERT(len <= MHLEN, ("filter request too big for an mbuf")); 3333 3334 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes - 3335 sc->params.mc5.nfilters; 3336 3337 m = m_gethdr(M_WAITOK, MT_DATA); 3338 m->m_len = m->m_pkthdr.len = len; 3339 bzero(mtod(m, char *), len); 3340 3341 wr = mtod(m, struct work_request_hdr *); 3342 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 3343 3344 oreq = (struct cpl_pass_open_req *)(wr + 1); 3345 txpkt = (struct ulp_txpkt *)oreq; 3346 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3347 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8)); 3348 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id)); 3349 oreq->local_port = htons(f->dport); 3350 oreq->peer_port = htons(f->sport); 3351 oreq->local_ip = htonl(f->dip); 3352 oreq->peer_ip = htonl(f->sip); 3353 oreq->peer_netmask = htonl(f->sip_mask); 3354 oreq->opt0h = 0; 3355 oreq->opt0l = htonl(F_NO_OFFLOAD); 3356 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) | 3357 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) | 3358 V_VLAN_PRI(f->vlan_prio >> 1) | 3359 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) | 3360 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) | 3361 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4))); 3362 3363 sreq = (struct cpl_set_tcb_field *)(oreq + 1); 3364 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL, 3365 (f->report_filter_id << 15) | (1 << 23) | 3366 ((u64)f->pass << 35) | ((u64)!f->rss << 36)); 3367 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1); 3368 t3_mgmt_tx(sc, m); 3369 3370 if (f->pass && !f->rss) { 3371 len = sizeof(*sreq); 3372 m = m_gethdr(M_WAITOK, MT_DATA); 3373 m->m_len = m->m_pkthdr.len = len; 3374 bzero(mtod(m, char *), len); 3375 sreq = mtod(m, struct cpl_set_tcb_field *); 3376 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3377 mk_set_tcb_field(sreq, id, 25, 0x3f80000, 3378 (u64)sc->rrss_map[f->qset] << 19); 3379 t3_mgmt_tx(sc, m); 3380 } 3381 return 0; 3382} 3383 3384static inline void 3385mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid, 3386 unsigned int word, u64 mask, u64 val) 3387{ 3388 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 3389 req->reply = V_NO_REPLY(1); 3390 req->cpu_idx = 0; 3391 req->word = htons(word); 3392 req->mask = htobe64(mask); 3393 req->val = htobe64(val); 3394} 3395 3396static inline void 3397set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 3398 unsigned int word, u64 mask, u64 val) 3399{ 3400 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 3401 3402 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3403 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 3404 mk_set_tcb_field(req, tid, word, mask, val); 3405} 3406 3407MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1); 3408