1/************************************************************************** 2 3Copyright (c) 2007-2009, Chelsio Inc. 4All rights reserved. 5 6Redistribution and use in source and binary forms, with or without 7modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26POSSIBILITY OF SUCH DAMAGE. 27 28***************************************************************************/ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: stable/10/sys/dev/cxgb/cxgb_main.c 331320 2018-03-21 20:13:24Z np $"); 32 33#include "opt_inet.h" 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/pciio.h> 41#include <sys/conf.h> 42#include <machine/bus.h> 43#include <machine/resource.h> 44#include <sys/bus_dma.h> 45#include <sys/ktr.h> 46#include <sys/rman.h> 47#include <sys/ioccom.h> 48#include <sys/mbuf.h> 49#include <sys/linker.h> 50#include <sys/firmware.h> 51#include <sys/socket.h> 52#include <sys/sockio.h> 53#include <sys/smp.h> 54#include <sys/sysctl.h> 55#include <sys/syslog.h> 56#include <sys/queue.h> 57#include <sys/taskqueue.h> 58#include <sys/proc.h> 59 60#include <net/bpf.h> 61#include <net/ethernet.h> 62#include <net/if.h> 63#include <net/if_arp.h> 64#include <net/if_dl.h> 65#include <net/if_media.h> 66#include <net/if_types.h> 67#include <net/if_vlan_var.h> 68 69#include <netinet/in_systm.h> 70#include <netinet/in.h> 71#include <netinet/if_ether.h> 72#include <netinet/ip.h> 73#include <netinet/ip.h> 74#include <netinet/tcp.h> 75#include <netinet/udp.h> 76 77#include <dev/pci/pcireg.h> 78#include <dev/pci/pcivar.h> 79#include <dev/pci/pci_private.h> 80 81#include <cxgb_include.h> 82 83#ifdef PRIV_SUPPORTED 84#include <sys/priv.h> 85#endif 86 87static int cxgb_setup_interrupts(adapter_t *); 88static void cxgb_teardown_interrupts(adapter_t *); 89static void cxgb_init(void *); 90static int cxgb_init_locked(struct port_info *); 91static int cxgb_uninit_locked(struct port_info *); 92static int cxgb_uninit_synchronized(struct port_info *); 93static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); 94static int cxgb_media_change(struct ifnet *); 95static int cxgb_ifm_type(int); 96static void cxgb_build_medialist(struct port_info *); 97static void cxgb_media_status(struct ifnet *, struct ifmediareq *); 98static int setup_sge_qsets(adapter_t *); 99static void cxgb_async_intr(void *); 100static void cxgb_tick_handler(void *, int); 101static void cxgb_tick(void *); 102static void link_check_callout(void *); 103static void check_link_status(void *, int); 104static void setup_rss(adapter_t *sc); 105static int alloc_filters(struct adapter *); 106static int setup_hw_filters(struct adapter *); 107static int set_filter(struct adapter *, int, const struct filter_info *); 108static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int, 109 unsigned int, u64, u64); 110static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int, 111 unsigned int, u64, u64); 112#ifdef TCP_OFFLOAD 113static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *); 114#endif 115 116/* Attachment glue for the PCI controller end of the device. Each port of 117 * the device is attached separately, as defined later. 118 */ 119static int cxgb_controller_probe(device_t); 120static int cxgb_controller_attach(device_t); 121static int cxgb_controller_detach(device_t); 122static void cxgb_free(struct adapter *); 123static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 124 unsigned int end); 125static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf); 126static int cxgb_get_regs_len(void); 127static void touch_bars(device_t dev); 128static void cxgb_update_mac_settings(struct port_info *p); 129#ifdef TCP_OFFLOAD 130static int toe_capability(struct port_info *, int); 131#endif 132 133static device_method_t cxgb_controller_methods[] = { 134 DEVMETHOD(device_probe, cxgb_controller_probe), 135 DEVMETHOD(device_attach, cxgb_controller_attach), 136 DEVMETHOD(device_detach, cxgb_controller_detach), 137 138 DEVMETHOD_END 139}; 140 141static driver_t cxgb_controller_driver = { 142 "cxgbc", 143 cxgb_controller_methods, 144 sizeof(struct adapter) 145}; 146 147static int cxgbc_mod_event(module_t, int, void *); 148static devclass_t cxgb_controller_devclass; 149DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 150 cxgbc_mod_event, 0); 151MODULE_VERSION(cxgbc, 1); 152MODULE_DEPEND(cxgbc, firmware, 1, 1, 1); 153 154/* 155 * Attachment glue for the ports. Attachment is done directly to the 156 * controller device. 157 */ 158static int cxgb_port_probe(device_t); 159static int cxgb_port_attach(device_t); 160static int cxgb_port_detach(device_t); 161 162static device_method_t cxgb_port_methods[] = { 163 DEVMETHOD(device_probe, cxgb_port_probe), 164 DEVMETHOD(device_attach, cxgb_port_attach), 165 DEVMETHOD(device_detach, cxgb_port_detach), 166 { 0, 0 } 167}; 168 169static driver_t cxgb_port_driver = { 170 "cxgb", 171 cxgb_port_methods, 172 0 173}; 174 175static d_ioctl_t cxgb_extension_ioctl; 176static d_open_t cxgb_extension_open; 177static d_close_t cxgb_extension_close; 178 179static struct cdevsw cxgb_cdevsw = { 180 .d_version = D_VERSION, 181 .d_flags = 0, 182 .d_open = cxgb_extension_open, 183 .d_close = cxgb_extension_close, 184 .d_ioctl = cxgb_extension_ioctl, 185 .d_name = "cxgb", 186}; 187 188static devclass_t cxgb_port_devclass; 189DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0); 190MODULE_VERSION(cxgb, 1); 191 192static struct mtx t3_list_lock; 193static SLIST_HEAD(, adapter) t3_list; 194#ifdef TCP_OFFLOAD 195static struct mtx t3_uld_list_lock; 196static SLIST_HEAD(, uld_info) t3_uld_list; 197#endif 198 199/* 200 * The driver uses the best interrupt scheme available on a platform in the 201 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which 202 * of these schemes the driver may consider as follows: 203 * 204 * msi = 2: choose from among all three options 205 * msi = 1 : only consider MSI and pin interrupts 206 * msi = 0: force pin interrupts 207 */ 208static int msi_allowed = 2; 209 210TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed); 211SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters"); 212SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0, 213 "MSI-X, MSI, INTx selector"); 214 215/* 216 * The driver uses an auto-queue algorithm by default. 217 * To disable it and force a single queue-set per port, use multiq = 0 218 */ 219static int multiq = 1; 220TUNABLE_INT("hw.cxgb.multiq", &multiq); 221SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0, 222 "use min(ncpus/ports, 8) queue-sets per port"); 223 224/* 225 * By default the driver will not update the firmware unless 226 * it was compiled against a newer version 227 * 228 */ 229static int force_fw_update = 0; 230TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update); 231SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, 232 "update firmware even if up to date"); 233 234int cxgb_use_16k_clusters = -1; 235TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters); 236SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, 237 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); 238 239static int nfilters = -1; 240TUNABLE_INT("hw.cxgb.nfilters", &nfilters); 241SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN, 242 &nfilters, 0, "max number of entries in the filter table"); 243 244enum { 245 MAX_TXQ_ENTRIES = 16384, 246 MAX_CTRL_TXQ_ENTRIES = 1024, 247 MAX_RSPQ_ENTRIES = 16384, 248 MAX_RX_BUFFERS = 16384, 249 MAX_RX_JUMBO_BUFFERS = 16384, 250 MIN_TXQ_ENTRIES = 4, 251 MIN_CTRL_TXQ_ENTRIES = 4, 252 MIN_RSPQ_ENTRIES = 32, 253 MIN_FL_ENTRIES = 32, 254 MIN_FL_JUMBO_ENTRIES = 32 255}; 256 257struct filter_info { 258 u32 sip; 259 u32 sip_mask; 260 u32 dip; 261 u16 sport; 262 u16 dport; 263 u32 vlan:12; 264 u32 vlan_prio:3; 265 u32 mac_hit:1; 266 u32 mac_idx:4; 267 u32 mac_vld:1; 268 u32 pkt_type:2; 269 u32 report_filter_id:1; 270 u32 pass:1; 271 u32 rss:1; 272 u32 qset:3; 273 u32 locked:1; 274 u32 valid:1; 275}; 276 277enum { FILTER_NO_VLAN_PRI = 7 }; 278 279#define EEPROM_MAGIC 0x38E2F10C 280 281#define PORT_MASK ((1 << MAX_NPORTS) - 1) 282 283/* Table for probing the cards. The desc field isn't actually used */ 284struct cxgb_ident { 285 uint16_t vendor; 286 uint16_t device; 287 int index; 288 char *desc; 289} cxgb_identifiers[] = { 290 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"}, 291 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"}, 292 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"}, 293 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"}, 294 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"}, 295 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"}, 296 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"}, 297 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"}, 298 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"}, 299 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"}, 300 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"}, 301 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"}, 302 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"}, 303 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"}, 304 {0, 0, 0, NULL} 305}; 306 307static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); 308 309 310static __inline char 311t3rev2char(struct adapter *adapter) 312{ 313 char rev = 'z'; 314 315 switch(adapter->params.rev) { 316 case T3_REV_A: 317 rev = 'a'; 318 break; 319 case T3_REV_B: 320 case T3_REV_B2: 321 rev = 'b'; 322 break; 323 case T3_REV_C: 324 rev = 'c'; 325 break; 326 } 327 return rev; 328} 329 330static struct cxgb_ident * 331cxgb_get_ident(device_t dev) 332{ 333 struct cxgb_ident *id; 334 335 for (id = cxgb_identifiers; id->desc != NULL; id++) { 336 if ((id->vendor == pci_get_vendor(dev)) && 337 (id->device == pci_get_device(dev))) { 338 return (id); 339 } 340 } 341 return (NULL); 342} 343 344static const struct adapter_info * 345cxgb_get_adapter_info(device_t dev) 346{ 347 struct cxgb_ident *id; 348 const struct adapter_info *ai; 349 350 id = cxgb_get_ident(dev); 351 if (id == NULL) 352 return (NULL); 353 354 ai = t3_get_adapter_info(id->index); 355 356 return (ai); 357} 358 359static int 360cxgb_controller_probe(device_t dev) 361{ 362 const struct adapter_info *ai; 363 char *ports, buf[80]; 364 int nports; 365 366 ai = cxgb_get_adapter_info(dev); 367 if (ai == NULL) 368 return (ENXIO); 369 370 nports = ai->nports0 + ai->nports1; 371 if (nports == 1) 372 ports = "port"; 373 else 374 ports = "ports"; 375 376 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports); 377 device_set_desc_copy(dev, buf); 378 return (BUS_PROBE_DEFAULT); 379} 380 381#define FW_FNAME "cxgb_t3fw" 382#define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom" 383#define TPSRAM_NAME "cxgb_t3%c_protocol_sram" 384 385static int 386upgrade_fw(adapter_t *sc) 387{ 388 const struct firmware *fw; 389 int status; 390 u32 vers; 391 392 if ((fw = firmware_get(FW_FNAME)) == NULL) { 393 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME); 394 return (ENOENT); 395 } else 396 device_printf(sc->dev, "installing firmware on card\n"); 397 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize); 398 399 if (status != 0) { 400 device_printf(sc->dev, "failed to install firmware: %d\n", 401 status); 402 } else { 403 t3_get_fw_version(sc, &vers); 404 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 405 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 406 G_FW_VERSION_MICRO(vers)); 407 } 408 409 firmware_put(fw, FIRMWARE_UNLOAD); 410 411 return (status); 412} 413 414/* 415 * The cxgb_controller_attach function is responsible for the initial 416 * bringup of the device. Its responsibilities include: 417 * 418 * 1. Determine if the device supports MSI or MSI-X. 419 * 2. Allocate bus resources so that we can access the Base Address Register 420 * 3. Create and initialize mutexes for the controller and its control 421 * logic such as SGE and MDIO. 422 * 4. Call hardware specific setup routine for the adapter as a whole. 423 * 5. Allocate the BAR for doing MSI-X. 424 * 6. Setup the line interrupt iff MSI-X is not supported. 425 * 7. Create the driver's taskq. 426 * 8. Start one task queue service thread. 427 * 9. Check if the firmware and SRAM are up-to-date. They will be 428 * auto-updated later (before FULL_INIT_DONE), if required. 429 * 10. Create a child device for each MAC (port) 430 * 11. Initialize T3 private state. 431 * 12. Trigger the LED 432 * 13. Setup offload iff supported. 433 * 14. Reset/restart the tick callout. 434 * 15. Attach sysctls 435 * 436 * NOTE: Any modification or deviation from this list MUST be reflected in 437 * the above comment. Failure to do so will result in problems on various 438 * error conditions including link flapping. 439 */ 440static int 441cxgb_controller_attach(device_t dev) 442{ 443 device_t child; 444 const struct adapter_info *ai; 445 struct adapter *sc; 446 int i, error = 0; 447 uint32_t vers; 448 int port_qsets = 1; 449 int msi_needed, reg; 450 char buf[80]; 451 452 sc = device_get_softc(dev); 453 sc->dev = dev; 454 sc->msi_count = 0; 455 ai = cxgb_get_adapter_info(dev); 456 457 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d", 458 device_get_unit(dev)); 459 ADAPTER_LOCK_INIT(sc, sc->lockbuf); 460 461 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d", 462 device_get_unit(dev)); 463 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d", 464 device_get_unit(dev)); 465 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d", 466 device_get_unit(dev)); 467 468 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN); 469 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF); 470 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF); 471 472 mtx_lock(&t3_list_lock); 473 SLIST_INSERT_HEAD(&t3_list, sc, link); 474 mtx_unlock(&t3_list_lock); 475 476 /* find the PCIe link width and set max read request to 4KB*/ 477 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) { 478 uint16_t lnk; 479 480 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2); 481 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4; 482 if (sc->link_width < 8 && 483 (ai->caps & SUPPORTED_10000baseT_Full)) { 484 device_printf(sc->dev, 485 "PCIe x%d Link, expect reduced performance\n", 486 sc->link_width); 487 } 488 489 pci_set_max_read_req(dev, 4096); 490 } 491 492 touch_bars(dev); 493 pci_enable_busmaster(dev); 494 /* 495 * Allocate the registers and make them available to the driver. 496 * The registers that we care about for NIC mode are in BAR 0 497 */ 498 sc->regs_rid = PCIR_BAR(0); 499 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 500 &sc->regs_rid, RF_ACTIVE)) == NULL) { 501 device_printf(dev, "Cannot allocate BAR region 0\n"); 502 error = ENXIO; 503 goto out; 504 } 505 506 sc->bt = rman_get_bustag(sc->regs_res); 507 sc->bh = rman_get_bushandle(sc->regs_res); 508 sc->mmio_len = rman_get_size(sc->regs_res); 509 510 for (i = 0; i < MAX_NPORTS; i++) 511 sc->port[i].adapter = sc; 512 513 if (t3_prep_adapter(sc, ai, 1) < 0) { 514 printf("prep adapter failed\n"); 515 error = ENODEV; 516 goto out; 517 } 518 519 sc->udbs_rid = PCIR_BAR(2); 520 sc->udbs_res = NULL; 521 if (is_offload(sc) && 522 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 523 &sc->udbs_rid, RF_ACTIVE)) == NULL)) { 524 device_printf(dev, "Cannot allocate BAR region 1\n"); 525 error = ENXIO; 526 goto out; 527 } 528 529 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate 530 * enough messages for the queue sets. If that fails, try falling 531 * back to MSI. If that fails, then try falling back to the legacy 532 * interrupt pin model. 533 */ 534 sc->msix_regs_rid = 0x20; 535 if ((msi_allowed >= 2) && 536 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 537 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) { 538 539 if (multiq) 540 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus); 541 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1; 542 543 if (pci_msix_count(dev) == 0 || 544 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 || 545 sc->msi_count != msi_needed) { 546 device_printf(dev, "alloc msix failed - " 547 "msi_count=%d, msi_needed=%d, err=%d; " 548 "will try MSI\n", sc->msi_count, 549 msi_needed, error); 550 sc->msi_count = 0; 551 port_qsets = 1; 552 pci_release_msi(dev); 553 bus_release_resource(dev, SYS_RES_MEMORY, 554 sc->msix_regs_rid, sc->msix_regs_res); 555 sc->msix_regs_res = NULL; 556 } else { 557 sc->flags |= USING_MSIX; 558 sc->cxgb_intr = cxgb_async_intr; 559 device_printf(dev, 560 "using MSI-X interrupts (%u vectors)\n", 561 sc->msi_count); 562 } 563 } 564 565 if ((msi_allowed >= 1) && (sc->msi_count == 0)) { 566 sc->msi_count = 1; 567 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) { 568 device_printf(dev, "alloc msi failed - " 569 "err=%d; will try INTx\n", error); 570 sc->msi_count = 0; 571 port_qsets = 1; 572 pci_release_msi(dev); 573 } else { 574 sc->flags |= USING_MSI; 575 sc->cxgb_intr = t3_intr_msi; 576 device_printf(dev, "using MSI interrupts\n"); 577 } 578 } 579 if (sc->msi_count == 0) { 580 device_printf(dev, "using line interrupts\n"); 581 sc->cxgb_intr = t3b_intr; 582 } 583 584 /* Create a private taskqueue thread for handling driver events */ 585 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT, 586 taskqueue_thread_enqueue, &sc->tq); 587 if (sc->tq == NULL) { 588 device_printf(dev, "failed to allocate controller task queue\n"); 589 goto out; 590 } 591 592 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", 593 device_get_nameunit(dev)); 594 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc); 595 596 597 /* Create a periodic callout for checking adapter status */ 598 callout_init(&sc->cxgb_tick_ch, TRUE); 599 600 if (t3_check_fw_version(sc) < 0 || force_fw_update) { 601 /* 602 * Warn user that a firmware update will be attempted in init. 603 */ 604 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n", 605 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); 606 sc->flags &= ~FW_UPTODATE; 607 } else { 608 sc->flags |= FW_UPTODATE; 609 } 610 611 if (t3_check_tpsram_version(sc) < 0) { 612 /* 613 * Warn user that a firmware update will be attempted in init. 614 */ 615 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n", 616 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 617 sc->flags &= ~TPS_UPTODATE; 618 } else { 619 sc->flags |= TPS_UPTODATE; 620 } 621 622 /* 623 * Create a child device for each MAC. The ethernet attachment 624 * will be done in these children. 625 */ 626 for (i = 0; i < (sc)->params.nports; i++) { 627 struct port_info *pi; 628 629 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) { 630 device_printf(dev, "failed to add child port\n"); 631 error = EINVAL; 632 goto out; 633 } 634 pi = &sc->port[i]; 635 pi->adapter = sc; 636 pi->nqsets = port_qsets; 637 pi->first_qset = i*port_qsets; 638 pi->port_id = i; 639 pi->tx_chan = i >= ai->nports0; 640 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i; 641 sc->rxpkt_map[pi->txpkt_intf] = i; 642 sc->port[i].tx_chan = i >= ai->nports0; 643 sc->portdev[i] = child; 644 device_set_softc(child, pi); 645 } 646 if ((error = bus_generic_attach(dev)) != 0) 647 goto out; 648 649 /* initialize sge private state */ 650 t3_sge_init_adapter(sc); 651 652 t3_led_ready(sc); 653 654 error = t3_get_fw_version(sc, &vers); 655 if (error) 656 goto out; 657 658 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d", 659 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers), 660 G_FW_VERSION_MICRO(vers)); 661 662 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s", 663 ai->desc, is_offload(sc) ? "R" : "", 664 sc->params.vpd.ec, sc->params.vpd.sn); 665 device_set_desc_copy(dev, buf); 666 667 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x", 668 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1], 669 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]); 670 671 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]); 672 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 673 t3_add_attach_sysctls(sc); 674 675#ifdef TCP_OFFLOAD 676 for (i = 0; i < NUM_CPL_HANDLERS; i++) 677 sc->cpl_handler[i] = cpl_not_handled; 678#endif 679 680 t3_intr_clear(sc); 681 error = cxgb_setup_interrupts(sc); 682out: 683 if (error) 684 cxgb_free(sc); 685 686 return (error); 687} 688 689/* 690 * The cxgb_controller_detach routine is called with the device is 691 * unloaded from the system. 692 */ 693 694static int 695cxgb_controller_detach(device_t dev) 696{ 697 struct adapter *sc; 698 699 sc = device_get_softc(dev); 700 701 cxgb_free(sc); 702 703 return (0); 704} 705 706/* 707 * The cxgb_free() is called by the cxgb_controller_detach() routine 708 * to tear down the structures that were built up in 709 * cxgb_controller_attach(), and should be the final piece of work 710 * done when fully unloading the driver. 711 * 712 * 713 * 1. Shutting down the threads started by the cxgb_controller_attach() 714 * routine. 715 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()). 716 * 3. Detaching all of the port devices created during the 717 * cxgb_controller_attach() routine. 718 * 4. Removing the device children created via cxgb_controller_attach(). 719 * 5. Releasing PCI resources associated with the device. 720 * 6. Turning off the offload support, iff it was turned on. 721 * 7. Destroying the mutexes created in cxgb_controller_attach(). 722 * 723 */ 724static void 725cxgb_free(struct adapter *sc) 726{ 727 int i, nqsets = 0; 728 729 ADAPTER_LOCK(sc); 730 sc->flags |= CXGB_SHUTDOWN; 731 ADAPTER_UNLOCK(sc); 732 733 /* 734 * Make sure all child devices are gone. 735 */ 736 bus_generic_detach(sc->dev); 737 for (i = 0; i < (sc)->params.nports; i++) { 738 if (sc->portdev[i] && 739 device_delete_child(sc->dev, sc->portdev[i]) != 0) 740 device_printf(sc->dev, "failed to delete child port\n"); 741 nqsets += sc->port[i].nqsets; 742 } 743 744 /* 745 * At this point, it is as if cxgb_port_detach has run on all ports, and 746 * cxgb_down has run on the adapter. All interrupts have been silenced, 747 * all open devices have been closed. 748 */ 749 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)", 750 __func__, sc->open_device_map)); 751 for (i = 0; i < sc->params.nports; i++) { 752 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!", 753 __func__, i)); 754 } 755 756 /* 757 * Finish off the adapter's callouts. 758 */ 759 callout_drain(&sc->cxgb_tick_ch); 760 callout_drain(&sc->sge_timer_ch); 761 762 /* 763 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The 764 * sysctls are cleaned up by the kernel linker. 765 */ 766 if (sc->flags & FULL_INIT_DONE) { 767 t3_free_sge_resources(sc, nqsets); 768 sc->flags &= ~FULL_INIT_DONE; 769 } 770 771 /* 772 * Release all interrupt resources. 773 */ 774 cxgb_teardown_interrupts(sc); 775 if (sc->flags & (USING_MSI | USING_MSIX)) { 776 device_printf(sc->dev, "releasing msi message(s)\n"); 777 pci_release_msi(sc->dev); 778 } else { 779 device_printf(sc->dev, "no msi message to release\n"); 780 } 781 782 if (sc->msix_regs_res != NULL) { 783 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid, 784 sc->msix_regs_res); 785 } 786 787 /* 788 * Free the adapter's taskqueue. 789 */ 790 if (sc->tq != NULL) { 791 taskqueue_free(sc->tq); 792 sc->tq = NULL; 793 } 794 795 free(sc->filters, M_DEVBUF); 796 t3_sge_free(sc); 797 798 if (sc->udbs_res != NULL) 799 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid, 800 sc->udbs_res); 801 802 if (sc->regs_res != NULL) 803 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid, 804 sc->regs_res); 805 806 MTX_DESTROY(&sc->mdio_lock); 807 MTX_DESTROY(&sc->sge.reg_lock); 808 MTX_DESTROY(&sc->elmer_lock); 809 mtx_lock(&t3_list_lock); 810 SLIST_REMOVE(&t3_list, sc, adapter, link); 811 mtx_unlock(&t3_list_lock); 812 ADAPTER_LOCK_DEINIT(sc); 813} 814 815/** 816 * setup_sge_qsets - configure SGE Tx/Rx/response queues 817 * @sc: the controller softc 818 * 819 * Determines how many sets of SGE queues to use and initializes them. 820 * We support multiple queue sets per port if we have MSI-X, otherwise 821 * just one queue set per port. 822 */ 823static int 824setup_sge_qsets(adapter_t *sc) 825{ 826 int i, j, err, irq_idx = 0, qset_idx = 0; 827 u_int ntxq = SGE_TXQ_PER_SET; 828 829 if ((err = t3_sge_alloc(sc)) != 0) { 830 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err); 831 return (err); 832 } 833 834 if (sc->params.rev > 0 && !(sc->flags & USING_MSI)) 835 irq_idx = -1; 836 837 for (i = 0; i < (sc)->params.nports; i++) { 838 struct port_info *pi = &sc->port[i]; 839 840 for (j = 0; j < pi->nqsets; j++, qset_idx++) { 841 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports, 842 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx, 843 &sc->params.sge.qset[qset_idx], ntxq, pi); 844 if (err) { 845 t3_free_sge_resources(sc, qset_idx); 846 device_printf(sc->dev, 847 "t3_sge_alloc_qset failed with %d\n", err); 848 return (err); 849 } 850 } 851 } 852 853 return (0); 854} 855 856static void 857cxgb_teardown_interrupts(adapter_t *sc) 858{ 859 int i; 860 861 for (i = 0; i < SGE_QSETS; i++) { 862 if (sc->msix_intr_tag[i] == NULL) { 863 864 /* Should have been setup fully or not at all */ 865 KASSERT(sc->msix_irq_res[i] == NULL && 866 sc->msix_irq_rid[i] == 0, 867 ("%s: half-done interrupt (%d).", __func__, i)); 868 869 continue; 870 } 871 872 bus_teardown_intr(sc->dev, sc->msix_irq_res[i], 873 sc->msix_intr_tag[i]); 874 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i], 875 sc->msix_irq_res[i]); 876 877 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL; 878 sc->msix_irq_rid[i] = 0; 879 } 880 881 if (sc->intr_tag) { 882 KASSERT(sc->irq_res != NULL, 883 ("%s: half-done interrupt.", __func__)); 884 885 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag); 886 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 887 sc->irq_res); 888 889 sc->irq_res = sc->intr_tag = NULL; 890 sc->irq_rid = 0; 891 } 892} 893 894static int 895cxgb_setup_interrupts(adapter_t *sc) 896{ 897 struct resource *res; 898 void *tag; 899 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX); 900 901 sc->irq_rid = intr_flag ? 1 : 0; 902 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, 903 RF_SHAREABLE | RF_ACTIVE); 904 if (sc->irq_res == NULL) { 905 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n", 906 intr_flag, sc->irq_rid); 907 err = EINVAL; 908 sc->irq_rid = 0; 909 } else { 910 err = bus_setup_intr(sc->dev, sc->irq_res, 911 INTR_MPSAFE | INTR_TYPE_NET, NULL, 912 sc->cxgb_intr, sc, &sc->intr_tag); 913 914 if (err) { 915 device_printf(sc->dev, 916 "Cannot set up interrupt (%x, %u, %d)\n", 917 intr_flag, sc->irq_rid, err); 918 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, 919 sc->irq_res); 920 sc->irq_res = sc->intr_tag = NULL; 921 sc->irq_rid = 0; 922 } 923 } 924 925 /* That's all for INTx or MSI */ 926 if (!(intr_flag & USING_MSIX) || err) 927 return (err); 928 929 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err"); 930 for (i = 0; i < sc->msi_count - 1; i++) { 931 rid = i + 2; 932 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, 933 RF_SHAREABLE | RF_ACTIVE); 934 if (res == NULL) { 935 device_printf(sc->dev, "Cannot allocate interrupt " 936 "for message %d\n", rid); 937 err = EINVAL; 938 break; 939 } 940 941 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET, 942 NULL, t3_intr_msix, &sc->sge.qs[i], &tag); 943 if (err) { 944 device_printf(sc->dev, "Cannot set up interrupt " 945 "for message %d (%d)\n", rid, err); 946 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); 947 break; 948 } 949 950 sc->msix_irq_rid[i] = rid; 951 sc->msix_irq_res[i] = res; 952 sc->msix_intr_tag[i] = tag; 953 bus_describe_intr(sc->dev, res, tag, "qs%d", i); 954 } 955 956 if (err) 957 cxgb_teardown_interrupts(sc); 958 959 return (err); 960} 961 962 963static int 964cxgb_port_probe(device_t dev) 965{ 966 struct port_info *p; 967 char buf[80]; 968 const char *desc; 969 970 p = device_get_softc(dev); 971 desc = p->phy.desc; 972 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc); 973 device_set_desc_copy(dev, buf); 974 return (0); 975} 976 977 978static int 979cxgb_makedev(struct port_info *pi) 980{ 981 982 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit, 983 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp)); 984 985 if (pi->port_cdev == NULL) 986 return (ENOMEM); 987 988 pi->port_cdev->si_drv1 = (void *)pi; 989 990 return (0); 991} 992 993#define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 994 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 995 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 996#define CXGB_CAP_ENABLE CXGB_CAP 997 998static int 999cxgb_port_attach(device_t dev) 1000{ 1001 struct port_info *p; 1002 struct ifnet *ifp; 1003 int err; 1004 struct adapter *sc; 1005 1006 p = device_get_softc(dev); 1007 sc = p->adapter; 1008 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d", 1009 device_get_unit(device_get_parent(dev)), p->port_id); 1010 PORT_LOCK_INIT(p, p->lockbuf); 1011 1012 callout_init(&p->link_check_ch, 1); 1013 TASK_INIT(&p->link_check_task, 0, check_link_status, p); 1014 1015 /* Allocate an ifnet object and set it up */ 1016 ifp = p->ifp = if_alloc(IFT_ETHER); 1017 if (ifp == NULL) { 1018 device_printf(dev, "Cannot allocate ifnet\n"); 1019 return (ENOMEM); 1020 } 1021 1022 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1023 ifp->if_init = cxgb_init; 1024 ifp->if_softc = p; 1025 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1026 ifp->if_ioctl = cxgb_ioctl; 1027 ifp->if_transmit = cxgb_transmit; 1028 ifp->if_qflush = cxgb_qflush; 1029 1030 ifp->if_capabilities = CXGB_CAP; 1031#ifdef TCP_OFFLOAD 1032 if (is_offload(sc)) 1033 ifp->if_capabilities |= IFCAP_TOE4; 1034#endif 1035 ifp->if_capenable = CXGB_CAP_ENABLE; 1036 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1037 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1038 1039 /* 1040 * Disable TSO on 4-port - it isn't supported by the firmware. 1041 */ 1042 if (sc->params.nports > 2) { 1043 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1044 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO); 1045 ifp->if_hwassist &= ~CSUM_TSO; 1046 } 1047 1048 ether_ifattach(ifp, p->hw_addr); 1049 1050#ifdef DEFAULT_JUMBO 1051 if (sc->params.nports <= 2) 1052 ifp->if_mtu = ETHERMTU_JUMBO; 1053#endif 1054 if ((err = cxgb_makedev(p)) != 0) { 1055 printf("makedev failed %d\n", err); 1056 return (err); 1057 } 1058 1059 /* Create a list of media supported by this port */ 1060 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change, 1061 cxgb_media_status); 1062 cxgb_build_medialist(p); 1063 1064 t3_sge_init_port(p); 1065 1066 return (err); 1067} 1068 1069/* 1070 * cxgb_port_detach() is called via the device_detach methods when 1071 * cxgb_free() calls the bus_generic_detach. It is responsible for 1072 * removing the device from the view of the kernel, i.e. from all 1073 * interfaces lists etc. This routine is only called when the driver is 1074 * being unloaded, not when the link goes down. 1075 */ 1076static int 1077cxgb_port_detach(device_t dev) 1078{ 1079 struct port_info *p; 1080 struct adapter *sc; 1081 int i; 1082 1083 p = device_get_softc(dev); 1084 sc = p->adapter; 1085 1086 /* Tell cxgb_ioctl and if_init that the port is going away */ 1087 ADAPTER_LOCK(sc); 1088 SET_DOOMED(p); 1089 wakeup(&sc->flags); 1090 while (IS_BUSY(sc)) 1091 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0); 1092 SET_BUSY(sc); 1093 ADAPTER_UNLOCK(sc); 1094 1095 if (p->port_cdev != NULL) 1096 destroy_dev(p->port_cdev); 1097 1098 cxgb_uninit_synchronized(p); 1099 ether_ifdetach(p->ifp); 1100 1101 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1102 struct sge_qset *qs = &sc->sge.qs[i]; 1103 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1104 1105 callout_drain(&txq->txq_watchdog); 1106 callout_drain(&txq->txq_timer); 1107 } 1108 1109 PORT_LOCK_DEINIT(p); 1110 if_free(p->ifp); 1111 p->ifp = NULL; 1112 1113 ADAPTER_LOCK(sc); 1114 CLR_BUSY(sc); 1115 wakeup_one(&sc->flags); 1116 ADAPTER_UNLOCK(sc); 1117 return (0); 1118} 1119 1120void 1121t3_fatal_err(struct adapter *sc) 1122{ 1123 u_int fw_status[4]; 1124 1125 if (sc->flags & FULL_INIT_DONE) { 1126 t3_sge_stop(sc); 1127 t3_write_reg(sc, A_XGM_TX_CTRL, 0); 1128 t3_write_reg(sc, A_XGM_RX_CTRL, 0); 1129 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0); 1130 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0); 1131 t3_intr_disable(sc); 1132 } 1133 device_printf(sc->dev,"encountered fatal error, operation suspended\n"); 1134 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) 1135 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n", 1136 fw_status[0], fw_status[1], fw_status[2], fw_status[3]); 1137} 1138 1139int 1140t3_os_find_pci_capability(adapter_t *sc, int cap) 1141{ 1142 device_t dev; 1143 struct pci_devinfo *dinfo; 1144 pcicfgregs *cfg; 1145 uint32_t status; 1146 uint8_t ptr; 1147 1148 dev = sc->dev; 1149 dinfo = device_get_ivars(dev); 1150 cfg = &dinfo->cfg; 1151 1152 status = pci_read_config(dev, PCIR_STATUS, 2); 1153 if (!(status & PCIM_STATUS_CAPPRESENT)) 1154 return (0); 1155 1156 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1157 case 0: 1158 case 1: 1159 ptr = PCIR_CAP_PTR; 1160 break; 1161 case 2: 1162 ptr = PCIR_CAP_PTR_2; 1163 break; 1164 default: 1165 return (0); 1166 break; 1167 } 1168 ptr = pci_read_config(dev, ptr, 1); 1169 1170 while (ptr != 0) { 1171 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 1172 return (ptr); 1173 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 1174 } 1175 1176 return (0); 1177} 1178 1179int 1180t3_os_pci_save_state(struct adapter *sc) 1181{ 1182 device_t dev; 1183 struct pci_devinfo *dinfo; 1184 1185 dev = sc->dev; 1186 dinfo = device_get_ivars(dev); 1187 1188 pci_cfg_save(dev, dinfo, 0); 1189 return (0); 1190} 1191 1192int 1193t3_os_pci_restore_state(struct adapter *sc) 1194{ 1195 device_t dev; 1196 struct pci_devinfo *dinfo; 1197 1198 dev = sc->dev; 1199 dinfo = device_get_ivars(dev); 1200 1201 pci_cfg_restore(dev, dinfo); 1202 return (0); 1203} 1204 1205/** 1206 * t3_os_link_changed - handle link status changes 1207 * @sc: the adapter associated with the link change 1208 * @port_id: the port index whose link status has changed 1209 * @link_status: the new status of the link 1210 * @speed: the new speed setting 1211 * @duplex: the new duplex setting 1212 * @fc: the new flow-control setting 1213 * 1214 * This is the OS-dependent handler for link status changes. The OS 1215 * neutral handler takes care of most of the processing for these events, 1216 * then calls this handler for any OS-specific processing. 1217 */ 1218void 1219t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed, 1220 int duplex, int fc, int mac_was_reset) 1221{ 1222 struct port_info *pi = &adapter->port[port_id]; 1223 struct ifnet *ifp = pi->ifp; 1224 1225 /* no race with detach, so ifp should always be good */ 1226 KASSERT(ifp, ("%s: if detached.", __func__)); 1227 1228 /* Reapply mac settings if they were lost due to a reset */ 1229 if (mac_was_reset) { 1230 PORT_LOCK(pi); 1231 cxgb_update_mac_settings(pi); 1232 PORT_UNLOCK(pi); 1233 } 1234 1235 if (link_status) { 1236 ifp->if_baudrate = IF_Mbps(speed); 1237 if_link_state_change(ifp, LINK_STATE_UP); 1238 } else 1239 if_link_state_change(ifp, LINK_STATE_DOWN); 1240} 1241 1242/** 1243 * t3_os_phymod_changed - handle PHY module changes 1244 * @phy: the PHY reporting the module change 1245 * @mod_type: new module type 1246 * 1247 * This is the OS-dependent handler for PHY module changes. It is 1248 * invoked when a PHY module is removed or inserted for any OS-specific 1249 * processing. 1250 */ 1251void t3_os_phymod_changed(struct adapter *adap, int port_id) 1252{ 1253 static const char *mod_str[] = { 1254 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown" 1255 }; 1256 struct port_info *pi = &adap->port[port_id]; 1257 int mod = pi->phy.modtype; 1258 1259 if (mod != pi->media.ifm_cur->ifm_data) 1260 cxgb_build_medialist(pi); 1261 1262 if (mod == phy_modtype_none) 1263 if_printf(pi->ifp, "PHY module unplugged\n"); 1264 else { 1265 KASSERT(mod < ARRAY_SIZE(mod_str), 1266 ("invalid PHY module type %d", mod)); 1267 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]); 1268 } 1269} 1270 1271void 1272t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]) 1273{ 1274 1275 /* 1276 * The ifnet might not be allocated before this gets called, 1277 * as this is called early on in attach by t3_prep_adapter 1278 * save the address off in the port structure 1279 */ 1280 if (cxgb_debug) 1281 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":"); 1282 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); 1283} 1284 1285/* 1286 * Programs the XGMAC based on the settings in the ifnet. These settings 1287 * include MTU, MAC address, mcast addresses, etc. 1288 */ 1289static void 1290cxgb_update_mac_settings(struct port_info *p) 1291{ 1292 struct ifnet *ifp = p->ifp; 1293 struct t3_rx_mode rm; 1294 struct cmac *mac = &p->mac; 1295 int mtu, hwtagging; 1296 1297 PORT_LOCK_ASSERT_OWNED(p); 1298 1299 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); 1300 1301 mtu = ifp->if_mtu; 1302 if (ifp->if_capenable & IFCAP_VLAN_MTU) 1303 mtu += ETHER_VLAN_ENCAP_LEN; 1304 1305 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; 1306 1307 t3_mac_set_mtu(mac, mtu); 1308 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); 1309 t3_mac_set_address(mac, 0, p->hw_addr); 1310 t3_init_rx_mode(&rm, p); 1311 t3_mac_set_rx_mode(mac, &rm); 1312} 1313 1314 1315static int 1316await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, 1317 unsigned long n) 1318{ 1319 int attempts = 5; 1320 1321 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { 1322 if (!--attempts) 1323 return (ETIMEDOUT); 1324 t3_os_sleep(10); 1325 } 1326 return 0; 1327} 1328 1329static int 1330init_tp_parity(struct adapter *adap) 1331{ 1332 int i; 1333 struct mbuf *m; 1334 struct cpl_set_tcb_field *greq; 1335 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; 1336 1337 t3_tp_set_offload_mode(adap, 1); 1338 1339 for (i = 0; i < 16; i++) { 1340 struct cpl_smt_write_req *req; 1341 1342 m = m_gethdr(M_WAITOK, MT_DATA); 1343 req = mtod(m, struct cpl_smt_write_req *); 1344 m->m_len = m->m_pkthdr.len = sizeof(*req); 1345 memset(req, 0, sizeof(*req)); 1346 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1347 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); 1348 req->iff = i; 1349 t3_mgmt_tx(adap, m); 1350 } 1351 1352 for (i = 0; i < 2048; i++) { 1353 struct cpl_l2t_write_req *req; 1354 1355 m = m_gethdr(M_WAITOK, MT_DATA); 1356 req = mtod(m, struct cpl_l2t_write_req *); 1357 m->m_len = m->m_pkthdr.len = sizeof(*req); 1358 memset(req, 0, sizeof(*req)); 1359 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1360 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); 1361 req->params = htonl(V_L2T_W_IDX(i)); 1362 t3_mgmt_tx(adap, m); 1363 } 1364 1365 for (i = 0; i < 2048; i++) { 1366 struct cpl_rte_write_req *req; 1367 1368 m = m_gethdr(M_WAITOK, MT_DATA); 1369 req = mtod(m, struct cpl_rte_write_req *); 1370 m->m_len = m->m_pkthdr.len = sizeof(*req); 1371 memset(req, 0, sizeof(*req)); 1372 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1373 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); 1374 req->l2t_idx = htonl(V_L2T_W_IDX(i)); 1375 t3_mgmt_tx(adap, m); 1376 } 1377 1378 m = m_gethdr(M_WAITOK, MT_DATA); 1379 greq = mtod(m, struct cpl_set_tcb_field *); 1380 m->m_len = m->m_pkthdr.len = sizeof(*greq); 1381 memset(greq, 0, sizeof(*greq)); 1382 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1383 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); 1384 greq->mask = htobe64(1); 1385 t3_mgmt_tx(adap, m); 1386 1387 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); 1388 t3_tp_set_offload_mode(adap, 0); 1389 return (i); 1390} 1391 1392/** 1393 * setup_rss - configure Receive Side Steering (per-queue connection demux) 1394 * @adap: the adapter 1395 * 1396 * Sets up RSS to distribute packets to multiple receive queues. We 1397 * configure the RSS CPU lookup table to distribute to the number of HW 1398 * receive queues, and the response queue lookup table to narrow that 1399 * down to the response queues actually configured for each port. 1400 * We always configure the RSS mapping for two ports since the mapping 1401 * table has plenty of entries. 1402 */ 1403static void 1404setup_rss(adapter_t *adap) 1405{ 1406 int i; 1407 u_int nq[2]; 1408 uint8_t cpus[SGE_QSETS + 1]; 1409 uint16_t rspq_map[RSS_TABLE_SIZE]; 1410 1411 for (i = 0; i < SGE_QSETS; ++i) 1412 cpus[i] = i; 1413 cpus[SGE_QSETS] = 0xff; 1414 1415 nq[0] = nq[1] = 0; 1416 for_each_port(adap, i) { 1417 const struct port_info *pi = adap2pinfo(adap, i); 1418 1419 nq[pi->tx_chan] += pi->nqsets; 1420 } 1421 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { 1422 rspq_map[i] = nq[0] ? i % nq[0] : 0; 1423 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0; 1424 } 1425 1426 /* Calculate the reverse RSS map table */ 1427 for (i = 0; i < SGE_QSETS; ++i) 1428 adap->rrss_map[i] = 0xff; 1429 for (i = 0; i < RSS_TABLE_SIZE; ++i) 1430 if (adap->rrss_map[rspq_map[i]] == 0xff) 1431 adap->rrss_map[rspq_map[i]] = i; 1432 1433 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | 1434 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN | 1435 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, 1436 cpus, rspq_map); 1437 1438} 1439static void 1440send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, 1441 int hi, int port) 1442{ 1443 struct mbuf *m; 1444 struct mngt_pktsched_wr *req; 1445 1446 m = m_gethdr(M_NOWAIT, MT_DATA); 1447 if (m) { 1448 req = mtod(m, struct mngt_pktsched_wr *); 1449 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); 1450 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; 1451 req->sched = sched; 1452 req->idx = qidx; 1453 req->min = lo; 1454 req->max = hi; 1455 req->binding = port; 1456 m->m_len = m->m_pkthdr.len = sizeof(*req); 1457 t3_mgmt_tx(adap, m); 1458 } 1459} 1460 1461static void 1462bind_qsets(adapter_t *sc) 1463{ 1464 int i, j; 1465 1466 for (i = 0; i < (sc)->params.nports; ++i) { 1467 const struct port_info *pi = adap2pinfo(sc, i); 1468 1469 for (j = 0; j < pi->nqsets; ++j) { 1470 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1, 1471 -1, pi->tx_chan); 1472 1473 } 1474 } 1475} 1476 1477static void 1478update_tpeeprom(struct adapter *adap) 1479{ 1480 const struct firmware *tpeeprom; 1481 1482 uint32_t version; 1483 unsigned int major, minor; 1484 int ret, len; 1485 char rev, name[32]; 1486 1487 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version); 1488 1489 major = G_TP_VERSION_MAJOR(version); 1490 minor = G_TP_VERSION_MINOR(version); 1491 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 1492 return; 1493 1494 rev = t3rev2char(adap); 1495 snprintf(name, sizeof(name), TPEEPROM_NAME, rev); 1496 1497 tpeeprom = firmware_get(name); 1498 if (tpeeprom == NULL) { 1499 device_printf(adap->dev, 1500 "could not load TP EEPROM: unable to load %s\n", 1501 name); 1502 return; 1503 } 1504 1505 len = tpeeprom->datasize - 4; 1506 1507 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize); 1508 if (ret) 1509 goto release_tpeeprom; 1510 1511 if (len != TP_SRAM_LEN) { 1512 device_printf(adap->dev, 1513 "%s length is wrong len=%d expected=%d\n", name, 1514 len, TP_SRAM_LEN); 1515 return; 1516 } 1517 1518 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize, 1519 TP_SRAM_OFFSET); 1520 1521 if (!ret) { 1522 device_printf(adap->dev, 1523 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n", 1524 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); 1525 } else 1526 device_printf(adap->dev, 1527 "Protocol SRAM image update in EEPROM failed\n"); 1528 1529release_tpeeprom: 1530 firmware_put(tpeeprom, FIRMWARE_UNLOAD); 1531 1532 return; 1533} 1534 1535static int 1536update_tpsram(struct adapter *adap) 1537{ 1538 const struct firmware *tpsram; 1539 int ret; 1540 char rev, name[32]; 1541 1542 rev = t3rev2char(adap); 1543 snprintf(name, sizeof(name), TPSRAM_NAME, rev); 1544 1545 update_tpeeprom(adap); 1546 1547 tpsram = firmware_get(name); 1548 if (tpsram == NULL){ 1549 device_printf(adap->dev, "could not load TP SRAM\n"); 1550 return (EINVAL); 1551 } else 1552 device_printf(adap->dev, "updating TP SRAM\n"); 1553 1554 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize); 1555 if (ret) 1556 goto release_tpsram; 1557 1558 ret = t3_set_proto_sram(adap, tpsram->data); 1559 if (ret) 1560 device_printf(adap->dev, "loading protocol SRAM failed\n"); 1561 1562release_tpsram: 1563 firmware_put(tpsram, FIRMWARE_UNLOAD); 1564 1565 return ret; 1566} 1567 1568/** 1569 * cxgb_up - enable the adapter 1570 * @adap: adapter being enabled 1571 * 1572 * Called when the first port is enabled, this function performs the 1573 * actions necessary to make an adapter operational, such as completing 1574 * the initialization of HW modules, and enabling interrupts. 1575 */ 1576static int 1577cxgb_up(struct adapter *sc) 1578{ 1579 int err = 0; 1580 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS; 1581 1582 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)", 1583 __func__, sc->open_device_map)); 1584 1585 if ((sc->flags & FULL_INIT_DONE) == 0) { 1586 1587 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1588 1589 if ((sc->flags & FW_UPTODATE) == 0) 1590 if ((err = upgrade_fw(sc))) 1591 goto out; 1592 1593 if ((sc->flags & TPS_UPTODATE) == 0) 1594 if ((err = update_tpsram(sc))) 1595 goto out; 1596 1597 if (is_offload(sc) && nfilters != 0) { 1598 sc->params.mc5.nservers = 0; 1599 1600 if (nfilters < 0) 1601 sc->params.mc5.nfilters = mxf; 1602 else 1603 sc->params.mc5.nfilters = min(nfilters, mxf); 1604 } 1605 1606 err = t3_init_hw(sc, 0); 1607 if (err) 1608 goto out; 1609 1610 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); 1611 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 1612 1613 err = setup_sge_qsets(sc); 1614 if (err) 1615 goto out; 1616 1617 alloc_filters(sc); 1618 setup_rss(sc); 1619 1620 t3_add_configured_sysctls(sc); 1621 sc->flags |= FULL_INIT_DONE; 1622 } 1623 1624 t3_intr_clear(sc); 1625 t3_sge_start(sc); 1626 t3_intr_enable(sc); 1627 1628 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) && 1629 is_offload(sc) && init_tp_parity(sc) == 0) 1630 sc->flags |= TP_PARITY_INIT; 1631 1632 if (sc->flags & TP_PARITY_INIT) { 1633 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR); 1634 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); 1635 } 1636 1637 if (!(sc->flags & QUEUES_BOUND)) { 1638 bind_qsets(sc); 1639 setup_hw_filters(sc); 1640 sc->flags |= QUEUES_BOUND; 1641 } 1642 1643 t3_sge_reset_adapter(sc); 1644out: 1645 return (err); 1646} 1647 1648/* 1649 * Called when the last open device is closed. Does NOT undo all of cxgb_up's 1650 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released 1651 * during controller_detach, not here. 1652 */ 1653static void 1654cxgb_down(struct adapter *sc) 1655{ 1656 t3_sge_stop(sc); 1657 t3_intr_disable(sc); 1658} 1659 1660/* 1661 * if_init for cxgb ports. 1662 */ 1663static void 1664cxgb_init(void *arg) 1665{ 1666 struct port_info *p = arg; 1667 struct adapter *sc = p->adapter; 1668 1669 ADAPTER_LOCK(sc); 1670 cxgb_init_locked(p); /* releases adapter lock */ 1671 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1672} 1673 1674static int 1675cxgb_init_locked(struct port_info *p) 1676{ 1677 struct adapter *sc = p->adapter; 1678 struct ifnet *ifp = p->ifp; 1679 struct cmac *mac = &p->mac; 1680 int i, rc = 0, may_sleep = 0, gave_up_lock = 0; 1681 1682 ADAPTER_LOCK_ASSERT_OWNED(sc); 1683 1684 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1685 gave_up_lock = 1; 1686 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) { 1687 rc = EINTR; 1688 goto done; 1689 } 1690 } 1691 if (IS_DOOMED(p)) { 1692 rc = ENXIO; 1693 goto done; 1694 } 1695 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1696 1697 /* 1698 * The code that runs during one-time adapter initialization can sleep 1699 * so it's important not to hold any locks across it. 1700 */ 1701 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1; 1702 1703 if (may_sleep) { 1704 SET_BUSY(sc); 1705 gave_up_lock = 1; 1706 ADAPTER_UNLOCK(sc); 1707 } 1708 1709 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0)) 1710 goto done; 1711 1712 PORT_LOCK(p); 1713 if (isset(&sc->open_device_map, p->port_id) && 1714 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1715 PORT_UNLOCK(p); 1716 goto done; 1717 } 1718 t3_port_intr_enable(sc, p->port_id); 1719 if (!mac->multiport) 1720 t3_mac_init(mac); 1721 cxgb_update_mac_settings(p); 1722 t3_link_start(&p->phy, mac, &p->link_config); 1723 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 1724 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1725 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1726 PORT_UNLOCK(p); 1727 1728 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { 1729 struct sge_qset *qs = &sc->sge.qs[i]; 1730 struct sge_txq *txq = &qs->txq[TXQ_ETH]; 1731 1732 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs, 1733 txq->txq_watchdog.c_cpu); 1734 } 1735 1736 /* all ok */ 1737 setbit(&sc->open_device_map, p->port_id); 1738 callout_reset(&p->link_check_ch, 1739 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4, 1740 link_check_callout, p); 1741 1742done: 1743 if (may_sleep) { 1744 ADAPTER_LOCK(sc); 1745 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1746 CLR_BUSY(sc); 1747 } 1748 if (gave_up_lock) 1749 wakeup_one(&sc->flags); 1750 ADAPTER_UNLOCK(sc); 1751 return (rc); 1752} 1753 1754static int 1755cxgb_uninit_locked(struct port_info *p) 1756{ 1757 struct adapter *sc = p->adapter; 1758 int rc; 1759 1760 ADAPTER_LOCK_ASSERT_OWNED(sc); 1761 1762 while (!IS_DOOMED(p) && IS_BUSY(sc)) { 1763 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) { 1764 rc = EINTR; 1765 goto done; 1766 } 1767 } 1768 if (IS_DOOMED(p)) { 1769 rc = ENXIO; 1770 goto done; 1771 } 1772 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1773 SET_BUSY(sc); 1774 ADAPTER_UNLOCK(sc); 1775 1776 rc = cxgb_uninit_synchronized(p); 1777 1778 ADAPTER_LOCK(sc); 1779 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1780 CLR_BUSY(sc); 1781 wakeup_one(&sc->flags); 1782done: 1783 ADAPTER_UNLOCK(sc); 1784 return (rc); 1785} 1786 1787/* 1788 * Called on "ifconfig down", and from port_detach 1789 */ 1790static int 1791cxgb_uninit_synchronized(struct port_info *pi) 1792{ 1793 struct adapter *sc = pi->adapter; 1794 struct ifnet *ifp = pi->ifp; 1795 1796 /* 1797 * taskqueue_drain may cause a deadlock if the adapter lock is held. 1798 */ 1799 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1800 1801 /* 1802 * Clear this port's bit from the open device map, and then drain all 1803 * the tasks that can access/manipulate this port's port_info or ifp. 1804 * We disable this port's interrupts here and so the slow/ext 1805 * interrupt tasks won't be enqueued. The tick task will continue to 1806 * be enqueued every second but the runs after this drain will not see 1807 * this port in the open device map. 1808 * 1809 * A well behaved task must take open_device_map into account and ignore 1810 * ports that are not open. 1811 */ 1812 clrbit(&sc->open_device_map, pi->port_id); 1813 t3_port_intr_disable(sc, pi->port_id); 1814 taskqueue_drain(sc->tq, &sc->slow_intr_task); 1815 taskqueue_drain(sc->tq, &sc->tick_task); 1816 1817 callout_drain(&pi->link_check_ch); 1818 taskqueue_drain(sc->tq, &pi->link_check_task); 1819 1820 PORT_LOCK(pi); 1821 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1822 1823 /* disable pause frames */ 1824 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0); 1825 1826 /* Reset RX FIFO HWM */ 1827 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset, 1828 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0); 1829 1830 DELAY(100 * 1000); 1831 1832 /* Wait for TXFIFO empty */ 1833 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset, 1834 F_TXFIFO_EMPTY, 1, 20, 5); 1835 1836 DELAY(100 * 1000); 1837 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX); 1838 1839 pi->phy.ops->power_down(&pi->phy, 1); 1840 1841 PORT_UNLOCK(pi); 1842 1843 pi->link_config.link_ok = 0; 1844 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0); 1845 1846 if (sc->open_device_map == 0) 1847 cxgb_down(pi->adapter); 1848 1849 return (0); 1850} 1851 1852/* 1853 * Mark lro enabled or disabled in all qsets for this port 1854 */ 1855static int 1856cxgb_set_lro(struct port_info *p, int enabled) 1857{ 1858 int i; 1859 struct adapter *adp = p->adapter; 1860 struct sge_qset *q; 1861 1862 for (i = 0; i < p->nqsets; i++) { 1863 q = &adp->sge.qs[p->first_qset + i]; 1864 q->lro.enabled = (enabled != 0); 1865 } 1866 return (0); 1867} 1868 1869static int 1870cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) 1871{ 1872 struct port_info *p = ifp->if_softc; 1873 struct adapter *sc = p->adapter; 1874 struct ifreq *ifr = (struct ifreq *)data; 1875 int flags, error = 0, mtu; 1876 uint32_t mask; 1877 1878 switch (command) { 1879 case SIOCSIFMTU: 1880 ADAPTER_LOCK(sc); 1881 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1882 if (error) { 1883fail: 1884 ADAPTER_UNLOCK(sc); 1885 return (error); 1886 } 1887 1888 mtu = ifr->ifr_mtu; 1889 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 1890 error = EINVAL; 1891 } else { 1892 ifp->if_mtu = mtu; 1893 PORT_LOCK(p); 1894 cxgb_update_mac_settings(p); 1895 PORT_UNLOCK(p); 1896 } 1897 ADAPTER_UNLOCK(sc); 1898 break; 1899 case SIOCSIFFLAGS: 1900 ADAPTER_LOCK(sc); 1901 if (IS_DOOMED(p)) { 1902 error = ENXIO; 1903 goto fail; 1904 } 1905 if (ifp->if_flags & IFF_UP) { 1906 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1907 flags = p->if_flags; 1908 if (((ifp->if_flags ^ flags) & IFF_PROMISC) || 1909 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) { 1910 if (IS_BUSY(sc)) { 1911 error = EBUSY; 1912 goto fail; 1913 } 1914 PORT_LOCK(p); 1915 cxgb_update_mac_settings(p); 1916 PORT_UNLOCK(p); 1917 } 1918 ADAPTER_UNLOCK(sc); 1919 } else 1920 error = cxgb_init_locked(p); 1921 p->if_flags = ifp->if_flags; 1922 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1923 error = cxgb_uninit_locked(p); 1924 else 1925 ADAPTER_UNLOCK(sc); 1926 1927 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1928 break; 1929 case SIOCADDMULTI: 1930 case SIOCDELMULTI: 1931 ADAPTER_LOCK(sc); 1932 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1933 if (error) 1934 goto fail; 1935 1936 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1937 PORT_LOCK(p); 1938 cxgb_update_mac_settings(p); 1939 PORT_UNLOCK(p); 1940 } 1941 ADAPTER_UNLOCK(sc); 1942 1943 break; 1944 case SIOCSIFCAP: 1945 ADAPTER_LOCK(sc); 1946 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1947 if (error) 1948 goto fail; 1949 1950 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1951 if (mask & IFCAP_TXCSUM) { 1952 ifp->if_capenable ^= IFCAP_TXCSUM; 1953 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1954 1955 if (IFCAP_TSO4 & ifp->if_capenable && 1956 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1957 ifp->if_capenable &= ~IFCAP_TSO4; 1958 if_printf(ifp, 1959 "tso4 disabled due to -txcsum.\n"); 1960 } 1961 } 1962 if (mask & IFCAP_TXCSUM_IPV6) { 1963 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1964 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1965 1966 if (IFCAP_TSO6 & ifp->if_capenable && 1967 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1968 ifp->if_capenable &= ~IFCAP_TSO6; 1969 if_printf(ifp, 1970 "tso6 disabled due to -txcsum6.\n"); 1971 } 1972 } 1973 if (mask & IFCAP_RXCSUM) 1974 ifp->if_capenable ^= IFCAP_RXCSUM; 1975 if (mask & IFCAP_RXCSUM_IPV6) 1976 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1977 1978 /* 1979 * Note that we leave CSUM_TSO alone (it is always set). The 1980 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1981 * sending a TSO request our way, so it's sufficient to toggle 1982 * IFCAP_TSOx only. 1983 */ 1984 if (mask & IFCAP_TSO4) { 1985 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1986 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1987 if_printf(ifp, "enable txcsum first.\n"); 1988 error = EAGAIN; 1989 goto fail; 1990 } 1991 ifp->if_capenable ^= IFCAP_TSO4; 1992 } 1993 if (mask & IFCAP_TSO6) { 1994 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1995 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1996 if_printf(ifp, "enable txcsum6 first.\n"); 1997 error = EAGAIN; 1998 goto fail; 1999 } 2000 ifp->if_capenable ^= IFCAP_TSO6; 2001 } 2002 if (mask & IFCAP_LRO) { 2003 ifp->if_capenable ^= IFCAP_LRO; 2004 2005 /* Safe to do this even if cxgb_up not called yet */ 2006 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO); 2007 } 2008#ifdef TCP_OFFLOAD 2009 if (mask & IFCAP_TOE4) { 2010 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4; 2011 2012 error = toe_capability(p, enable); 2013 if (error == 0) 2014 ifp->if_capenable ^= mask; 2015 } 2016#endif 2017 if (mask & IFCAP_VLAN_HWTAGGING) { 2018 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2019 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2020 PORT_LOCK(p); 2021 cxgb_update_mac_settings(p); 2022 PORT_UNLOCK(p); 2023 } 2024 } 2025 if (mask & IFCAP_VLAN_MTU) { 2026 ifp->if_capenable ^= IFCAP_VLAN_MTU; 2027 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2028 PORT_LOCK(p); 2029 cxgb_update_mac_settings(p); 2030 PORT_UNLOCK(p); 2031 } 2032 } 2033 if (mask & IFCAP_VLAN_HWTSO) 2034 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 2035 if (mask & IFCAP_VLAN_HWCSUM) 2036 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 2037 2038#ifdef VLAN_CAPABILITIES 2039 VLAN_CAPABILITIES(ifp); 2040#endif 2041 ADAPTER_UNLOCK(sc); 2042 break; 2043 case SIOCSIFMEDIA: 2044 case SIOCGIFMEDIA: 2045 error = ifmedia_ioctl(ifp, ifr, &p->media, command); 2046 break; 2047 default: 2048 error = ether_ioctl(ifp, command, data); 2049 } 2050 2051 return (error); 2052} 2053 2054static int 2055cxgb_media_change(struct ifnet *ifp) 2056{ 2057 return (EOPNOTSUPP); 2058} 2059 2060/* 2061 * Translates phy->modtype to the correct Ethernet media subtype. 2062 */ 2063static int 2064cxgb_ifm_type(int mod) 2065{ 2066 switch (mod) { 2067 case phy_modtype_sr: 2068 return (IFM_10G_SR); 2069 case phy_modtype_lr: 2070 return (IFM_10G_LR); 2071 case phy_modtype_lrm: 2072 return (IFM_10G_LRM); 2073 case phy_modtype_twinax: 2074 return (IFM_10G_TWINAX); 2075 case phy_modtype_twinax_long: 2076 return (IFM_10G_TWINAX_LONG); 2077 case phy_modtype_none: 2078 return (IFM_NONE); 2079 case phy_modtype_unknown: 2080 return (IFM_UNKNOWN); 2081 } 2082 2083 KASSERT(0, ("%s: modtype %d unknown", __func__, mod)); 2084 return (IFM_UNKNOWN); 2085} 2086 2087/* 2088 * Rebuilds the ifmedia list for this port, and sets the current media. 2089 */ 2090static void 2091cxgb_build_medialist(struct port_info *p) 2092{ 2093 struct cphy *phy = &p->phy; 2094 struct ifmedia *media = &p->media; 2095 int mod = phy->modtype; 2096 int m = IFM_ETHER | IFM_FDX; 2097 2098 PORT_LOCK(p); 2099 2100 ifmedia_removeall(media); 2101 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) { 2102 /* Copper (RJ45) */ 2103 2104 if (phy->caps & SUPPORTED_10000baseT_Full) 2105 ifmedia_add(media, m | IFM_10G_T, mod, NULL); 2106 2107 if (phy->caps & SUPPORTED_1000baseT_Full) 2108 ifmedia_add(media, m | IFM_1000_T, mod, NULL); 2109 2110 if (phy->caps & SUPPORTED_100baseT_Full) 2111 ifmedia_add(media, m | IFM_100_TX, mod, NULL); 2112 2113 if (phy->caps & SUPPORTED_10baseT_Full) 2114 ifmedia_add(media, m | IFM_10_T, mod, NULL); 2115 2116 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL); 2117 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2118 2119 } else if (phy->caps & SUPPORTED_TP) { 2120 /* Copper (CX4) */ 2121 2122 KASSERT(phy->caps & SUPPORTED_10000baseT_Full, 2123 ("%s: unexpected cap 0x%x", __func__, phy->caps)); 2124 2125 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL); 2126 ifmedia_set(media, m | IFM_10G_CX4); 2127 2128 } else if (phy->caps & SUPPORTED_FIBRE && 2129 phy->caps & SUPPORTED_10000baseT_Full) { 2130 /* 10G optical (but includes SFP+ twinax) */ 2131 2132 m |= cxgb_ifm_type(mod); 2133 if (IFM_SUBTYPE(m) == IFM_NONE) 2134 m &= ~IFM_FDX; 2135 2136 ifmedia_add(media, m, mod, NULL); 2137 ifmedia_set(media, m); 2138 2139 } else if (phy->caps & SUPPORTED_FIBRE && 2140 phy->caps & SUPPORTED_1000baseT_Full) { 2141 /* 1G optical */ 2142 2143 /* XXX: Lie and claim to be SX, could actually be any 1G-X */ 2144 ifmedia_add(media, m | IFM_1000_SX, mod, NULL); 2145 ifmedia_set(media, m | IFM_1000_SX); 2146 2147 } else { 2148 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__, 2149 phy->caps)); 2150 } 2151 2152 PORT_UNLOCK(p); 2153} 2154 2155static void 2156cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2157{ 2158 struct port_info *p = ifp->if_softc; 2159 struct ifmedia_entry *cur = p->media.ifm_cur; 2160 int speed = p->link_config.speed; 2161 2162 if (cur->ifm_data != p->phy.modtype) { 2163 cxgb_build_medialist(p); 2164 cur = p->media.ifm_cur; 2165 } 2166 2167 ifmr->ifm_status = IFM_AVALID; 2168 if (!p->link_config.link_ok) 2169 return; 2170 2171 ifmr->ifm_status |= IFM_ACTIVE; 2172 2173 /* 2174 * active and current will differ iff current media is autoselect. That 2175 * can happen only for copper RJ45. 2176 */ 2177 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 2178 return; 2179 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg, 2180 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps)); 2181 2182 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 2183 if (speed == SPEED_10000) 2184 ifmr->ifm_active |= IFM_10G_T; 2185 else if (speed == SPEED_1000) 2186 ifmr->ifm_active |= IFM_1000_T; 2187 else if (speed == SPEED_100) 2188 ifmr->ifm_active |= IFM_100_TX; 2189 else if (speed == SPEED_10) 2190 ifmr->ifm_active |= IFM_10_T; 2191 else 2192 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 2193 speed)); 2194} 2195 2196static void 2197cxgb_async_intr(void *data) 2198{ 2199 adapter_t *sc = data; 2200 2201 t3_write_reg(sc, A_PL_INT_ENABLE0, 0); 2202 (void) t3_read_reg(sc, A_PL_INT_ENABLE0); 2203 taskqueue_enqueue(sc->tq, &sc->slow_intr_task); 2204} 2205 2206static void 2207link_check_callout(void *arg) 2208{ 2209 struct port_info *pi = arg; 2210 struct adapter *sc = pi->adapter; 2211 2212 if (!isset(&sc->open_device_map, pi->port_id)) 2213 return; 2214 2215 taskqueue_enqueue(sc->tq, &pi->link_check_task); 2216} 2217 2218static void 2219check_link_status(void *arg, int pending) 2220{ 2221 struct port_info *pi = arg; 2222 struct adapter *sc = pi->adapter; 2223 2224 if (!isset(&sc->open_device_map, pi->port_id)) 2225 return; 2226 2227 t3_link_changed(sc, pi->port_id); 2228 2229 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) || 2230 pi->link_config.link_ok == 0) 2231 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi); 2232} 2233 2234void 2235t3_os_link_intr(struct port_info *pi) 2236{ 2237 /* 2238 * Schedule a link check in the near future. If the link is flapping 2239 * rapidly we'll keep resetting the callout and delaying the check until 2240 * things stabilize a bit. 2241 */ 2242 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi); 2243} 2244 2245static void 2246check_t3b2_mac(struct adapter *sc) 2247{ 2248 int i; 2249 2250 if (sc->flags & CXGB_SHUTDOWN) 2251 return; 2252 2253 for_each_port(sc, i) { 2254 struct port_info *p = &sc->port[i]; 2255 int status; 2256#ifdef INVARIANTS 2257 struct ifnet *ifp = p->ifp; 2258#endif 2259 2260 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault || 2261 !p->link_config.link_ok) 2262 continue; 2263 2264 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2265 ("%s: state mismatch (drv_flags %x, device_map %x)", 2266 __func__, ifp->if_drv_flags, sc->open_device_map)); 2267 2268 PORT_LOCK(p); 2269 status = t3b2_mac_watchdog_task(&p->mac); 2270 if (status == 1) 2271 p->mac.stats.num_toggled++; 2272 else if (status == 2) { 2273 struct cmac *mac = &p->mac; 2274 2275 cxgb_update_mac_settings(p); 2276 t3_link_start(&p->phy, mac, &p->link_config); 2277 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); 2278 t3_port_intr_enable(sc, p->port_id); 2279 p->mac.stats.num_resets++; 2280 } 2281 PORT_UNLOCK(p); 2282 } 2283} 2284 2285static void 2286cxgb_tick(void *arg) 2287{ 2288 adapter_t *sc = (adapter_t *)arg; 2289 2290 if (sc->flags & CXGB_SHUTDOWN) 2291 return; 2292 2293 taskqueue_enqueue(sc->tq, &sc->tick_task); 2294 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); 2295} 2296 2297static void 2298cxgb_tick_handler(void *arg, int count) 2299{ 2300 adapter_t *sc = (adapter_t *)arg; 2301 const struct adapter_params *p = &sc->params; 2302 int i; 2303 uint32_t cause, reset; 2304 2305 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE)) 2306 return; 2307 2308 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map) 2309 check_t3b2_mac(sc); 2310 2311 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY); 2312 if (cause) { 2313 struct sge_qset *qs = &sc->sge.qs[0]; 2314 uint32_t mask, v; 2315 2316 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00; 2317 2318 mask = 1; 2319 for (i = 0; i < SGE_QSETS; i++) { 2320 if (v & mask) 2321 qs[i].rspq.starved++; 2322 mask <<= 1; 2323 } 2324 2325 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */ 2326 2327 for (i = 0; i < SGE_QSETS * 2; i++) { 2328 if (v & mask) { 2329 qs[i / 2].fl[i % 2].empty++; 2330 } 2331 mask <<= 1; 2332 } 2333 2334 /* clear */ 2335 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v); 2336 t3_write_reg(sc, A_SG_INT_CAUSE, cause); 2337 } 2338 2339 for (i = 0; i < sc->params.nports; i++) { 2340 struct port_info *pi = &sc->port[i]; 2341 struct ifnet *ifp = pi->ifp; 2342 struct cmac *mac = &pi->mac; 2343 struct mac_stats *mstats = &mac->stats; 2344 int drops, j; 2345 2346 if (!isset(&sc->open_device_map, pi->port_id)) 2347 continue; 2348 2349 PORT_LOCK(pi); 2350 t3_mac_update_stats(mac); 2351 PORT_UNLOCK(pi); 2352 2353 ifp->if_opackets = mstats->tx_frames; 2354 ifp->if_ipackets = mstats->rx_frames; 2355 ifp->if_obytes = mstats->tx_octets; 2356 ifp->if_ibytes = mstats->rx_octets; 2357 ifp->if_omcasts = mstats->tx_mcast_frames; 2358 ifp->if_imcasts = mstats->rx_mcast_frames; 2359 ifp->if_collisions = mstats->tx_total_collisions; 2360 ifp->if_iqdrops = mstats->rx_cong_drops; 2361 2362 drops = 0; 2363 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets; j++) 2364 drops += sc->sge.qs[j].txq[TXQ_ETH].txq_mr->br_drops; 2365 ifp->if_snd.ifq_drops = drops; 2366 2367 ifp->if_oerrors = 2368 mstats->tx_excess_collisions + 2369 mstats->tx_underrun + 2370 mstats->tx_len_errs + 2371 mstats->tx_mac_internal_errs + 2372 mstats->tx_excess_deferral + 2373 mstats->tx_fcs_errs; 2374 ifp->if_ierrors = 2375 mstats->rx_jabber + 2376 mstats->rx_data_errs + 2377 mstats->rx_sequence_errs + 2378 mstats->rx_runt + 2379 mstats->rx_too_long + 2380 mstats->rx_mac_internal_errs + 2381 mstats->rx_short + 2382 mstats->rx_fcs_errs; 2383 2384 if (mac->multiport) 2385 continue; 2386 2387 /* Count rx fifo overflows, once per second */ 2388 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset); 2389 reset = 0; 2390 if (cause & F_RXFIFO_OVERFLOW) { 2391 mac->stats.rx_fifo_ovfl++; 2392 reset |= F_RXFIFO_OVERFLOW; 2393 } 2394 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset); 2395 } 2396} 2397 2398static void 2399touch_bars(device_t dev) 2400{ 2401 /* 2402 * Don't enable yet 2403 */ 2404#if !defined(__LP64__) && 0 2405 u32 v; 2406 2407 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v); 2408 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v); 2409 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v); 2410 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v); 2411 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v); 2412 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v); 2413#endif 2414} 2415 2416static int 2417set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset) 2418{ 2419 uint8_t *buf; 2420 int err = 0; 2421 u32 aligned_offset, aligned_len, *p; 2422 struct adapter *adapter = pi->adapter; 2423 2424 2425 aligned_offset = offset & ~3; 2426 aligned_len = (len + (offset & 3) + 3) & ~3; 2427 2428 if (aligned_offset != offset || aligned_len != len) { 2429 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO); 2430 if (!buf) 2431 return (ENOMEM); 2432 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf); 2433 if (!err && aligned_len > 4) 2434 err = t3_seeprom_read(adapter, 2435 aligned_offset + aligned_len - 4, 2436 (u32 *)&buf[aligned_len - 4]); 2437 if (err) 2438 goto out; 2439 memcpy(buf + (offset & 3), data, len); 2440 } else 2441 buf = (uint8_t *)(uintptr_t)data; 2442 2443 err = t3_seeprom_wp(adapter, 0); 2444 if (err) 2445 goto out; 2446 2447 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2448 err = t3_seeprom_write(adapter, aligned_offset, *p); 2449 aligned_offset += 4; 2450 } 2451 2452 if (!err) 2453 err = t3_seeprom_wp(adapter, 1); 2454out: 2455 if (buf != data) 2456 free(buf, M_DEVBUF); 2457 return err; 2458} 2459 2460 2461static int 2462in_range(int val, int lo, int hi) 2463{ 2464 return val < 0 || (val <= hi && val >= lo); 2465} 2466 2467static int 2468cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td) 2469{ 2470 return (0); 2471} 2472 2473static int 2474cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td) 2475{ 2476 return (0); 2477} 2478 2479static int 2480cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, 2481 int fflag, struct thread *td) 2482{ 2483 int mmd, error = 0; 2484 struct port_info *pi = dev->si_drv1; 2485 adapter_t *sc = pi->adapter; 2486 2487#ifdef PRIV_SUPPORTED 2488 if (priv_check(td, PRIV_DRIVER)) { 2489 if (cxgb_debug) 2490 printf("user does not have access to privileged ioctls\n"); 2491 return (EPERM); 2492 } 2493#else 2494 if (suser(td)) { 2495 if (cxgb_debug) 2496 printf("user does not have access to privileged ioctls\n"); 2497 return (EPERM); 2498 } 2499#endif 2500 2501 switch (cmd) { 2502 case CHELSIO_GET_MIIREG: { 2503 uint32_t val; 2504 struct cphy *phy = &pi->phy; 2505 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2506 2507 if (!phy->mdio_read) 2508 return (EOPNOTSUPP); 2509 if (is_10G(sc)) { 2510 mmd = mid->phy_id >> 8; 2511 if (!mmd) 2512 mmd = MDIO_DEV_PCS; 2513 else if (mmd > MDIO_DEV_VEND2) 2514 return (EINVAL); 2515 2516 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd, 2517 mid->reg_num, &val); 2518 } else 2519 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0, 2520 mid->reg_num & 0x1f, &val); 2521 if (error == 0) 2522 mid->val_out = val; 2523 break; 2524 } 2525 case CHELSIO_SET_MIIREG: { 2526 struct cphy *phy = &pi->phy; 2527 struct ch_mii_data *mid = (struct ch_mii_data *)data; 2528 2529 if (!phy->mdio_write) 2530 return (EOPNOTSUPP); 2531 if (is_10G(sc)) { 2532 mmd = mid->phy_id >> 8; 2533 if (!mmd) 2534 mmd = MDIO_DEV_PCS; 2535 else if (mmd > MDIO_DEV_VEND2) 2536 return (EINVAL); 2537 2538 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 2539 mmd, mid->reg_num, mid->val_in); 2540 } else 2541 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0, 2542 mid->reg_num & 0x1f, 2543 mid->val_in); 2544 break; 2545 } 2546 case CHELSIO_SETREG: { 2547 struct ch_reg *edata = (struct ch_reg *)data; 2548 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2549 return (EFAULT); 2550 t3_write_reg(sc, edata->addr, edata->val); 2551 break; 2552 } 2553 case CHELSIO_GETREG: { 2554 struct ch_reg *edata = (struct ch_reg *)data; 2555 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2556 return (EFAULT); 2557 edata->val = t3_read_reg(sc, edata->addr); 2558 break; 2559 } 2560 case CHELSIO_GET_SGE_CONTEXT: { 2561 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data; 2562 mtx_lock_spin(&sc->sge.reg_lock); 2563 switch (ecntxt->cntxt_type) { 2564 case CNTXT_TYPE_EGRESS: 2565 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id, 2566 ecntxt->data); 2567 break; 2568 case CNTXT_TYPE_FL: 2569 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id, 2570 ecntxt->data); 2571 break; 2572 case CNTXT_TYPE_RSP: 2573 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id, 2574 ecntxt->data); 2575 break; 2576 case CNTXT_TYPE_CQ: 2577 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id, 2578 ecntxt->data); 2579 break; 2580 default: 2581 error = EINVAL; 2582 break; 2583 } 2584 mtx_unlock_spin(&sc->sge.reg_lock); 2585 break; 2586 } 2587 case CHELSIO_GET_SGE_DESC: { 2588 struct ch_desc *edesc = (struct ch_desc *)data; 2589 int ret; 2590 if (edesc->queue_num >= SGE_QSETS * 6) 2591 return (EINVAL); 2592 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6], 2593 edesc->queue_num % 6, edesc->idx, edesc->data); 2594 if (ret < 0) 2595 return (EINVAL); 2596 edesc->size = ret; 2597 break; 2598 } 2599 case CHELSIO_GET_QSET_PARAMS: { 2600 struct qset_params *q; 2601 struct ch_qset_params *t = (struct ch_qset_params *)data; 2602 int q1 = pi->first_qset; 2603 int nqsets = pi->nqsets; 2604 int i; 2605 2606 if (t->qset_idx >= nqsets) 2607 return EINVAL; 2608 2609 i = q1 + t->qset_idx; 2610 q = &sc->params.sge.qset[i]; 2611 t->rspq_size = q->rspq_size; 2612 t->txq_size[0] = q->txq_size[0]; 2613 t->txq_size[1] = q->txq_size[1]; 2614 t->txq_size[2] = q->txq_size[2]; 2615 t->fl_size[0] = q->fl_size; 2616 t->fl_size[1] = q->jumbo_size; 2617 t->polling = q->polling; 2618 t->lro = q->lro; 2619 t->intr_lat = q->coalesce_usecs; 2620 t->cong_thres = q->cong_thres; 2621 t->qnum = i; 2622 2623 if ((sc->flags & FULL_INIT_DONE) == 0) 2624 t->vector = 0; 2625 else if (sc->flags & USING_MSIX) 2626 t->vector = rman_get_start(sc->msix_irq_res[i]); 2627 else 2628 t->vector = rman_get_start(sc->irq_res); 2629 2630 break; 2631 } 2632 case CHELSIO_GET_QSET_NUM: { 2633 struct ch_reg *edata = (struct ch_reg *)data; 2634 edata->val = pi->nqsets; 2635 break; 2636 } 2637 case CHELSIO_LOAD_FW: { 2638 uint8_t *fw_data; 2639 uint32_t vers; 2640 struct ch_mem_range *t = (struct ch_mem_range *)data; 2641 2642 /* 2643 * You're allowed to load a firmware only before FULL_INIT_DONE 2644 * 2645 * FW_UPTODATE is also set so the rest of the initialization 2646 * will not overwrite what was loaded here. This gives you the 2647 * flexibility to load any firmware (and maybe shoot yourself in 2648 * the foot). 2649 */ 2650 2651 ADAPTER_LOCK(sc); 2652 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) { 2653 ADAPTER_UNLOCK(sc); 2654 return (EBUSY); 2655 } 2656 2657 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2658 if (!fw_data) 2659 error = ENOMEM; 2660 else 2661 error = copyin(t->buf, fw_data, t->len); 2662 2663 if (!error) 2664 error = -t3_load_fw(sc, fw_data, t->len); 2665 2666 if (t3_get_fw_version(sc, &vers) == 0) { 2667 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), 2668 "%d.%d.%d", G_FW_VERSION_MAJOR(vers), 2669 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers)); 2670 } 2671 2672 if (!error) 2673 sc->flags |= FW_UPTODATE; 2674 2675 free(fw_data, M_DEVBUF); 2676 ADAPTER_UNLOCK(sc); 2677 break; 2678 } 2679 case CHELSIO_LOAD_BOOT: { 2680 uint8_t *boot_data; 2681 struct ch_mem_range *t = (struct ch_mem_range *)data; 2682 2683 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT); 2684 if (!boot_data) 2685 return ENOMEM; 2686 2687 error = copyin(t->buf, boot_data, t->len); 2688 if (!error) 2689 error = -t3_load_boot(sc, boot_data, t->len); 2690 2691 free(boot_data, M_DEVBUF); 2692 break; 2693 } 2694 case CHELSIO_GET_PM: { 2695 struct ch_pm *m = (struct ch_pm *)data; 2696 struct tp_params *p = &sc->params.tp; 2697 2698 if (!is_offload(sc)) 2699 return (EOPNOTSUPP); 2700 2701 m->tx_pg_sz = p->tx_pg_size; 2702 m->tx_num_pg = p->tx_num_pgs; 2703 m->rx_pg_sz = p->rx_pg_size; 2704 m->rx_num_pg = p->rx_num_pgs; 2705 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; 2706 2707 break; 2708 } 2709 case CHELSIO_SET_PM: { 2710 struct ch_pm *m = (struct ch_pm *)data; 2711 struct tp_params *p = &sc->params.tp; 2712 2713 if (!is_offload(sc)) 2714 return (EOPNOTSUPP); 2715 if (sc->flags & FULL_INIT_DONE) 2716 return (EBUSY); 2717 2718 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) || 2719 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1))) 2720 return (EINVAL); /* not power of 2 */ 2721 if (!(m->rx_pg_sz & 0x14000)) 2722 return (EINVAL); /* not 16KB or 64KB */ 2723 if (!(m->tx_pg_sz & 0x1554000)) 2724 return (EINVAL); 2725 if (m->tx_num_pg == -1) 2726 m->tx_num_pg = p->tx_num_pgs; 2727 if (m->rx_num_pg == -1) 2728 m->rx_num_pg = p->rx_num_pgs; 2729 if (m->tx_num_pg % 24 || m->rx_num_pg % 24) 2730 return (EINVAL); 2731 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size || 2732 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size) 2733 return (EINVAL); 2734 2735 p->rx_pg_size = m->rx_pg_sz; 2736 p->tx_pg_size = m->tx_pg_sz; 2737 p->rx_num_pgs = m->rx_num_pg; 2738 p->tx_num_pgs = m->tx_num_pg; 2739 break; 2740 } 2741 case CHELSIO_SETMTUTAB: { 2742 struct ch_mtus *m = (struct ch_mtus *)data; 2743 int i; 2744 2745 if (!is_offload(sc)) 2746 return (EOPNOTSUPP); 2747 if (offload_running(sc)) 2748 return (EBUSY); 2749 if (m->nmtus != NMTUS) 2750 return (EINVAL); 2751 if (m->mtus[0] < 81) /* accommodate SACK */ 2752 return (EINVAL); 2753 2754 /* 2755 * MTUs must be in ascending order 2756 */ 2757 for (i = 1; i < NMTUS; ++i) 2758 if (m->mtus[i] < m->mtus[i - 1]) 2759 return (EINVAL); 2760 2761 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus)); 2762 break; 2763 } 2764 case CHELSIO_GETMTUTAB: { 2765 struct ch_mtus *m = (struct ch_mtus *)data; 2766 2767 if (!is_offload(sc)) 2768 return (EOPNOTSUPP); 2769 2770 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus)); 2771 m->nmtus = NMTUS; 2772 break; 2773 } 2774 case CHELSIO_GET_MEM: { 2775 struct ch_mem_range *t = (struct ch_mem_range *)data; 2776 struct mc7 *mem; 2777 uint8_t *useraddr; 2778 u64 buf[32]; 2779 2780 /* 2781 * Use these to avoid modifying len/addr in the return 2782 * struct 2783 */ 2784 uint32_t len = t->len, addr = t->addr; 2785 2786 if (!is_offload(sc)) 2787 return (EOPNOTSUPP); 2788 if (!(sc->flags & FULL_INIT_DONE)) 2789 return (EIO); /* need the memory controllers */ 2790 if ((addr & 0x7) || (len & 0x7)) 2791 return (EINVAL); 2792 if (t->mem_id == MEM_CM) 2793 mem = &sc->cm; 2794 else if (t->mem_id == MEM_PMRX) 2795 mem = &sc->pmrx; 2796 else if (t->mem_id == MEM_PMTX) 2797 mem = &sc->pmtx; 2798 else 2799 return (EINVAL); 2800 2801 /* 2802 * Version scheme: 2803 * bits 0..9: chip version 2804 * bits 10..15: chip revision 2805 */ 2806 t->version = 3 | (sc->params.rev << 10); 2807 2808 /* 2809 * Read 256 bytes at a time as len can be large and we don't 2810 * want to use huge intermediate buffers. 2811 */ 2812 useraddr = (uint8_t *)t->buf; 2813 while (len) { 2814 unsigned int chunk = min(len, sizeof(buf)); 2815 2816 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf); 2817 if (error) 2818 return (-error); 2819 if (copyout(buf, useraddr, chunk)) 2820 return (EFAULT); 2821 useraddr += chunk; 2822 addr += chunk; 2823 len -= chunk; 2824 } 2825 break; 2826 } 2827 case CHELSIO_READ_TCAM_WORD: { 2828 struct ch_tcam_word *t = (struct ch_tcam_word *)data; 2829 2830 if (!is_offload(sc)) 2831 return (EOPNOTSUPP); 2832 if (!(sc->flags & FULL_INIT_DONE)) 2833 return (EIO); /* need MC5 */ 2834 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf); 2835 break; 2836 } 2837 case CHELSIO_SET_TRACE_FILTER: { 2838 struct ch_trace *t = (struct ch_trace *)data; 2839 const struct trace_params *tp; 2840 2841 tp = (const struct trace_params *)&t->sip; 2842 if (t->config_tx) 2843 t3_config_trace_filter(sc, tp, 0, t->invert_match, 2844 t->trace_tx); 2845 if (t->config_rx) 2846 t3_config_trace_filter(sc, tp, 1, t->invert_match, 2847 t->trace_rx); 2848 break; 2849 } 2850 case CHELSIO_SET_PKTSCHED: { 2851 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data; 2852 if (sc->open_device_map == 0) 2853 return (EAGAIN); 2854 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max, 2855 p->binding); 2856 break; 2857 } 2858 case CHELSIO_IFCONF_GETREGS: { 2859 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data; 2860 int reglen = cxgb_get_regs_len(); 2861 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT); 2862 if (buf == NULL) { 2863 return (ENOMEM); 2864 } 2865 if (regs->len > reglen) 2866 regs->len = reglen; 2867 else if (regs->len < reglen) 2868 error = ENOBUFS; 2869 2870 if (!error) { 2871 cxgb_get_regs(sc, regs, buf); 2872 error = copyout(buf, regs->data, reglen); 2873 } 2874 free(buf, M_DEVBUF); 2875 2876 break; 2877 } 2878 case CHELSIO_SET_HW_SCHED: { 2879 struct ch_hw_sched *t = (struct ch_hw_sched *)data; 2880 unsigned int ticks_per_usec = core_ticks_per_usec(sc); 2881 2882 if ((sc->flags & FULL_INIT_DONE) == 0) 2883 return (EAGAIN); /* need TP to be initialized */ 2884 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) || 2885 !in_range(t->channel, 0, 1) || 2886 !in_range(t->kbps, 0, 10000000) || 2887 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) || 2888 !in_range(t->flow_ipg, 0, 2889 dack_ticks_to_usec(sc, 0x7ff))) 2890 return (EINVAL); 2891 2892 if (t->kbps >= 0) { 2893 error = t3_config_sched(sc, t->kbps, t->sched); 2894 if (error < 0) 2895 return (-error); 2896 } 2897 if (t->class_ipg >= 0) 2898 t3_set_sched_ipg(sc, t->sched, t->class_ipg); 2899 if (t->flow_ipg >= 0) { 2900 t->flow_ipg *= 1000; /* us -> ns */ 2901 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1); 2902 } 2903 if (t->mode >= 0) { 2904 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched); 2905 2906 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2907 bit, t->mode ? bit : 0); 2908 } 2909 if (t->channel >= 0) 2910 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP, 2911 1 << t->sched, t->channel << t->sched); 2912 break; 2913 } 2914 case CHELSIO_GET_EEPROM: { 2915 int i; 2916 struct ch_eeprom *e = (struct ch_eeprom *)data; 2917 uint8_t *buf; 2918 2919 if (e->offset & 3 || e->offset >= EEPROMSIZE || 2920 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) { 2921 return (EINVAL); 2922 } 2923 2924 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT); 2925 if (buf == NULL) { 2926 return (ENOMEM); 2927 } 2928 e->magic = EEPROM_MAGIC; 2929 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4) 2930 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]); 2931 2932 if (!error) 2933 error = copyout(buf + e->offset, e->data, e->len); 2934 2935 free(buf, M_DEVBUF); 2936 break; 2937 } 2938 case CHELSIO_CLEAR_STATS: { 2939 if (!(sc->flags & FULL_INIT_DONE)) 2940 return EAGAIN; 2941 2942 PORT_LOCK(pi); 2943 t3_mac_update_stats(&pi->mac); 2944 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats)); 2945 PORT_UNLOCK(pi); 2946 break; 2947 } 2948 case CHELSIO_GET_UP_LA: { 2949 struct ch_up_la *la = (struct ch_up_la *)data; 2950 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT); 2951 if (buf == NULL) { 2952 return (ENOMEM); 2953 } 2954 if (la->bufsize < LA_BUFSIZE) 2955 error = ENOBUFS; 2956 2957 if (!error) 2958 error = -t3_get_up_la(sc, &la->stopped, &la->idx, 2959 &la->bufsize, buf); 2960 if (!error) 2961 error = copyout(buf, la->data, la->bufsize); 2962 2963 free(buf, M_DEVBUF); 2964 break; 2965 } 2966 case CHELSIO_GET_UP_IOQS: { 2967 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data; 2968 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT); 2969 uint32_t *v; 2970 2971 if (buf == NULL) { 2972 return (ENOMEM); 2973 } 2974 if (ioqs->bufsize < IOQS_BUFSIZE) 2975 error = ENOBUFS; 2976 2977 if (!error) 2978 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf); 2979 2980 if (!error) { 2981 v = (uint32_t *)buf; 2982 2983 ioqs->ioq_rx_enable = *v++; 2984 ioqs->ioq_tx_enable = *v++; 2985 ioqs->ioq_rx_status = *v++; 2986 ioqs->ioq_tx_status = *v++; 2987 2988 error = copyout(v, ioqs->data, ioqs->bufsize); 2989 } 2990 2991 free(buf, M_DEVBUF); 2992 break; 2993 } 2994 case CHELSIO_SET_FILTER: { 2995 struct ch_filter *f = (struct ch_filter *)data; 2996 struct filter_info *p; 2997 unsigned int nfilters = sc->params.mc5.nfilters; 2998 2999 if (!is_offload(sc)) 3000 return (EOPNOTSUPP); /* No TCAM */ 3001 if (!(sc->flags & FULL_INIT_DONE)) 3002 return (EAGAIN); /* mc5 not setup yet */ 3003 if (nfilters == 0) 3004 return (EBUSY); /* TOE will use TCAM */ 3005 3006 /* sanity checks */ 3007 if (f->filter_id >= nfilters || 3008 (f->val.dip && f->mask.dip != 0xffffffff) || 3009 (f->val.sport && f->mask.sport != 0xffff) || 3010 (f->val.dport && f->mask.dport != 0xffff) || 3011 (f->val.vlan && f->mask.vlan != 0xfff) || 3012 (f->val.vlan_prio && 3013 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) || 3014 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) || 3015 f->qset >= SGE_QSETS || 3016 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE) 3017 return (EINVAL); 3018 3019 /* Was allocated with M_WAITOK */ 3020 KASSERT(sc->filters, ("filter table NULL\n")); 3021 3022 p = &sc->filters[f->filter_id]; 3023 if (p->locked) 3024 return (EPERM); 3025 3026 bzero(p, sizeof(*p)); 3027 p->sip = f->val.sip; 3028 p->sip_mask = f->mask.sip; 3029 p->dip = f->val.dip; 3030 p->sport = f->val.sport; 3031 p->dport = f->val.dport; 3032 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff; 3033 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) : 3034 FILTER_NO_VLAN_PRI; 3035 p->mac_hit = f->mac_hit; 3036 p->mac_vld = f->mac_addr_idx != 0xffff; 3037 p->mac_idx = f->mac_addr_idx; 3038 p->pkt_type = f->proto; 3039 p->report_filter_id = f->want_filter_id; 3040 p->pass = f->pass; 3041 p->rss = f->rss; 3042 p->qset = f->qset; 3043 3044 error = set_filter(sc, f->filter_id, p); 3045 if (error == 0) 3046 p->valid = 1; 3047 break; 3048 } 3049 case CHELSIO_DEL_FILTER: { 3050 struct ch_filter *f = (struct ch_filter *)data; 3051 struct filter_info *p; 3052 unsigned int nfilters = sc->params.mc5.nfilters; 3053 3054 if (!is_offload(sc)) 3055 return (EOPNOTSUPP); 3056 if (!(sc->flags & FULL_INIT_DONE)) 3057 return (EAGAIN); 3058 if (nfilters == 0 || sc->filters == NULL) 3059 return (EINVAL); 3060 if (f->filter_id >= nfilters) 3061 return (EINVAL); 3062 3063 p = &sc->filters[f->filter_id]; 3064 if (p->locked) 3065 return (EPERM); 3066 if (!p->valid) 3067 return (EFAULT); /* Read "Bad address" as "Bad index" */ 3068 3069 bzero(p, sizeof(*p)); 3070 p->sip = p->sip_mask = 0xffffffff; 3071 p->vlan = 0xfff; 3072 p->vlan_prio = FILTER_NO_VLAN_PRI; 3073 p->pkt_type = 1; 3074 error = set_filter(sc, f->filter_id, p); 3075 break; 3076 } 3077 case CHELSIO_GET_FILTER: { 3078 struct ch_filter *f = (struct ch_filter *)data; 3079 struct filter_info *p; 3080 unsigned int i, nfilters = sc->params.mc5.nfilters; 3081 3082 if (!is_offload(sc)) 3083 return (EOPNOTSUPP); 3084 if (!(sc->flags & FULL_INIT_DONE)) 3085 return (EAGAIN); 3086 if (nfilters == 0 || sc->filters == NULL) 3087 return (EINVAL); 3088 3089 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1; 3090 for (; i < nfilters; i++) { 3091 p = &sc->filters[i]; 3092 if (!p->valid) 3093 continue; 3094 3095 bzero(f, sizeof(*f)); 3096 3097 f->filter_id = i; 3098 f->val.sip = p->sip; 3099 f->mask.sip = p->sip_mask; 3100 f->val.dip = p->dip; 3101 f->mask.dip = p->dip ? 0xffffffff : 0; 3102 f->val.sport = p->sport; 3103 f->mask.sport = p->sport ? 0xffff : 0; 3104 f->val.dport = p->dport; 3105 f->mask.dport = p->dport ? 0xffff : 0; 3106 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan; 3107 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff; 3108 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3109 0 : p->vlan_prio; 3110 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ? 3111 0 : FILTER_NO_VLAN_PRI; 3112 f->mac_hit = p->mac_hit; 3113 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff; 3114 f->proto = p->pkt_type; 3115 f->want_filter_id = p->report_filter_id; 3116 f->pass = p->pass; 3117 f->rss = p->rss; 3118 f->qset = p->qset; 3119 3120 break; 3121 } 3122 3123 if (i == nfilters) 3124 f->filter_id = 0xffffffff; 3125 break; 3126 } 3127 default: 3128 return (EOPNOTSUPP); 3129 break; 3130 } 3131 3132 return (error); 3133} 3134 3135static __inline void 3136reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start, 3137 unsigned int end) 3138{ 3139 uint32_t *p = (uint32_t *)(buf + start); 3140 3141 for ( ; start <= end; start += sizeof(uint32_t)) 3142 *p++ = t3_read_reg(ap, start); 3143} 3144 3145#define T3_REGMAP_SIZE (3 * 1024) 3146static int 3147cxgb_get_regs_len(void) 3148{ 3149 return T3_REGMAP_SIZE; 3150} 3151 3152static void 3153cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf) 3154{ 3155 3156 /* 3157 * Version scheme: 3158 * bits 0..9: chip version 3159 * bits 10..15: chip revision 3160 * bit 31: set for PCIe cards 3161 */ 3162 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31); 3163 3164 /* 3165 * We skip the MAC statistics registers because they are clear-on-read. 3166 * Also reading multi-register stats would need to synchronize with the 3167 * periodic mac stats accumulation. Hard to justify the complexity. 3168 */ 3169 memset(buf, 0, cxgb_get_regs_len()); 3170 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN); 3171 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); 3172 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); 3173 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); 3174 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); 3175 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0, 3176 XGM_REG(A_XGM_SERDES_STAT3, 1)); 3177 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), 3178 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); 3179} 3180 3181static int 3182alloc_filters(struct adapter *sc) 3183{ 3184 struct filter_info *p; 3185 unsigned int nfilters = sc->params.mc5.nfilters; 3186 3187 if (nfilters == 0) 3188 return (0); 3189 3190 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO); 3191 sc->filters = p; 3192 3193 p = &sc->filters[nfilters - 1]; 3194 p->vlan = 0xfff; 3195 p->vlan_prio = FILTER_NO_VLAN_PRI; 3196 p->pass = p->rss = p->valid = p->locked = 1; 3197 3198 return (0); 3199} 3200 3201static int 3202setup_hw_filters(struct adapter *sc) 3203{ 3204 int i, rc; 3205 unsigned int nfilters = sc->params.mc5.nfilters; 3206 3207 if (!sc->filters) 3208 return (0); 3209 3210 t3_enable_filters(sc); 3211 3212 for (i = rc = 0; i < nfilters && !rc; i++) { 3213 if (sc->filters[i].locked) 3214 rc = set_filter(sc, i, &sc->filters[i]); 3215 } 3216 3217 return (rc); 3218} 3219 3220static int 3221set_filter(struct adapter *sc, int id, const struct filter_info *f) 3222{ 3223 int len; 3224 struct mbuf *m; 3225 struct ulp_txpkt *txpkt; 3226 struct work_request_hdr *wr; 3227 struct cpl_pass_open_req *oreq; 3228 struct cpl_set_tcb_field *sreq; 3229 3230 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq); 3231 KASSERT(len <= MHLEN, ("filter request too big for an mbuf")); 3232 3233 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes - 3234 sc->params.mc5.nfilters; 3235 3236 m = m_gethdr(M_WAITOK, MT_DATA); 3237 m->m_len = m->m_pkthdr.len = len; 3238 bzero(mtod(m, char *), len); 3239 3240 wr = mtod(m, struct work_request_hdr *); 3241 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 3242 3243 oreq = (struct cpl_pass_open_req *)(wr + 1); 3244 txpkt = (struct ulp_txpkt *)oreq; 3245 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3246 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8)); 3247 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id)); 3248 oreq->local_port = htons(f->dport); 3249 oreq->peer_port = htons(f->sport); 3250 oreq->local_ip = htonl(f->dip); 3251 oreq->peer_ip = htonl(f->sip); 3252 oreq->peer_netmask = htonl(f->sip_mask); 3253 oreq->opt0h = 0; 3254 oreq->opt0l = htonl(F_NO_OFFLOAD); 3255 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) | 3256 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) | 3257 V_VLAN_PRI(f->vlan_prio >> 1) | 3258 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) | 3259 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) | 3260 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4))); 3261 3262 sreq = (struct cpl_set_tcb_field *)(oreq + 1); 3263 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL, 3264 (f->report_filter_id << 15) | (1 << 23) | 3265 ((u64)f->pass << 35) | ((u64)!f->rss << 36)); 3266 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1); 3267 t3_mgmt_tx(sc, m); 3268 3269 if (f->pass && !f->rss) { 3270 len = sizeof(*sreq); 3271 m = m_gethdr(M_WAITOK, MT_DATA); 3272 m->m_len = m->m_pkthdr.len = len; 3273 bzero(mtod(m, char *), len); 3274 sreq = mtod(m, struct cpl_set_tcb_field *); 3275 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3276 mk_set_tcb_field(sreq, id, 25, 0x3f80000, 3277 (u64)sc->rrss_map[f->qset] << 19); 3278 t3_mgmt_tx(sc, m); 3279 } 3280 return 0; 3281} 3282 3283static inline void 3284mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid, 3285 unsigned int word, u64 mask, u64 val) 3286{ 3287 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 3288 req->reply = V_NO_REPLY(1); 3289 req->cpu_idx = 0; 3290 req->word = htons(word); 3291 req->mask = htobe64(mask); 3292 req->val = htobe64(val); 3293} 3294 3295static inline void 3296set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 3297 unsigned int word, u64 mask, u64 val) 3298{ 3299 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 3300 3301 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 3302 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 3303 mk_set_tcb_field(req, tid, word, mask, val); 3304} 3305 3306void 3307t3_iterate(void (*func)(struct adapter *, void *), void *arg) 3308{ 3309 struct adapter *sc; 3310 3311 mtx_lock(&t3_list_lock); 3312 SLIST_FOREACH(sc, &t3_list, link) { 3313 /* 3314 * func should not make any assumptions about what state sc is 3315 * in - the only guarantee is that sc->sc_lock is a valid lock. 3316 */ 3317 func(sc, arg); 3318 } 3319 mtx_unlock(&t3_list_lock); 3320} 3321 3322#ifdef TCP_OFFLOAD 3323static int 3324toe_capability(struct port_info *pi, int enable) 3325{ 3326 int rc; 3327 struct adapter *sc = pi->adapter; 3328 3329 ADAPTER_LOCK_ASSERT_OWNED(sc); 3330 3331 if (!is_offload(sc)) 3332 return (ENODEV); 3333 3334 if (enable) { 3335 if (!(sc->flags & FULL_INIT_DONE)) { 3336 log(LOG_WARNING, 3337 "You must enable a cxgb interface first\n"); 3338 return (EAGAIN); 3339 } 3340 3341 if (isset(&sc->offload_map, pi->port_id)) 3342 return (0); 3343 3344 if (!(sc->flags & TOM_INIT_DONE)) { 3345 rc = t3_activate_uld(sc, ULD_TOM); 3346 if (rc == EAGAIN) { 3347 log(LOG_WARNING, 3348 "You must kldload t3_tom.ko before trying " 3349 "to enable TOE on a cxgb interface.\n"); 3350 } 3351 if (rc != 0) 3352 return (rc); 3353 KASSERT(sc->tom_softc != NULL, 3354 ("%s: TOM activated but softc NULL", __func__)); 3355 KASSERT(sc->flags & TOM_INIT_DONE, 3356 ("%s: TOM activated but flag not set", __func__)); 3357 } 3358 3359 setbit(&sc->offload_map, pi->port_id); 3360 3361 /* 3362 * XXX: Temporary code to allow iWARP to be enabled when TOE is 3363 * enabled on any port. Need to figure out how to enable, 3364 * disable, load, and unload iWARP cleanly. 3365 */ 3366 if (!isset(&sc->offload_map, MAX_NPORTS) && 3367 t3_activate_uld(sc, ULD_IWARP) == 0) 3368 setbit(&sc->offload_map, MAX_NPORTS); 3369 } else { 3370 if (!isset(&sc->offload_map, pi->port_id)) 3371 return (0); 3372 3373 KASSERT(sc->flags & TOM_INIT_DONE, 3374 ("%s: TOM never initialized?", __func__)); 3375 clrbit(&sc->offload_map, pi->port_id); 3376 } 3377 3378 return (0); 3379} 3380 3381/* 3382 * Add an upper layer driver to the global list. 3383 */ 3384int 3385t3_register_uld(struct uld_info *ui) 3386{ 3387 int rc = 0; 3388 struct uld_info *u; 3389 3390 mtx_lock(&t3_uld_list_lock); 3391 SLIST_FOREACH(u, &t3_uld_list, link) { 3392 if (u->uld_id == ui->uld_id) { 3393 rc = EEXIST; 3394 goto done; 3395 } 3396 } 3397 3398 SLIST_INSERT_HEAD(&t3_uld_list, ui, link); 3399 ui->refcount = 0; 3400done: 3401 mtx_unlock(&t3_uld_list_lock); 3402 return (rc); 3403} 3404 3405int 3406t3_unregister_uld(struct uld_info *ui) 3407{ 3408 int rc = EINVAL; 3409 struct uld_info *u; 3410 3411 mtx_lock(&t3_uld_list_lock); 3412 3413 SLIST_FOREACH(u, &t3_uld_list, link) { 3414 if (u == ui) { 3415 if (ui->refcount > 0) { 3416 rc = EBUSY; 3417 goto done; 3418 } 3419 3420 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link); 3421 rc = 0; 3422 goto done; 3423 } 3424 } 3425done: 3426 mtx_unlock(&t3_uld_list_lock); 3427 return (rc); 3428} 3429 3430int 3431t3_activate_uld(struct adapter *sc, int id) 3432{ 3433 int rc = EAGAIN; 3434 struct uld_info *ui; 3435 3436 mtx_lock(&t3_uld_list_lock); 3437 3438 SLIST_FOREACH(ui, &t3_uld_list, link) { 3439 if (ui->uld_id == id) { 3440 rc = ui->activate(sc); 3441 if (rc == 0) 3442 ui->refcount++; 3443 goto done; 3444 } 3445 } 3446done: 3447 mtx_unlock(&t3_uld_list_lock); 3448 3449 return (rc); 3450} 3451 3452int 3453t3_deactivate_uld(struct adapter *sc, int id) 3454{ 3455 int rc = EINVAL; 3456 struct uld_info *ui; 3457 3458 mtx_lock(&t3_uld_list_lock); 3459 3460 SLIST_FOREACH(ui, &t3_uld_list, link) { 3461 if (ui->uld_id == id) { 3462 rc = ui->deactivate(sc); 3463 if (rc == 0) 3464 ui->refcount--; 3465 goto done; 3466 } 3467 } 3468done: 3469 mtx_unlock(&t3_uld_list_lock); 3470 3471 return (rc); 3472} 3473 3474static int 3475cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused, 3476 struct mbuf *m) 3477{ 3478 m_freem(m); 3479 return (EDOOFUS); 3480} 3481 3482int 3483t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3484{ 3485 uintptr_t *loc, new; 3486 3487 if (opcode >= NUM_CPL_HANDLERS) 3488 return (EINVAL); 3489 3490 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3491 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3492 atomic_store_rel_ptr(loc, new); 3493 3494 return (0); 3495} 3496#endif 3497 3498static int 3499cxgbc_mod_event(module_t mod, int cmd, void *arg) 3500{ 3501 int rc = 0; 3502 3503 switch (cmd) { 3504 case MOD_LOAD: 3505 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF); 3506 SLIST_INIT(&t3_list); 3507#ifdef TCP_OFFLOAD 3508 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF); 3509 SLIST_INIT(&t3_uld_list); 3510#endif 3511 break; 3512 3513 case MOD_UNLOAD: 3514#ifdef TCP_OFFLOAD 3515 mtx_lock(&t3_uld_list_lock); 3516 if (!SLIST_EMPTY(&t3_uld_list)) { 3517 rc = EBUSY; 3518 mtx_unlock(&t3_uld_list_lock); 3519 break; 3520 } 3521 mtx_unlock(&t3_uld_list_lock); 3522 mtx_destroy(&t3_uld_list_lock); 3523#endif 3524 mtx_lock(&t3_list_lock); 3525 if (!SLIST_EMPTY(&t3_list)) { 3526 rc = EBUSY; 3527 mtx_unlock(&t3_list_lock); 3528 break; 3529 } 3530 mtx_unlock(&t3_list_lock); 3531 mtx_destroy(&t3_list_lock); 3532 break; 3533 } 3534 3535 return (rc); 3536} 3537