1/* $NetBSD: if_ixl.c,v 1.98 2024/02/09 22:08:35 andvar Exp $ */ 2 3/* 4 * Copyright (c) 2013-2015, Intel Corporation 5 * All rights reserved. 6 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34/* 35 * Copyright (c) 2016,2017 David Gwynne <dlg@openbsd.org> 36 * 37 * Permission to use, copy, modify, and distribute this software for any 38 * purpose with or without fee is hereby granted, provided that the above 39 * copyright notice and this permission notice appear in all copies. 40 * 41 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 42 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 43 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 44 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 45 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 46 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 47 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 48 */ 49 50/* 51 * Copyright (c) 2019 Internet Initiative Japan, Inc. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 64 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 65 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 66 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 67 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 68 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 69 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 70 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 71 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 72 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 73 * POSSIBILITY OF SUCH DAMAGE. 74 */ 75 76#include <sys/cdefs.h> 77__KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.98 2024/02/09 22:08:35 andvar Exp $"); 78 79#ifdef _KERNEL_OPT 80#include "opt_if_ixl.h" 81#endif 82 83#include <sys/param.h> 84#include <sys/types.h> 85 86#include <sys/bitops.h> 87#include <sys/cpu.h> 88#include <sys/device.h> 89#include <sys/evcnt.h> 90#include <sys/interrupt.h> 91#include <sys/kmem.h> 92#include <sys/module.h> 93#include <sys/mutex.h> 94#include <sys/pcq.h> 95#include <sys/syslog.h> 96#include <sys/workqueue.h> 97#include <sys/xcall.h> 98 99#include <sys/bus.h> 100 101#include <net/bpf.h> 102#include <net/if.h> 103#include <net/if_dl.h> 104#include <net/if_media.h> 105#include <net/if_ether.h> 106#include <net/rss_config.h> 107 108#include <netinet/tcp.h> /* for struct tcphdr */ 109#include <netinet/udp.h> /* for struct udphdr */ 110 111#include <dev/pci/pcivar.h> 112#include <dev/pci/pcidevs.h> 113 114#include <dev/pci/if_ixlreg.h> 115#include <dev/pci/if_ixlvar.h> 116 117#include <prop/proplib.h> 118 119struct ixl_softc; /* defined */ 120 121#define I40E_PF_RESET_WAIT_COUNT 200 122#define I40E_AQ_LARGE_BUF 512 123 124/* bitfields for Tx queue mapping in QTX_CTL */ 125#define I40E_QTX_CTL_VF_QUEUE 0x0 126#define I40E_QTX_CTL_VM_QUEUE 0x1 127#define I40E_QTX_CTL_PF_QUEUE 0x2 128 129#define I40E_QUEUE_TYPE_EOL 0x7ff 130#define I40E_INTR_NOTX_QUEUE 0 131 132#define I40E_QUEUE_TYPE_RX 0x0 133#define I40E_QUEUE_TYPE_TX 0x1 134#define I40E_QUEUE_TYPE_PE_CEQ 0x2 135#define I40E_QUEUE_TYPE_UNKNOWN 0x3 136 137#define I40E_ITR_INDEX_RX 0x0 138#define I40E_ITR_INDEX_TX 0x1 139#define I40E_ITR_INDEX_OTHER 0x2 140#define I40E_ITR_INDEX_NONE 0x3 141#define IXL_ITR_RX 0x7a /* 4K intrs/sec */ 142#define IXL_ITR_TX 0x7a /* 4K intrs/sec */ 143 144#define I40E_INTR_NOTX_QUEUE 0 145#define I40E_INTR_NOTX_INTR 0 146#define I40E_INTR_NOTX_RX_QUEUE 0 147#define I40E_INTR_NOTX_TX_QUEUE 1 148#define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK 149#define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK 150 151#define I40E_HASH_LUT_SIZE_128 0 152 153#define IXL_ICR0_CRIT_ERR_MASK \ 154 (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ 155 I40E_PFINT_ICR0_ECC_ERR_MASK | \ 156 I40E_PFINT_ICR0_PE_CRITERR_MASK) 157 158#define IXL_QUEUE_MAX_XL710 64 159#define IXL_QUEUE_MAX_X722 128 160 161#define IXL_TX_PKT_DESCS 8 162#define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS) 163#define IXL_TX_QUEUE_ALIGN 128 164#define IXL_RX_QUEUE_ALIGN 128 165 166#define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN) 167#define IXL_MTU_ETHERLEN ETHER_HDR_LEN \ 168 + ETHER_CRC_LEN \ 169 + ETHER_VLAN_ENCAP_LEN 170#if 0 171#define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN) 172#else 173/* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */ 174#define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN) 175#endif 176#define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN) 177 178#define IXL_PCIREG PCI_MAPREG_START 179 180#define IXL_ITR0 0x0 181#define IXL_ITR1 0x1 182#define IXL_ITR2 0x2 183#define IXL_NOITR 0x3 184 185#define IXL_AQ_NUM 256 186#define IXL_AQ_MASK (IXL_AQ_NUM - 1) 187#define IXL_AQ_ALIGN 64 /* lol */ 188#define IXL_AQ_BUFLEN 4096 189 190#define IXL_HMC_ROUNDUP 512 191#define IXL_HMC_PGSIZE 4096 192#define IXL_HMC_DVASZ sizeof(uint64_t) 193#define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ) 194#define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS) 195#define IXL_HMC_PDVALID 1ULL 196 197#define IXL_ATQ_EXEC_TIMEOUT (10 * hz) 198 199#define IXL_SRRD_SRCTL_ATTEMPTS 100000 200 201struct ixl_aq_regs { 202 bus_size_t atq_tail; 203 bus_size_t atq_head; 204 bus_size_t atq_len; 205 bus_size_t atq_bal; 206 bus_size_t atq_bah; 207 208 bus_size_t arq_tail; 209 bus_size_t arq_head; 210 bus_size_t arq_len; 211 bus_size_t arq_bal; 212 bus_size_t arq_bah; 213 214 uint32_t atq_len_enable; 215 uint32_t atq_tail_mask; 216 uint32_t atq_head_mask; 217 218 uint32_t arq_len_enable; 219 uint32_t arq_tail_mask; 220 uint32_t arq_head_mask; 221}; 222 223struct ixl_phy_type { 224 uint64_t phy_type; 225 uint64_t ifm_type; 226}; 227 228struct ixl_speed_type { 229 uint8_t dev_speed; 230 uint64_t net_speed; 231}; 232 233struct ixl_hmc_entry { 234 uint64_t hmc_base; 235 uint32_t hmc_count; 236 uint64_t hmc_size; 237}; 238 239enum ixl_hmc_types { 240 IXL_HMC_LAN_TX = 0, 241 IXL_HMC_LAN_RX, 242 IXL_HMC_FCOE_CTX, 243 IXL_HMC_FCOE_FILTER, 244 IXL_HMC_COUNT 245}; 246 247struct ixl_hmc_pack { 248 uint16_t offset; 249 uint16_t width; 250 uint16_t lsb; 251}; 252 253/* 254 * these hmc objects have weird sizes and alignments, so these are abstract 255 * representations of them that are nice for c to populate. 256 * 257 * the packing code relies on little-endian values being stored in the fields, 258 * no high bits in the fields being set, and the fields must be packed in the 259 * same order as they are in the ctx structure. 260 */ 261 262struct ixl_hmc_rxq { 263 uint16_t head; 264 uint8_t cpuid; 265 uint64_t base; 266#define IXL_HMC_RXQ_BASE_UNIT 128 267 uint16_t qlen; 268 uint16_t dbuff; 269#define IXL_HMC_RXQ_DBUFF_UNIT 128 270 uint8_t hbuff; 271#define IXL_HMC_RXQ_HBUFF_UNIT 64 272 uint8_t dtype; 273#define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0 274#define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1 275#define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2 276 uint8_t dsize; 277#define IXL_HMC_RXQ_DSIZE_16 0 278#define IXL_HMC_RXQ_DSIZE_32 1 279 uint8_t crcstrip; 280 uint8_t fc_ena; 281 uint8_t l2sel; 282 uint8_t hsplit_0; 283 uint8_t hsplit_1; 284 uint8_t showiv; 285 uint16_t rxmax; 286 uint8_t tphrdesc_ena; 287 uint8_t tphwdesc_ena; 288 uint8_t tphdata_ena; 289 uint8_t tphhead_ena; 290 uint8_t lrxqthresh; 291 uint8_t prefena; 292}; 293 294static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = { 295 { offsetof(struct ixl_hmc_rxq, head), 13, 0 }, 296 { offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 }, 297 { offsetof(struct ixl_hmc_rxq, base), 57, 32 }, 298 { offsetof(struct ixl_hmc_rxq, qlen), 13, 89 }, 299 { offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 }, 300 { offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 }, 301 { offsetof(struct ixl_hmc_rxq, dtype), 2, 114 }, 302 { offsetof(struct ixl_hmc_rxq, dsize), 1, 116 }, 303 { offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 }, 304 { offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 }, 305 { offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 }, 306 { offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 }, 307 { offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 }, 308 { offsetof(struct ixl_hmc_rxq, showiv), 1, 127 }, 309 { offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 }, 310 { offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 }, 311 { offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 }, 312 { offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 }, 313 { offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 }, 314 { offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 }, 315 { offsetof(struct ixl_hmc_rxq, prefena), 1, 201 }, 316}; 317 318#define IXL_HMC_RXQ_MINSIZE (201 + 1) 319 320struct ixl_hmc_txq { 321 uint16_t head; 322 uint8_t new_context; 323 uint64_t base; 324#define IXL_HMC_TXQ_BASE_UNIT 128 325 uint8_t fc_ena; 326 uint8_t timesync_ena; 327 uint8_t fd_ena; 328 uint8_t alt_vlan_ena; 329 uint8_t cpuid; 330 uint16_t thead_wb; 331 uint8_t head_wb_ena; 332#define IXL_HMC_TXQ_DESC_WB 0 333#define IXL_HMC_TXQ_HEAD_WB 1 334 uint16_t qlen; 335 uint8_t tphrdesc_ena; 336 uint8_t tphrpacket_ena; 337 uint8_t tphwdesc_ena; 338 uint64_t head_wb_addr; 339 uint32_t crc; 340 uint16_t rdylist; 341 uint8_t rdylist_act; 342}; 343 344static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = { 345 { offsetof(struct ixl_hmc_txq, head), 13, 0 }, 346 { offsetof(struct ixl_hmc_txq, new_context), 1, 30 }, 347 { offsetof(struct ixl_hmc_txq, base), 57, 32 }, 348 { offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 }, 349 { offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 }, 350 { offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 }, 351 { offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 }, 352 { offsetof(struct ixl_hmc_txq, cpuid), 8, 96 }, 353/* line 1 */ 354 { offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 }, 355 { offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 }, 356 { offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 }, 357 { offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 }, 358 { offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 }, 359 { offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 }, 360 { offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 }, 361/* line 7 */ 362 { offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) }, 363 { offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) }, 364 { offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) }, 365}; 366 367#define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1) 368 369struct ixl_work { 370 struct work ixw_cookie; 371 void (*ixw_func)(void *); 372 void *ixw_arg; 373 unsigned int ixw_added; 374}; 375#define IXL_WORKQUEUE_PRI PRI_SOFTNET 376 377struct ixl_tx_map { 378 struct mbuf *txm_m; 379 bus_dmamap_t txm_map; 380 unsigned int txm_eop; 381}; 382 383struct ixl_tx_ring { 384 kmutex_t txr_lock; 385 struct ixl_softc *txr_sc; 386 387 unsigned int txr_prod; 388 unsigned int txr_cons; 389 390 struct ixl_tx_map *txr_maps; 391 struct ixl_dmamem txr_mem; 392 393 bus_size_t txr_tail; 394 unsigned int txr_qid; 395 pcq_t *txr_intrq; 396 void *txr_si; 397 398 struct evcnt txr_defragged; 399 struct evcnt txr_defrag_failed; 400 struct evcnt txr_pcqdrop; 401 struct evcnt txr_transmitdef; 402 struct evcnt txr_intr; 403 struct evcnt txr_defer; 404}; 405 406struct ixl_rx_map { 407 struct mbuf *rxm_m; 408 bus_dmamap_t rxm_map; 409}; 410 411struct ixl_rx_ring { 412 kmutex_t rxr_lock; 413 414 unsigned int rxr_prod; 415 unsigned int rxr_cons; 416 417 struct ixl_rx_map *rxr_maps; 418 struct ixl_dmamem rxr_mem; 419 420 struct mbuf *rxr_m_head; 421 struct mbuf **rxr_m_tail; 422 423 bus_size_t rxr_tail; 424 unsigned int rxr_qid; 425 426 struct evcnt rxr_mgethdr_failed; 427 struct evcnt rxr_mgetcl_failed; 428 struct evcnt rxr_mbuf_load_failed; 429 struct evcnt rxr_intr; 430 struct evcnt rxr_defer; 431}; 432 433struct ixl_queue_pair { 434 struct ixl_softc *qp_sc; 435 struct ixl_tx_ring *qp_txr; 436 struct ixl_rx_ring *qp_rxr; 437 438 char qp_name[16]; 439 440 void *qp_si; 441 struct work qp_work; 442 bool qp_workqueue; 443}; 444 445struct ixl_atq { 446 struct ixl_aq_desc iatq_desc; 447 void (*iatq_fn)(struct ixl_softc *, 448 const struct ixl_aq_desc *); 449 bool iatq_inuse; 450}; 451SIMPLEQ_HEAD(ixl_atq_list, ixl_atq); 452 453struct ixl_product { 454 unsigned int vendor_id; 455 unsigned int product_id; 456}; 457 458struct ixl_stats_counters { 459 bool isc_has_offset; 460 struct evcnt isc_crc_errors; 461 uint64_t isc_crc_errors_offset; 462 struct evcnt isc_illegal_bytes; 463 uint64_t isc_illegal_bytes_offset; 464 struct evcnt isc_rx_bytes; 465 uint64_t isc_rx_bytes_offset; 466 struct evcnt isc_rx_discards; 467 uint64_t isc_rx_discards_offset; 468 struct evcnt isc_rx_unicast; 469 uint64_t isc_rx_unicast_offset; 470 struct evcnt isc_rx_multicast; 471 uint64_t isc_rx_multicast_offset; 472 struct evcnt isc_rx_broadcast; 473 uint64_t isc_rx_broadcast_offset; 474 struct evcnt isc_rx_size_64; 475 uint64_t isc_rx_size_64_offset; 476 struct evcnt isc_rx_size_127; 477 uint64_t isc_rx_size_127_offset; 478 struct evcnt isc_rx_size_255; 479 uint64_t isc_rx_size_255_offset; 480 struct evcnt isc_rx_size_511; 481 uint64_t isc_rx_size_511_offset; 482 struct evcnt isc_rx_size_1023; 483 uint64_t isc_rx_size_1023_offset; 484 struct evcnt isc_rx_size_1522; 485 uint64_t isc_rx_size_1522_offset; 486 struct evcnt isc_rx_size_big; 487 uint64_t isc_rx_size_big_offset; 488 struct evcnt isc_rx_undersize; 489 uint64_t isc_rx_undersize_offset; 490 struct evcnt isc_rx_oversize; 491 uint64_t isc_rx_oversize_offset; 492 struct evcnt isc_rx_fragments; 493 uint64_t isc_rx_fragments_offset; 494 struct evcnt isc_rx_jabber; 495 uint64_t isc_rx_jabber_offset; 496 struct evcnt isc_tx_bytes; 497 uint64_t isc_tx_bytes_offset; 498 struct evcnt isc_tx_dropped_link_down; 499 uint64_t isc_tx_dropped_link_down_offset; 500 struct evcnt isc_tx_unicast; 501 uint64_t isc_tx_unicast_offset; 502 struct evcnt isc_tx_multicast; 503 uint64_t isc_tx_multicast_offset; 504 struct evcnt isc_tx_broadcast; 505 uint64_t isc_tx_broadcast_offset; 506 struct evcnt isc_tx_size_64; 507 uint64_t isc_tx_size_64_offset; 508 struct evcnt isc_tx_size_127; 509 uint64_t isc_tx_size_127_offset; 510 struct evcnt isc_tx_size_255; 511 uint64_t isc_tx_size_255_offset; 512 struct evcnt isc_tx_size_511; 513 uint64_t isc_tx_size_511_offset; 514 struct evcnt isc_tx_size_1023; 515 uint64_t isc_tx_size_1023_offset; 516 struct evcnt isc_tx_size_1522; 517 uint64_t isc_tx_size_1522_offset; 518 struct evcnt isc_tx_size_big; 519 uint64_t isc_tx_size_big_offset; 520 struct evcnt isc_mac_local_faults; 521 uint64_t isc_mac_local_faults_offset; 522 struct evcnt isc_mac_remote_faults; 523 uint64_t isc_mac_remote_faults_offset; 524 struct evcnt isc_link_xon_rx; 525 uint64_t isc_link_xon_rx_offset; 526 struct evcnt isc_link_xon_tx; 527 uint64_t isc_link_xon_tx_offset; 528 struct evcnt isc_link_xoff_rx; 529 uint64_t isc_link_xoff_rx_offset; 530 struct evcnt isc_link_xoff_tx; 531 uint64_t isc_link_xoff_tx_offset; 532 struct evcnt isc_vsi_rx_discards; 533 uint64_t isc_vsi_rx_discards_offset; 534 struct evcnt isc_vsi_rx_bytes; 535 uint64_t isc_vsi_rx_bytes_offset; 536 struct evcnt isc_vsi_rx_unicast; 537 uint64_t isc_vsi_rx_unicast_offset; 538 struct evcnt isc_vsi_rx_multicast; 539 uint64_t isc_vsi_rx_multicast_offset; 540 struct evcnt isc_vsi_rx_broadcast; 541 uint64_t isc_vsi_rx_broadcast_offset; 542 struct evcnt isc_vsi_tx_errors; 543 uint64_t isc_vsi_tx_errors_offset; 544 struct evcnt isc_vsi_tx_bytes; 545 uint64_t isc_vsi_tx_bytes_offset; 546 struct evcnt isc_vsi_tx_unicast; 547 uint64_t isc_vsi_tx_unicast_offset; 548 struct evcnt isc_vsi_tx_multicast; 549 uint64_t isc_vsi_tx_multicast_offset; 550 struct evcnt isc_vsi_tx_broadcast; 551 uint64_t isc_vsi_tx_broadcast_offset; 552}; 553 554/* 555 * Locking notes: 556 * + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and 557 * a field in ixl_rx_ring is protected by rxr_lock (a spin mutex). 558 * - more than one lock of them cannot be held at once. 559 * + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock 560 * (a spin mutex). 561 * - the lock cannot held with txr_lock or rxr_lock. 562 * + a field named sc_arq_* is not protected by any lock. 563 * - operations for sc_arq_* is done in one context related to 564 * sc_arq_task. 565 * + other fields in ixl_softc is protected by sc_cfg_lock 566 * (an adaptive mutex) 567 * - It must be held before another lock is held, and It can be 568 * released after the other lock is released. 569 * */ 570 571struct ixl_softc { 572 device_t sc_dev; 573 struct ethercom sc_ec; 574 bool sc_attached; 575 bool sc_dead; 576 uint32_t sc_port; 577 struct sysctllog *sc_sysctllog; 578 struct workqueue *sc_workq; 579 struct workqueue *sc_workq_txrx; 580 int sc_stats_intval; 581 callout_t sc_stats_callout; 582 struct ixl_work sc_stats_task; 583 struct ixl_stats_counters 584 sc_stats_counters; 585 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 586 struct ifmedia sc_media; 587 uint64_t sc_media_status; 588 uint64_t sc_media_active; 589 uint64_t sc_phy_types; 590 uint8_t sc_phy_abilities; 591 uint8_t sc_phy_linkspeed; 592 uint8_t sc_phy_fec_cfg; 593 uint16_t sc_eee_cap; 594 uint32_t sc_eeer_val; 595 uint8_t sc_d3_lpan; 596 kmutex_t sc_cfg_lock; 597 enum i40e_mac_type sc_mac_type; 598 uint32_t sc_rss_table_size; 599 uint32_t sc_rss_table_entry_width; 600 bool sc_txrx_workqueue; 601 u_int sc_tx_process_limit; 602 u_int sc_rx_process_limit; 603 u_int sc_tx_intr_process_limit; 604 u_int sc_rx_intr_process_limit; 605 606 int sc_cur_ec_capenable; 607 608 struct pci_attach_args sc_pa; 609 pci_intr_handle_t *sc_ihp; 610 void **sc_ihs; 611 unsigned int sc_nintrs; 612 613 bus_dma_tag_t sc_dmat; 614 bus_space_tag_t sc_memt; 615 bus_space_handle_t sc_memh; 616 bus_size_t sc_mems; 617 618 uint8_t sc_pf_id; 619 uint16_t sc_uplink_seid; /* le */ 620 uint16_t sc_downlink_seid; /* le */ 621 uint16_t sc_vsi_number; 622 uint16_t sc_vsi_stat_counter_idx; 623 uint16_t sc_seid; 624 unsigned int sc_base_queue; 625 626 pci_intr_type_t sc_intrtype; 627 unsigned int sc_msix_vector_queue; 628 629 struct ixl_dmamem sc_scratch; 630 struct ixl_dmamem sc_aqbuf; 631 632 const struct ixl_aq_regs * 633 sc_aq_regs; 634 uint32_t sc_aq_flags; 635#define IXL_SC_AQ_FLAG_RXCTL __BIT(0) 636#define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1) 637#define IXL_SC_AQ_FLAG_NVMREAD __BIT(2) 638#define IXL_SC_AQ_FLAG_RSS __BIT(3) 639 640 kmutex_t sc_atq_lock; 641 kcondvar_t sc_atq_cv; 642 struct ixl_dmamem sc_atq; 643 unsigned int sc_atq_prod; 644 unsigned int sc_atq_cons; 645 646 struct ixl_dmamem sc_arq; 647 struct ixl_work sc_arq_task; 648 struct ixl_aq_bufs sc_arq_idle; 649 struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM]; 650 unsigned int sc_arq_prod; 651 unsigned int sc_arq_cons; 652 653 struct ixl_work sc_link_state_task; 654 struct ixl_work sc_link_state_done_task; 655 struct ixl_atq sc_link_state_atq; 656 657 struct ixl_dmamem sc_hmc_sd; 658 struct ixl_dmamem sc_hmc_pd; 659 struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT]; 660 661 struct if_percpuq *sc_ipq; 662 unsigned int sc_tx_ring_ndescs; 663 unsigned int sc_rx_ring_ndescs; 664 unsigned int sc_nqueue_pairs; 665 unsigned int sc_nqueue_pairs_max; 666 unsigned int sc_nqueue_pairs_device; 667 struct ixl_queue_pair *sc_qps; 668 uint32_t sc_itr_rx; 669 uint32_t sc_itr_tx; 670 671 struct evcnt sc_event_atq; 672 struct evcnt sc_event_link; 673 struct evcnt sc_event_ecc_err; 674 struct evcnt sc_event_pci_exception; 675 struct evcnt sc_event_crit_err; 676}; 677 678#define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX 679#define IXL_TX_PROCESS_LIMIT 256 680#define IXL_RX_PROCESS_LIMIT 256 681#define IXL_TX_INTR_PROCESS_LIMIT 256 682#define IXL_RX_INTR_PROCESS_LIMIT 0U 683 684#define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \ 685 IFCAP_CSUM_TCPv4_Rx | \ 686 IFCAP_CSUM_UDPv4_Rx | \ 687 IFCAP_CSUM_TCPv6_Rx | \ 688 IFCAP_CSUM_UDPv6_Rx) 689#define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \ 690 IFCAP_CSUM_TCPv4_Tx | \ 691 IFCAP_CSUM_UDPv4_Tx | \ 692 IFCAP_CSUM_TCPv6_Tx | \ 693 IFCAP_CSUM_UDPv6_Tx) 694#define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \ 695 M_CSUM_TCPv4 | M_CSUM_TCPv6 | \ 696 M_CSUM_UDPv4 | M_CSUM_UDPv6) 697 698#define delaymsec(_x) DELAY(1000 * (_x)) 699#ifdef IXL_DEBUG 700#define DDPRINTF(sc, fmt, args...) \ 701do { \ 702 if ((sc) != NULL) { \ 703 device_printf( \ 704 ((struct ixl_softc *)(sc))->sc_dev, \ 705 ""); \ 706 } \ 707 printf("%s:\t" fmt, __func__, ##args); \ 708} while (0) 709#else 710#define DDPRINTF(sc, fmt, args...) __nothing 711#endif 712#ifndef IXL_STATS_INTERVAL_MSEC 713#define IXL_STATS_INTERVAL_MSEC 10000 714#endif 715#ifndef IXL_QUEUE_NUM 716#define IXL_QUEUE_NUM 0 717#endif 718 719enum ixl_link_flags { 720 IXL_LINK_NOFLAGS = 0, 721 IXL_LINK_FLAG_WAITDONE = __BIT(0), 722}; 723 724static bool ixl_param_nomsix = false; 725static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC; 726static int ixl_param_nqps_limit = IXL_QUEUE_NUM; 727static unsigned int ixl_param_tx_ndescs = 512; 728static unsigned int ixl_param_rx_ndescs = 512; 729 730static enum i40e_mac_type 731 ixl_mactype(pci_product_id_t); 732static void ixl_pci_csr_setup(pci_chipset_tag_t, pcitag_t); 733static void ixl_clear_hw(struct ixl_softc *); 734static int ixl_pf_reset(struct ixl_softc *); 735 736static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *, 737 bus_size_t, bus_size_t); 738static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *); 739 740static int ixl_arq_fill(struct ixl_softc *); 741static void ixl_arq_unfill(struct ixl_softc *); 742 743static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *, 744 unsigned int); 745static void ixl_atq_set(struct ixl_atq *, 746 void (*)(struct ixl_softc *, const struct ixl_aq_desc *)); 747static void ixl_wakeup(struct ixl_softc *, const struct ixl_aq_desc *); 748static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *); 749static void ixl_atq_done(struct ixl_softc *); 750static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *); 751static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *); 752static int ixl_get_version(struct ixl_softc *); 753static int ixl_get_nvm_version(struct ixl_softc *); 754static int ixl_get_hw_capabilities(struct ixl_softc *); 755static int ixl_pxe_clear(struct ixl_softc *); 756static int ixl_lldp_shut(struct ixl_softc *); 757static int ixl_get_mac(struct ixl_softc *); 758static int ixl_get_switch_config(struct ixl_softc *); 759static int ixl_phy_mask_ints(struct ixl_softc *); 760static int ixl_get_phy_info(struct ixl_softc *); 761static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool); 762static int ixl_set_phy_autoselect(struct ixl_softc *); 763static int ixl_restart_an(struct ixl_softc *); 764static int ixl_hmc(struct ixl_softc *); 765static void ixl_hmc_free(struct ixl_softc *); 766static int ixl_get_vsi(struct ixl_softc *); 767static int ixl_set_vsi(struct ixl_softc *); 768static void ixl_set_filter_control(struct ixl_softc *); 769static int ixl_get_link_status(struct ixl_softc *, enum ixl_link_flags); 770static void ixl_get_link_status_work(void *); 771static int ixl_get_link_status_poll(struct ixl_softc *, int *); 772static void ixl_get_link_status_done(struct ixl_softc *, 773 const struct ixl_aq_desc *); 774static void ixl_get_link_status_done_work(void *); 775static int ixl_set_link_status_locked(struct ixl_softc *, 776 const struct ixl_aq_desc *); 777static uint64_t ixl_search_link_speed(uint8_t); 778static uint8_t ixl_search_baudrate(uint64_t); 779static void ixl_config_rss(struct ixl_softc *); 780static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *, 781 uint16_t, uint16_t); 782static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *, 783 uint16_t, uint16_t); 784static void ixl_arq(void *); 785static void ixl_hmc_pack(void *, const void *, 786 const struct ixl_hmc_pack *, unsigned int); 787static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t); 788static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t); 789static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *); 790 791static int ixl_match(device_t, cfdata_t, void *); 792static void ixl_attach(device_t, device_t, void *); 793static int ixl_detach(device_t, int); 794 795static void ixl_media_add(struct ixl_softc *); 796static int ixl_media_change(struct ifnet *); 797static void ixl_media_status(struct ifnet *, struct ifmediareq *); 798static int ixl_ioctl(struct ifnet *, u_long, void *); 799static void ixl_start(struct ifnet *); 800static int ixl_transmit(struct ifnet *, struct mbuf *); 801static void ixl_deferred_transmit(void *); 802static int ixl_intr(void *); 803static int ixl_queue_intr(void *); 804static int ixl_other_intr(void *); 805static void ixl_handle_queue(void *); 806static void ixl_handle_queue_wk(struct work *, void *); 807static void ixl_sched_handle_queue(struct ixl_softc *, 808 struct ixl_queue_pair *); 809static int ixl_init(struct ifnet *); 810static int ixl_init_locked(struct ixl_softc *); 811static void ixl_stop(struct ifnet *, int); 812static void ixl_stop_locked(struct ixl_softc *); 813static int ixl_iff(struct ixl_softc *); 814static int ixl_ifflags_cb(struct ethercom *); 815static int ixl_setup_interrupts(struct ixl_softc *); 816static int ixl_establish_intx(struct ixl_softc *); 817static int ixl_establish_msix(struct ixl_softc *); 818static void ixl_enable_queue_intr(struct ixl_softc *, 819 struct ixl_queue_pair *); 820static void ixl_disable_queue_intr(struct ixl_softc *, 821 struct ixl_queue_pair *); 822static void ixl_enable_other_intr(struct ixl_softc *); 823static void ixl_disable_other_intr(struct ixl_softc *); 824static void ixl_config_queue_intr(struct ixl_softc *); 825static void ixl_config_other_intr(struct ixl_softc *); 826 827static struct ixl_tx_ring * 828 ixl_txr_alloc(struct ixl_softc *, unsigned int); 829static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int); 830static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *); 831static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *); 832static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *); 833static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *); 834static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *); 835static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *); 836static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int); 837 838static struct ixl_rx_ring * 839 ixl_rxr_alloc(struct ixl_softc *, unsigned int); 840static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *); 841static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *); 842static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *); 843static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *); 844static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *); 845static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *); 846static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int); 847static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *); 848 849static struct workqueue * 850 ixl_workq_create(const char *, pri_t, int, int); 851static void ixl_workq_destroy(struct workqueue *); 852static int ixl_workqs_teardown(device_t); 853static void ixl_work_set(struct ixl_work *, void (*)(void *), void *); 854static void ixl_work_add(struct workqueue *, struct ixl_work *); 855static void ixl_work_wait(struct workqueue *, struct ixl_work *); 856static void ixl_workq_work(struct work *, void *); 857static const struct ixl_product * 858 ixl_lookup(const struct pci_attach_args *pa); 859static void ixl_link_state_update(struct ixl_softc *, 860 const struct ixl_aq_desc *); 861static int ixl_vlan_cb(struct ethercom *, uint16_t, bool); 862static int ixl_setup_vlan_hwfilter(struct ixl_softc *); 863static void ixl_teardown_vlan_hwfilter(struct ixl_softc *); 864static int ixl_update_macvlan(struct ixl_softc *); 865static int ixl_setup_interrupts(struct ixl_softc *); 866static void ixl_teardown_interrupts(struct ixl_softc *); 867static int ixl_setup_stats(struct ixl_softc *); 868static void ixl_teardown_stats(struct ixl_softc *); 869static void ixl_stats_callout(void *); 870static void ixl_stats_update(void *); 871static int ixl_setup_sysctls(struct ixl_softc *); 872static void ixl_teardown_sysctls(struct ixl_softc *); 873static int ixl_sysctl_itr_handler(SYSCTLFN_PROTO); 874static int ixl_queue_pairs_alloc(struct ixl_softc *); 875static void ixl_queue_pairs_free(struct ixl_softc *); 876 877static const struct ixl_phy_type ixl_phy_type_map[] = { 878 { 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII }, 879 { 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX }, 880 { 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 }, 881 { 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR }, 882 { 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 }, 883 { 1ULL << IXL_PHY_TYPE_XAUI | 884 1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 }, 885 { 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI }, 886 { 1ULL << IXL_PHY_TYPE_XLAUI | 887 1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI }, 888 { 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU | 889 1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 }, 890 { 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU | 891 1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 }, 892 { 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC }, 893 { 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC }, 894 { 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX }, 895 { 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL | 896 1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T }, 897 { 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T }, 898 { 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR }, 899 { 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR }, 900 { 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX }, 901 { 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 }, 902 { 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 }, 903 { 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX }, 904 { 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX }, 905 { 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 }, 906 { 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR }, 907 { 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR }, 908 { 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR }, 909 { 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR }, 910 { 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC }, 911 { 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC }, 912 { 1ULL << IXL_PHY_TYPE_2500BASE_T_1, IFM_2500_T }, 913 { 1ULL << IXL_PHY_TYPE_5000BASE_T_1, IFM_5000_T }, 914 { 1ULL << IXL_PHY_TYPE_2500BASE_T_2, IFM_2500_T }, 915 { 1ULL << IXL_PHY_TYPE_5000BASE_T_2, IFM_5000_T }, 916}; 917 918static const struct ixl_speed_type ixl_speed_type_map[] = { 919 { IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) }, 920 { IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) }, 921 { IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) }, 922 { IXL_AQ_LINK_SPEED_5000MB, IF_Mbps(5000) }, 923 { IXL_AQ_LINK_SPEED_2500MB, IF_Mbps(2500) }, 924 { IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) }, 925 { IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)}, 926}; 927 928static const struct ixl_aq_regs ixl_pf_aq_regs = { 929 .atq_tail = I40E_PF_ATQT, 930 .atq_tail_mask = I40E_PF_ATQT_ATQT_MASK, 931 .atq_head = I40E_PF_ATQH, 932 .atq_head_mask = I40E_PF_ATQH_ATQH_MASK, 933 .atq_len = I40E_PF_ATQLEN, 934 .atq_bal = I40E_PF_ATQBAL, 935 .atq_bah = I40E_PF_ATQBAH, 936 .atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK, 937 938 .arq_tail = I40E_PF_ARQT, 939 .arq_tail_mask = I40E_PF_ARQT_ARQT_MASK, 940 .arq_head = I40E_PF_ARQH, 941 .arq_head_mask = I40E_PF_ARQH_ARQH_MASK, 942 .arq_len = I40E_PF_ARQLEN, 943 .arq_bal = I40E_PF_ARQBAL, 944 .arq_bah = I40E_PF_ARQBAH, 945 .arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK, 946}; 947 948#define ixl_rd(_s, _r) \ 949 bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r)) 950#define ixl_wr(_s, _r, _v) \ 951 bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v)) 952#define ixl_barrier(_s, _r, _l, _o) \ 953 bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o)) 954#define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT) 955#define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1)) 956 957CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc), 958 ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL, 959 DVF_DETACH_SHUTDOWN); 960 961static const struct ixl_product ixl_products[] = { 962 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP }, 963 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B }, 964 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C }, 965 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A }, 966 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B }, 967 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C }, 968 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_1 }, 969 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_2 }, 970 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 }, 971 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 }, 972 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G }, 973 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP }, 974 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 }, 975 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX }, 976 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP }, 977 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP }, 978 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET }, 979 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET }, 980 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP }, 981 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_SFP }, 982 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_BP }, 983 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_V710_5G_T}, 984 /* required last entry */ 985 {0, 0} 986}; 987 988static const struct ixl_product * 989ixl_lookup(const struct pci_attach_args *pa) 990{ 991 const struct ixl_product *ixlp; 992 993 for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) { 994 if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id && 995 PCI_PRODUCT(pa->pa_id) == ixlp->product_id) 996 return ixlp; 997 } 998 999 return NULL; 1000} 1001 1002static void 1003ixl_intr_barrier(void) 1004{ 1005 1006 /* wait for finish of all handler */ 1007 xc_barrier(0); 1008} 1009 1010static int 1011ixl_match(device_t parent, cfdata_t match, void *aux) 1012{ 1013 const struct pci_attach_args *pa = aux; 1014 1015 return (ixl_lookup(pa) != NULL) ? 1 : 0; 1016} 1017 1018static void 1019ixl_attach(device_t parent, device_t self, void *aux) 1020{ 1021 struct ixl_softc *sc; 1022 struct pci_attach_args *pa = aux; 1023 struct ifnet *ifp; 1024 pcireg_t memtype; 1025 uint32_t firstq, port, ari, func; 1026 char xnamebuf[32]; 1027 int tries, rv, link; 1028 1029 sc = device_private(self); 1030 sc->sc_dev = self; 1031 ifp = &sc->sc_ec.ec_if; 1032 1033 sc->sc_pa = *pa; 1034 sc->sc_dmat = (pci_dma64_available(pa)) ? 1035 pa->pa_dmat64 : pa->pa_dmat; 1036 sc->sc_aq_regs = &ixl_pf_aq_regs; 1037 1038 sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id)); 1039 1040 ixl_pci_csr_setup(pa->pa_pc, pa->pa_tag); 1041 1042 pci_aprint_devinfo(pa, "Ethernet controller"); 1043 1044 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG); 1045 if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0, 1046 &sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) { 1047 aprint_error(": unable to map registers\n"); 1048 return; 1049 } 1050 1051 mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET); 1052 1053 firstq = ixl_rd(sc, I40E_PFLAN_QALLOC); 1054 firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK; 1055 firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 1056 sc->sc_base_queue = firstq; 1057 1058 ixl_clear_hw(sc); 1059 if (ixl_pf_reset(sc) == -1) { 1060 /* error printed by ixl pf_reset */ 1061 goto unmap; 1062 } 1063 1064 port = ixl_rd(sc, I40E_PFGEN_PORTNUM); 1065 port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK; 1066 port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT; 1067 sc->sc_port = port; 1068 aprint_normal_dev(self, "port %u", sc->sc_port); 1069 1070 ari = ixl_rd(sc, I40E_GLPCI_CAPSUP); 1071 ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK; 1072 ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT; 1073 1074 func = ixl_rd(sc, I40E_PF_FUNC_RID); 1075 sc->sc_pf_id = func & (ari ? 0xff : 0x7); 1076 1077 /* initialise the adminq */ 1078 1079 mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET); 1080 1081 if (ixl_dmamem_alloc(sc, &sc->sc_atq, 1082 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1083 aprint_error("\n" "%s: unable to allocate atq\n", 1084 device_xname(self)); 1085 goto unmap; 1086 } 1087 1088 SIMPLEQ_INIT(&sc->sc_arq_idle); 1089 ixl_work_set(&sc->sc_arq_task, ixl_arq, sc); 1090 sc->sc_arq_cons = 0; 1091 sc->sc_arq_prod = 0; 1092 1093 if (ixl_dmamem_alloc(sc, &sc->sc_arq, 1094 sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) { 1095 aprint_error("\n" "%s: unable to allocate arq\n", 1096 device_xname(self)); 1097 goto free_atq; 1098 } 1099 1100 if (!ixl_arq_fill(sc)) { 1101 aprint_error("\n" "%s: unable to fill arq descriptors\n", 1102 device_xname(self)); 1103 goto free_arq; 1104 } 1105 1106 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1107 0, IXL_DMA_LEN(&sc->sc_atq), 1108 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1109 1110 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1111 0, IXL_DMA_LEN(&sc->sc_arq), 1112 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1113 1114 for (tries = 0; tries < 10; tries++) { 1115 sc->sc_atq_cons = 0; 1116 sc->sc_atq_prod = 0; 1117 1118 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1119 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1120 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1121 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1122 1123 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 1124 1125 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 1126 ixl_dmamem_lo(&sc->sc_atq)); 1127 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 1128 ixl_dmamem_hi(&sc->sc_atq)); 1129 ixl_wr(sc, sc->sc_aq_regs->atq_len, 1130 sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM); 1131 1132 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 1133 ixl_dmamem_lo(&sc->sc_arq)); 1134 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 1135 ixl_dmamem_hi(&sc->sc_arq)); 1136 ixl_wr(sc, sc->sc_aq_regs->arq_len, 1137 sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM); 1138 1139 rv = ixl_get_version(sc); 1140 if (rv == 0) 1141 break; 1142 if (rv != ETIMEDOUT) { 1143 aprint_error(", unable to get firmware version\n"); 1144 goto shutdown; 1145 } 1146 1147 delaymsec(100); 1148 } 1149 1150 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 1151 1152 if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) { 1153 aprint_error_dev(self, ", unable to allocate nvm buffer\n"); 1154 goto shutdown; 1155 } 1156 1157 ixl_get_nvm_version(sc); 1158 1159 if (sc->sc_mac_type == I40E_MAC_X722) 1160 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722; 1161 else 1162 sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710; 1163 1164 rv = ixl_get_hw_capabilities(sc); 1165 if (rv != 0) { 1166 aprint_error(", GET HW CAPABILITIES %s\n", 1167 rv == ETIMEDOUT ? "timeout" : "error"); 1168 goto free_aqbuf; 1169 } 1170 1171 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu); 1172 if (ixl_param_nqps_limit > 0) { 1173 sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max, 1174 ixl_param_nqps_limit); 1175 } 1176 1177 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 1178 sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs; 1179 sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs; 1180 1181 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs); 1182 KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs); 1183 KASSERT(sc->sc_rx_ring_ndescs == 1184 (1U << (fls32(sc->sc_rx_ring_ndescs) - 1))); 1185 KASSERT(sc->sc_tx_ring_ndescs == 1186 (1U << (fls32(sc->sc_tx_ring_ndescs) - 1))); 1187 1188 if (ixl_get_mac(sc) != 0) { 1189 /* error printed by ixl_get_mac */ 1190 goto free_aqbuf; 1191 } 1192 1193 aprint_normal("\n"); 1194 aprint_naive("\n"); 1195 1196 aprint_normal_dev(self, "Ethernet address %s\n", 1197 ether_sprintf(sc->sc_enaddr)); 1198 1199 rv = ixl_pxe_clear(sc); 1200 if (rv != 0) { 1201 aprint_debug_dev(self, "CLEAR PXE MODE %s\n", 1202 rv == ETIMEDOUT ? "timeout" : "error"); 1203 } 1204 1205 ixl_set_filter_control(sc); 1206 1207 if (ixl_hmc(sc) != 0) { 1208 /* error printed by ixl_hmc */ 1209 goto free_aqbuf; 1210 } 1211 1212 if (ixl_lldp_shut(sc) != 0) { 1213 /* error printed by ixl_lldp_shut */ 1214 goto free_hmc; 1215 } 1216 1217 if (ixl_phy_mask_ints(sc) != 0) { 1218 /* error printed by ixl_phy_mask_ints */ 1219 goto free_hmc; 1220 } 1221 1222 if (ixl_restart_an(sc) != 0) { 1223 /* error printed by ixl_restart_an */ 1224 goto free_hmc; 1225 } 1226 1227 if (ixl_get_switch_config(sc) != 0) { 1228 /* error printed by ixl_get_switch_config */ 1229 goto free_hmc; 1230 } 1231 1232 rv = ixl_get_link_status_poll(sc, NULL); 1233 if (rv != 0) { 1234 aprint_error_dev(self, "GET LINK STATUS %s\n", 1235 rv == ETIMEDOUT ? "timeout" : "error"); 1236 goto free_hmc; 1237 } 1238 1239 /* 1240 * The FW often returns EIO in "Get PHY Abilities" command 1241 * if there is no delay 1242 */ 1243 DELAY(500); 1244 if (ixl_get_phy_info(sc) != 0) { 1245 /* error printed by ixl_get_phy_info */ 1246 goto free_hmc; 1247 } 1248 1249 if (ixl_dmamem_alloc(sc, &sc->sc_scratch, 1250 sizeof(struct ixl_aq_vsi_data), 8) != 0) { 1251 aprint_error_dev(self, "unable to allocate scratch buffer\n"); 1252 goto free_hmc; 1253 } 1254 1255 rv = ixl_get_vsi(sc); 1256 if (rv != 0) { 1257 aprint_error_dev(self, "GET VSI %s %d\n", 1258 rv == ETIMEDOUT ? "timeout" : "error", rv); 1259 goto free_scratch; 1260 } 1261 1262 rv = ixl_set_vsi(sc); 1263 if (rv != 0) { 1264 aprint_error_dev(self, "UPDATE VSI error %s %d\n", 1265 rv == ETIMEDOUT ? "timeout" : "error", rv); 1266 goto free_scratch; 1267 } 1268 1269 if (ixl_queue_pairs_alloc(sc) != 0) { 1270 /* error printed by ixl_queue_pairs_alloc */ 1271 goto free_scratch; 1272 } 1273 1274 if (ixl_setup_interrupts(sc) != 0) { 1275 /* error printed by ixl_setup_interrupts */ 1276 goto free_queue_pairs; 1277 } 1278 1279 if (ixl_setup_stats(sc) != 0) { 1280 aprint_error_dev(self, "failed to setup event counters\n"); 1281 goto teardown_intrs; 1282 } 1283 1284 if (ixl_setup_sysctls(sc) != 0) { 1285 /* error printed by ixl_setup_sysctls */ 1286 goto teardown_stats; 1287 } 1288 1289 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self)); 1290 sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI, 1291 IPL_NET, WQ_MPSAFE); 1292 if (sc->sc_workq == NULL) 1293 goto teardown_sysctls; 1294 1295 snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self)); 1296 rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk, 1297 sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 1298 if (rv != 0) { 1299 sc->sc_workq_txrx = NULL; 1300 goto teardown_wqs; 1301 } 1302 1303 snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self)); 1304 cv_init(&sc->sc_atq_cv, xnamebuf); 1305 1306 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 1307 1308 ifp->if_softc = sc; 1309 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1310 ifp->if_extflags = IFEF_MPSAFE; 1311 ifp->if_ioctl = ixl_ioctl; 1312 ifp->if_start = ixl_start; 1313 ifp->if_transmit = ixl_transmit; 1314 ifp->if_init = ixl_init; 1315 ifp->if_stop = ixl_stop; 1316 IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs); 1317 IFQ_SET_READY(&ifp->if_snd); 1318 ifp->if_capabilities |= IXL_IFCAP_RXCSUM; 1319 ifp->if_capabilities |= IXL_IFCAP_TXCSUM; 1320#if 0 1321 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1322#endif 1323 ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb); 1324 sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 1325 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU; 1326 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1327 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1328 1329 sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities; 1330 /* Disable VLAN_HWFILTER by default */ 1331 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1332 1333 sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable; 1334 1335 sc->sc_ec.ec_ifmedia = &sc->sc_media; 1336 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, ixl_media_change, 1337 ixl_media_status, &sc->sc_cfg_lock); 1338 1339 ixl_media_add(sc); 1340 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1341 if (ISSET(sc->sc_phy_abilities, 1342 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1343 ifmedia_add(&sc->sc_media, 1344 IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL); 1345 } 1346 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL); 1347 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 1348 1349 if_initialize(ifp); 1350 1351 sc->sc_ipq = if_percpuq_create(ifp); 1352 if_deferred_start_init(ifp, NULL); 1353 ether_ifattach(ifp, sc->sc_enaddr); 1354 ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb); 1355 1356 rv = ixl_get_link_status_poll(sc, &link); 1357 if (rv != 0) 1358 link = LINK_STATE_UNKNOWN; 1359 if_link_state_change(ifp, link); 1360 1361 ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done); 1362 ixl_work_set(&sc->sc_link_state_task, 1363 ixl_get_link_status_work, sc); 1364 ixl_work_set(&sc->sc_link_state_done_task, 1365 ixl_get_link_status_done_work, sc); 1366 1367 ixl_config_other_intr(sc); 1368 ixl_enable_other_intr(sc); 1369 1370 ixl_set_phy_autoselect(sc); 1371 1372 /* remove default mac filter and replace it so we can see vlans */ 1373 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0); 1374 if (rv != ENOENT) { 1375 aprint_debug_dev(self, 1376 "unable to remove macvlan %u\n", rv); 1377 } 1378 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 1379 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1380 if (rv != ENOENT) { 1381 aprint_debug_dev(self, 1382 "unable to remove macvlan, ignore vlan %u\n", rv); 1383 } 1384 1385 if (ixl_update_macvlan(sc) != 0) { 1386 aprint_debug_dev(self, 1387 "couldn't enable vlan hardware filter\n"); 1388 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 1389 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 1390 } 1391 1392 sc->sc_txrx_workqueue = true; 1393 sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT; 1394 sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT; 1395 sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT; 1396 sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT; 1397 1398 ixl_stats_update(sc); 1399 sc->sc_stats_counters.isc_has_offset = true; 1400 1401 if (pmf_device_register(self, NULL, NULL) != true) 1402 aprint_debug_dev(self, "couldn't establish power handler\n"); 1403 sc->sc_itr_rx = IXL_ITR_RX; 1404 sc->sc_itr_tx = IXL_ITR_TX; 1405 sc->sc_attached = true; 1406 if_register(ifp); 1407 1408 return; 1409 1410teardown_wqs: 1411 config_finalize_register(self, ixl_workqs_teardown); 1412teardown_sysctls: 1413 ixl_teardown_sysctls(sc); 1414teardown_stats: 1415 ixl_teardown_stats(sc); 1416teardown_intrs: 1417 ixl_teardown_interrupts(sc); 1418free_queue_pairs: 1419 ixl_queue_pairs_free(sc); 1420free_scratch: 1421 ixl_dmamem_free(sc, &sc->sc_scratch); 1422free_hmc: 1423 ixl_hmc_free(sc); 1424free_aqbuf: 1425 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1426shutdown: 1427 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1428 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1429 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1430 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1431 1432 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1433 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1434 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1435 1436 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1437 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1438 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1439 1440 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1441 0, IXL_DMA_LEN(&sc->sc_arq), 1442 BUS_DMASYNC_POSTREAD); 1443 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1444 0, IXL_DMA_LEN(&sc->sc_atq), 1445 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1446 1447 ixl_arq_unfill(sc); 1448free_arq: 1449 ixl_dmamem_free(sc, &sc->sc_arq); 1450free_atq: 1451 ixl_dmamem_free(sc, &sc->sc_atq); 1452unmap: 1453 mutex_destroy(&sc->sc_atq_lock); 1454 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1455 mutex_destroy(&sc->sc_cfg_lock); 1456 sc->sc_mems = 0; 1457 1458 sc->sc_attached = false; 1459} 1460 1461static int 1462ixl_detach(device_t self, int flags) 1463{ 1464 struct ixl_softc *sc = device_private(self); 1465 struct ifnet *ifp = &sc->sc_ec.ec_if; 1466 1467 if (!sc->sc_attached) 1468 return 0; 1469 1470 ixl_stop(ifp, 1); 1471 1472 callout_halt(&sc->sc_stats_callout, NULL); 1473 ixl_work_wait(sc->sc_workq, &sc->sc_stats_task); 1474 1475 /* detach the I/F before stop adminq due to callbacks */ 1476 ether_ifdetach(ifp); 1477 if_detach(ifp); 1478 ifmedia_fini(&sc->sc_media); 1479 if_percpuq_destroy(sc->sc_ipq); 1480 1481 ixl_disable_other_intr(sc); 1482 ixl_intr_barrier(); 1483 ixl_work_wait(sc->sc_workq, &sc->sc_arq_task); 1484 ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task); 1485 1486 if (sc->sc_workq != NULL) { 1487 ixl_workq_destroy(sc->sc_workq); 1488 sc->sc_workq = NULL; 1489 } 1490 1491 if (sc->sc_workq_txrx != NULL) { 1492 workqueue_destroy(sc->sc_workq_txrx); 1493 sc->sc_workq_txrx = NULL; 1494 } 1495 1496 ixl_teardown_interrupts(sc); 1497 ixl_teardown_stats(sc); 1498 ixl_teardown_sysctls(sc); 1499 1500 ixl_queue_pairs_free(sc); 1501 1502 ixl_dmamem_free(sc, &sc->sc_scratch); 1503 ixl_hmc_free(sc); 1504 1505 /* shutdown */ 1506 ixl_wr(sc, sc->sc_aq_regs->atq_head, 0); 1507 ixl_wr(sc, sc->sc_aq_regs->arq_head, 0); 1508 ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0); 1509 ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0); 1510 1511 ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0); 1512 ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0); 1513 ixl_wr(sc, sc->sc_aq_regs->atq_len, 0); 1514 1515 ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0); 1516 ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0); 1517 ixl_wr(sc, sc->sc_aq_regs->arq_len, 0); 1518 1519 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 1520 0, IXL_DMA_LEN(&sc->sc_arq), 1521 BUS_DMASYNC_POSTREAD); 1522 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 1523 0, IXL_DMA_LEN(&sc->sc_atq), 1524 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1525 1526 ixl_arq_unfill(sc); 1527 1528 ixl_dmamem_free(sc, &sc->sc_arq); 1529 ixl_dmamem_free(sc, &sc->sc_atq); 1530 ixl_dmamem_free(sc, &sc->sc_aqbuf); 1531 1532 cv_destroy(&sc->sc_atq_cv); 1533 mutex_destroy(&sc->sc_atq_lock); 1534 1535 if (sc->sc_mems != 0) { 1536 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1537 sc->sc_mems = 0; 1538 } 1539 1540 mutex_destroy(&sc->sc_cfg_lock); 1541 1542 return 0; 1543} 1544 1545static int 1546ixl_workqs_teardown(device_t self) 1547{ 1548 struct ixl_softc *sc = device_private(self); 1549 1550 if (sc->sc_workq != NULL) { 1551 ixl_workq_destroy(sc->sc_workq); 1552 sc->sc_workq = NULL; 1553 } 1554 1555 if (sc->sc_workq_txrx != NULL) { 1556 workqueue_destroy(sc->sc_workq_txrx); 1557 sc->sc_workq_txrx = NULL; 1558 } 1559 1560 return 0; 1561} 1562 1563static int 1564ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 1565{ 1566 struct ifnet *ifp = &ec->ec_if; 1567 struct ixl_softc *sc = ifp->if_softc; 1568 int rv; 1569 1570 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 1571 return 0; 1572 } 1573 1574 if (set) { 1575 rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid, 1576 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1577 if (rv == 0) { 1578 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 1579 vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 1580 } 1581 } else { 1582 rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid, 1583 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1584 (void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid, 1585 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 1586 } 1587 1588 return rv; 1589} 1590 1591static void 1592ixl_media_add(struct ixl_softc *sc) 1593{ 1594 struct ifmedia *ifm = &sc->sc_media; 1595 const struct ixl_phy_type *itype; 1596 unsigned int i; 1597 bool flow; 1598 1599 if (ISSET(sc->sc_phy_abilities, 1600 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) { 1601 flow = true; 1602 } else { 1603 flow = false; 1604 } 1605 1606 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 1607 itype = &ixl_phy_type_map[i]; 1608 1609 if (ISSET(sc->sc_phy_types, itype->phy_type)) { 1610 ifmedia_add(ifm, 1611 IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL); 1612 1613 if (flow) { 1614 ifmedia_add(ifm, 1615 IFM_ETHER | IFM_FDX | IFM_FLOW | 1616 itype->ifm_type, 0, NULL); 1617 } 1618 1619 if (itype->ifm_type != IFM_100_TX) 1620 continue; 1621 1622 ifmedia_add(ifm, IFM_ETHER | itype->ifm_type, 1623 0, NULL); 1624 if (flow) { 1625 ifmedia_add(ifm, 1626 IFM_ETHER | IFM_FLOW | itype->ifm_type, 1627 0, NULL); 1628 } 1629 } 1630 } 1631} 1632 1633static void 1634ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1635{ 1636 struct ixl_softc *sc = ifp->if_softc; 1637 1638 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1639 1640 ifmr->ifm_status = sc->sc_media_status; 1641 ifmr->ifm_active = sc->sc_media_active; 1642} 1643 1644static int 1645ixl_media_change(struct ifnet *ifp) 1646{ 1647 struct ixl_softc *sc = ifp->if_softc; 1648 struct ifmedia *ifm = &sc->sc_media; 1649 uint64_t ifm_active = sc->sc_media_active; 1650 uint8_t link_speed, abilities; 1651 1652 switch (IFM_SUBTYPE(ifm_active)) { 1653 case IFM_1000_SGMII: 1654 case IFM_1000_KX: 1655 case IFM_10G_KX4: 1656 case IFM_10G_KR: 1657 case IFM_40G_KR4: 1658 case IFM_20G_KR2: 1659 case IFM_25G_KR: 1660 /* backplanes */ 1661 return EINVAL; 1662 } 1663 1664 abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP; 1665 1666 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1667 case IFM_AUTO: 1668 link_speed = sc->sc_phy_linkspeed; 1669 break; 1670 case IFM_NONE: 1671 link_speed = 0; 1672 CLR(abilities, IXL_PHY_ABILITY_LINKUP); 1673 break; 1674 default: 1675 link_speed = ixl_search_baudrate( 1676 ifmedia_baudrate(ifm->ifm_media)); 1677 } 1678 1679 if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) { 1680 if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0) 1681 return EINVAL; 1682 } 1683 1684 if (ifm->ifm_media & IFM_FLOW) { 1685 abilities |= sc->sc_phy_abilities & 1686 (IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX); 1687 } 1688 1689 return ixl_set_phy_config(sc, link_speed, abilities, false); 1690} 1691 1692 1693static void 1694ixl_del_all_multiaddr(struct ixl_softc *sc) 1695{ 1696 struct ethercom *ec = &sc->sc_ec; 1697 struct ether_multi *enm; 1698 struct ether_multistep step; 1699 1700 ETHER_LOCK(ec); 1701 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1702 ETHER_NEXT_MULTI(step, enm)) { 1703 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1704 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1705 } 1706 ETHER_UNLOCK(ec); 1707} 1708 1709static int 1710ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1711{ 1712 struct ifnet *ifp = &sc->sc_ec.ec_if; 1713 int rv; 1714 1715 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) 1716 return 0; 1717 1718 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) { 1719 ixl_del_all_multiaddr(sc); 1720 SET(ifp->if_flags, IFF_ALLMULTI); 1721 return ENETRESET; 1722 } 1723 1724 /* multicast address can not use VLAN HWFILTER */ 1725 rv = ixl_add_macvlan(sc, addrlo, 0, 1726 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1727 1728 if (rv == ENOSPC) { 1729 ixl_del_all_multiaddr(sc); 1730 SET(ifp->if_flags, IFF_ALLMULTI); 1731 return ENETRESET; 1732 } 1733 1734 return rv; 1735} 1736 1737static int 1738ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi) 1739{ 1740 struct ifnet *ifp = &sc->sc_ec.ec_if; 1741 struct ethercom *ec = &sc->sc_ec; 1742 struct ether_multi *enm, *enm_last; 1743 struct ether_multistep step; 1744 int error, rv = 0; 1745 1746 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 1747 ixl_remove_macvlan(sc, addrlo, 0, 1748 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1749 return 0; 1750 } 1751 1752 ETHER_LOCK(ec); 1753 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1754 ETHER_NEXT_MULTI(step, enm)) { 1755 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1756 ETHER_ADDR_LEN) != 0) { 1757 goto out; 1758 } 1759 } 1760 1761 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1762 ETHER_NEXT_MULTI(step, enm)) { 1763 error = ixl_add_macvlan(sc, enm->enm_addrlo, 0, 1764 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 1765 if (error != 0) 1766 break; 1767 } 1768 1769 if (enm != NULL) { 1770 enm_last = enm; 1771 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL; 1772 ETHER_NEXT_MULTI(step, enm)) { 1773 if (enm == enm_last) 1774 break; 1775 1776 ixl_remove_macvlan(sc, enm->enm_addrlo, 0, 1777 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 1778 } 1779 } else { 1780 CLR(ifp->if_flags, IFF_ALLMULTI); 1781 rv = ENETRESET; 1782 } 1783 1784out: 1785 ETHER_UNLOCK(ec); 1786 return rv; 1787} 1788 1789static int 1790ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1791{ 1792 struct ifreq *ifr = (struct ifreq *)data; 1793 struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc; 1794 const struct sockaddr *sa; 1795 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 1796 int s, error = 0; 1797 unsigned int nmtu; 1798 1799 switch (cmd) { 1800 case SIOCSIFMTU: 1801 nmtu = ifr->ifr_mtu; 1802 1803 if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) { 1804 error = EINVAL; 1805 break; 1806 } 1807 if (ifp->if_mtu != nmtu) { 1808 s = splnet(); 1809 error = ether_ioctl(ifp, cmd, data); 1810 splx(s); 1811 if (error == ENETRESET) 1812 error = ixl_init(ifp); 1813 } 1814 break; 1815 case SIOCADDMULTI: 1816 sa = ifreq_getaddr(SIOCADDMULTI, ifr); 1817 if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) { 1818 error = ether_multiaddr(sa, addrlo, addrhi); 1819 if (error != 0) 1820 return error; 1821 1822 error = ixl_add_multi(sc, addrlo, addrhi); 1823 if (error != 0 && error != ENETRESET) { 1824 ether_delmulti(sa, &sc->sc_ec); 1825 error = EIO; 1826 } 1827 } 1828 break; 1829 1830 case SIOCDELMULTI: 1831 sa = ifreq_getaddr(SIOCDELMULTI, ifr); 1832 if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) { 1833 error = ether_multiaddr(sa, addrlo, addrhi); 1834 if (error != 0) 1835 return error; 1836 1837 error = ixl_del_multi(sc, addrlo, addrhi); 1838 } 1839 break; 1840 1841 default: 1842 s = splnet(); 1843 error = ether_ioctl(ifp, cmd, data); 1844 splx(s); 1845 } 1846 1847 if (error == ENETRESET) 1848 error = ixl_iff(sc); 1849 1850 return error; 1851} 1852 1853static enum i40e_mac_type 1854ixl_mactype(pci_product_id_t id) 1855{ 1856 1857 switch (id) { 1858 case PCI_PRODUCT_INTEL_XL710_SFP: 1859 case PCI_PRODUCT_INTEL_XL710_KX_B: 1860 case PCI_PRODUCT_INTEL_XL710_KX_C: 1861 case PCI_PRODUCT_INTEL_XL710_QSFP_A: 1862 case PCI_PRODUCT_INTEL_XL710_QSFP_B: 1863 case PCI_PRODUCT_INTEL_XL710_QSFP_C: 1864 case PCI_PRODUCT_INTEL_X710_10G_T_1: 1865 case PCI_PRODUCT_INTEL_X710_10G_T_2: 1866 case PCI_PRODUCT_INTEL_XL710_20G_BP_1: 1867 case PCI_PRODUCT_INTEL_XL710_20G_BP_2: 1868 case PCI_PRODUCT_INTEL_X710_T4_10G: 1869 case PCI_PRODUCT_INTEL_XXV710_25G_BP: 1870 case PCI_PRODUCT_INTEL_XXV710_25G_SFP28: 1871 case PCI_PRODUCT_INTEL_X710_10G_SFP: 1872 case PCI_PRODUCT_INTEL_X710_10G_BP: 1873 return I40E_MAC_XL710; 1874 1875 case PCI_PRODUCT_INTEL_X722_KX: 1876 case PCI_PRODUCT_INTEL_X722_QSFP: 1877 case PCI_PRODUCT_INTEL_X722_SFP: 1878 case PCI_PRODUCT_INTEL_X722_1G_BASET: 1879 case PCI_PRODUCT_INTEL_X722_10G_BASET: 1880 case PCI_PRODUCT_INTEL_X722_I_SFP: 1881 return I40E_MAC_X722; 1882 } 1883 1884 return I40E_MAC_GENERIC; 1885} 1886 1887static void 1888ixl_pci_csr_setup(pci_chipset_tag_t pc, pcitag_t tag) 1889{ 1890 pcireg_t csr; 1891 1892 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1893 csr |= (PCI_COMMAND_MASTER_ENABLE | 1894 PCI_COMMAND_MEM_ENABLE); 1895 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 1896} 1897 1898static inline void * 1899ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i) 1900{ 1901 uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 1902 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1903 1904 if (i >= e->hmc_count) 1905 return NULL; 1906 1907 kva += e->hmc_base; 1908 kva += i * e->hmc_size; 1909 1910 return kva; 1911} 1912 1913static inline size_t 1914ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type) 1915{ 1916 struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type]; 1917 1918 return e->hmc_size; 1919} 1920 1921static void 1922ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1923{ 1924 struct ixl_rx_ring *rxr = qp->qp_rxr; 1925 1926 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1927 I40E_PFINT_DYN_CTLN_INTENA_MASK | 1928 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 1929 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1930 ixl_flush(sc); 1931} 1932 1933static void 1934ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp) 1935{ 1936 struct ixl_rx_ring *rxr = qp->qp_rxr; 1937 1938 ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid), 1939 (IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 1940 ixl_flush(sc); 1941} 1942 1943static void 1944ixl_enable_other_intr(struct ixl_softc *sc) 1945{ 1946 1947 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1948 I40E_PFINT_DYN_CTL0_INTENA_MASK | 1949 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 1950 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1951 ixl_flush(sc); 1952} 1953 1954static void 1955ixl_disable_other_intr(struct ixl_softc *sc) 1956{ 1957 1958 ixl_wr(sc, I40E_PFINT_DYN_CTL0, 1959 (IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT)); 1960 ixl_flush(sc); 1961} 1962 1963static int 1964ixl_reinit(struct ixl_softc *sc) 1965{ 1966 struct ixl_rx_ring *rxr; 1967 struct ixl_tx_ring *txr; 1968 unsigned int i; 1969 uint32_t reg; 1970 1971 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 1972 1973 if (ixl_get_vsi(sc) != 0) 1974 return EIO; 1975 1976 if (ixl_set_vsi(sc) != 0) 1977 return EIO; 1978 1979 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1980 txr = sc->sc_qps[i].qp_txr; 1981 rxr = sc->sc_qps[i].qp_rxr; 1982 1983 ixl_txr_config(sc, txr); 1984 ixl_rxr_config(sc, rxr); 1985 } 1986 1987 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 1988 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE); 1989 1990 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 1991 txr = sc->sc_qps[i].qp_txr; 1992 rxr = sc->sc_qps[i].qp_rxr; 1993 1994 ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE | 1995 (sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT)); 1996 ixl_flush(sc); 1997 1998 ixl_wr(sc, txr->txr_tail, txr->txr_prod); 1999 ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod); 2000 2001 /* ixl_rxfill() needs lock held */ 2002 mutex_enter(&rxr->rxr_lock); 2003 ixl_rxfill(sc, rxr); 2004 mutex_exit(&rxr->rxr_lock); 2005 2006 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2007 SET(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2008 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2009 if (ixl_rxr_enabled(sc, rxr) != 0) 2010 goto stop; 2011 2012 ixl_txr_qdis(sc, txr, 1); 2013 2014 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2015 SET(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2016 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2017 2018 if (ixl_txr_enabled(sc, txr) != 0) 2019 goto stop; 2020 } 2021 2022 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2023 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2024 2025 return 0; 2026 2027stop: 2028 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 2029 0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE); 2030 2031 return ETIMEDOUT; 2032} 2033 2034static int 2035ixl_init_locked(struct ixl_softc *sc) 2036{ 2037 struct ifnet *ifp = &sc->sc_ec.ec_if; 2038 unsigned int i; 2039 int error, eccap_change; 2040 2041 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2042 2043 if (ISSET(ifp->if_flags, IFF_RUNNING)) 2044 ixl_stop_locked(sc); 2045 2046 if (sc->sc_dead) { 2047 return ENXIO; 2048 } 2049 2050 eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable; 2051 if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING)) 2052 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING; 2053 2054 if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) { 2055 if (ixl_update_macvlan(sc) == 0) { 2056 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 2057 } else { 2058 CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER); 2059 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 2060 } 2061 } 2062 2063 if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX) 2064 sc->sc_nqueue_pairs = 1; 2065 else 2066 sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max; 2067 2068 error = ixl_reinit(sc); 2069 if (error) { 2070 ixl_stop_locked(sc); 2071 return error; 2072 } 2073 2074 SET(ifp->if_flags, IFF_RUNNING); 2075 CLR(ifp->if_flags, IFF_OACTIVE); 2076 2077 ixl_config_rss(sc); 2078 ixl_config_queue_intr(sc); 2079 2080 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2081 ixl_enable_queue_intr(sc, &sc->sc_qps[i]); 2082 } 2083 2084 error = ixl_iff(sc); 2085 if (error) { 2086 ixl_stop_locked(sc); 2087 return error; 2088 } 2089 2090 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 2091 2092 return 0; 2093} 2094 2095static int 2096ixl_init(struct ifnet *ifp) 2097{ 2098 struct ixl_softc *sc = ifp->if_softc; 2099 int error; 2100 2101 mutex_enter(&sc->sc_cfg_lock); 2102 error = ixl_init_locked(sc); 2103 mutex_exit(&sc->sc_cfg_lock); 2104 2105 if (error == 0) { 2106 error = ixl_get_link_status(sc, 2107 IXL_LINK_FLAG_WAITDONE); 2108 } 2109 2110 return error; 2111} 2112 2113static int 2114ixl_iff(struct ixl_softc *sc) 2115{ 2116 struct ifnet *ifp = &sc->sc_ec.ec_if; 2117 struct ixl_atq iatq; 2118 struct ixl_aq_desc *iaq; 2119 struct ixl_aq_vsi_promisc_param *param; 2120 uint16_t flag_add, flag_del; 2121 int error; 2122 2123 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2124 return 0; 2125 2126 memset(&iatq, 0, sizeof(iatq)); 2127 2128 iaq = &iatq.iatq_desc; 2129 iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC); 2130 2131 param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param; 2132 param->flags = htole16(0); 2133 2134 if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER) 2135 || ISSET(ifp->if_flags, IFF_PROMISC)) { 2136 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2137 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2138 } 2139 2140 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 2141 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2142 IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2143 } else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 2144 param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST); 2145 } 2146 param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST | 2147 IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST | 2148 IXL_AQ_VSI_PROMISC_FLAG_VLAN); 2149 param->seid = sc->sc_seid; 2150 2151 error = ixl_atq_exec(sc, &iatq); 2152 if (error) 2153 return error; 2154 2155 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) 2156 return EIO; 2157 2158 if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) { 2159 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 2160 flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH; 2161 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH; 2162 } else { 2163 flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN; 2164 flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN; 2165 } 2166 2167 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del); 2168 2169 memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN); 2170 ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add); 2171 } 2172 return 0; 2173} 2174 2175static void 2176ixl_stop_locked(struct ixl_softc *sc) 2177{ 2178 struct ifnet *ifp = &sc->sc_ec.ec_if; 2179 struct ixl_rx_ring *rxr; 2180 struct ixl_tx_ring *txr; 2181 unsigned int i; 2182 uint32_t reg; 2183 2184 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 2185 2186 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 2187 callout_stop(&sc->sc_stats_callout); 2188 2189 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2190 txr = sc->sc_qps[i].qp_txr; 2191 rxr = sc->sc_qps[i].qp_rxr; 2192 2193 ixl_disable_queue_intr(sc, &sc->sc_qps[i]); 2194 2195 mutex_enter(&txr->txr_lock); 2196 ixl_txr_qdis(sc, txr, 0); 2197 mutex_exit(&txr->txr_lock); 2198 } 2199 2200 /* XXX wait at least 400 usec for all tx queues in one go */ 2201 ixl_flush(sc); 2202 DELAY(500); 2203 2204 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2205 txr = sc->sc_qps[i].qp_txr; 2206 rxr = sc->sc_qps[i].qp_rxr; 2207 2208 mutex_enter(&txr->txr_lock); 2209 reg = ixl_rd(sc, I40E_QTX_ENA(i)); 2210 CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK); 2211 ixl_wr(sc, I40E_QTX_ENA(i), reg); 2212 mutex_exit(&txr->txr_lock); 2213 2214 mutex_enter(&rxr->rxr_lock); 2215 reg = ixl_rd(sc, I40E_QRX_ENA(i)); 2216 CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK); 2217 ixl_wr(sc, I40E_QRX_ENA(i), reg); 2218 mutex_exit(&rxr->rxr_lock); 2219 } 2220 2221 /* XXX short wait for all queue disables to settle */ 2222 ixl_flush(sc); 2223 DELAY(50); 2224 2225 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2226 txr = sc->sc_qps[i].qp_txr; 2227 rxr = sc->sc_qps[i].qp_rxr; 2228 2229 mutex_enter(&txr->txr_lock); 2230 if (ixl_txr_disabled(sc, txr) != 0) { 2231 mutex_exit(&txr->txr_lock); 2232 goto die; 2233 } 2234 mutex_exit(&txr->txr_lock); 2235 2236 mutex_enter(&rxr->rxr_lock); 2237 if (ixl_rxr_disabled(sc, rxr) != 0) { 2238 mutex_exit(&rxr->rxr_lock); 2239 goto die; 2240 } 2241 mutex_exit(&rxr->rxr_lock); 2242 } 2243 2244 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2245 sc->sc_qps[i].qp_workqueue = false; 2246 workqueue_wait(sc->sc_workq_txrx, 2247 &sc->sc_qps[i].qp_work); 2248 } 2249 2250 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 2251 txr = sc->sc_qps[i].qp_txr; 2252 rxr = sc->sc_qps[i].qp_rxr; 2253 2254 mutex_enter(&txr->txr_lock); 2255 ixl_txr_unconfig(sc, txr); 2256 mutex_exit(&txr->txr_lock); 2257 2258 mutex_enter(&rxr->rxr_lock); 2259 ixl_rxr_unconfig(sc, rxr); 2260 mutex_exit(&rxr->rxr_lock); 2261 2262 ixl_txr_clean(sc, txr); 2263 ixl_rxr_clean(sc, rxr); 2264 } 2265 2266 return; 2267die: 2268 sc->sc_dead = true; 2269 log(LOG_CRIT, "%s: failed to shut down rings", 2270 device_xname(sc->sc_dev)); 2271 return; 2272} 2273 2274static void 2275ixl_stop(struct ifnet *ifp, int disable) 2276{ 2277 struct ixl_softc *sc = ifp->if_softc; 2278 2279 mutex_enter(&sc->sc_cfg_lock); 2280 ixl_stop_locked(sc); 2281 mutex_exit(&sc->sc_cfg_lock); 2282} 2283 2284static int 2285ixl_queue_pairs_alloc(struct ixl_softc *sc) 2286{ 2287 struct ixl_queue_pair *qp; 2288 unsigned int i; 2289 size_t sz; 2290 2291 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2292 sc->sc_qps = kmem_zalloc(sz, KM_SLEEP); 2293 2294 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2295 qp = &sc->sc_qps[i]; 2296 2297 qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2298 ixl_handle_queue, qp); 2299 if (qp->qp_si == NULL) 2300 goto free; 2301 2302 qp->qp_txr = ixl_txr_alloc(sc, i); 2303 if (qp->qp_txr == NULL) 2304 goto free; 2305 2306 qp->qp_rxr = ixl_rxr_alloc(sc, i); 2307 if (qp->qp_rxr == NULL) 2308 goto free; 2309 2310 qp->qp_sc = sc; 2311 snprintf(qp->qp_name, sizeof(qp->qp_name), 2312 "%s-TXRX%d", device_xname(sc->sc_dev), i); 2313 } 2314 2315 return 0; 2316free: 2317 if (sc->sc_qps != NULL) { 2318 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2319 qp = &sc->sc_qps[i]; 2320 2321 if (qp->qp_txr != NULL) 2322 ixl_txr_free(sc, qp->qp_txr); 2323 if (qp->qp_rxr != NULL) 2324 ixl_rxr_free(sc, qp->qp_rxr); 2325 if (qp->qp_si != NULL) 2326 softint_disestablish(qp->qp_si); 2327 } 2328 2329 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2330 kmem_free(sc->sc_qps, sz); 2331 sc->sc_qps = NULL; 2332 } 2333 2334 return -1; 2335} 2336 2337static void 2338ixl_queue_pairs_free(struct ixl_softc *sc) 2339{ 2340 struct ixl_queue_pair *qp; 2341 unsigned int i; 2342 size_t sz; 2343 2344 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 2345 qp = &sc->sc_qps[i]; 2346 ixl_txr_free(sc, qp->qp_txr); 2347 ixl_rxr_free(sc, qp->qp_rxr); 2348 softint_disestablish(qp->qp_si); 2349 } 2350 2351 sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max; 2352 kmem_free(sc->sc_qps, sz); 2353 sc->sc_qps = NULL; 2354} 2355 2356static struct ixl_tx_ring * 2357ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid) 2358{ 2359 struct ixl_tx_ring *txr = NULL; 2360 struct ixl_tx_map *maps = NULL, *txm; 2361 unsigned int i; 2362 2363 txr = kmem_zalloc(sizeof(*txr), KM_SLEEP); 2364 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs, 2365 KM_SLEEP); 2366 2367 if (ixl_dmamem_alloc(sc, &txr->txr_mem, 2368 sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs, 2369 IXL_TX_QUEUE_ALIGN) != 0) 2370 goto free; 2371 2372 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2373 txm = &maps[i]; 2374 2375 if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE, 2376 IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0, 2377 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0) 2378 goto uncreate; 2379 2380 txm->txm_eop = -1; 2381 txm->txm_m = NULL; 2382 } 2383 2384 txr->txr_cons = txr->txr_prod = 0; 2385 txr->txr_maps = maps; 2386 2387 txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP); 2388 if (txr->txr_intrq == NULL) 2389 goto uncreate; 2390 2391 txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 2392 ixl_deferred_transmit, txr); 2393 if (txr->txr_si == NULL) 2394 goto destroy_pcq; 2395 2396 txr->txr_tail = I40E_QTX_TAIL(qid); 2397 txr->txr_qid = qid; 2398 txr->txr_sc = sc; 2399 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2400 2401 return txr; 2402 2403destroy_pcq: 2404 pcq_destroy(txr->txr_intrq); 2405uncreate: 2406 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2407 txm = &maps[i]; 2408 2409 if (txm->txm_map == NULL) 2410 continue; 2411 2412 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2413 } 2414 2415 ixl_dmamem_free(sc, &txr->txr_mem); 2416free: 2417 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2418 kmem_free(txr, sizeof(*txr)); 2419 2420 return NULL; 2421} 2422 2423static void 2424ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable) 2425{ 2426 unsigned int qid; 2427 bus_size_t reg; 2428 uint32_t r; 2429 2430 qid = txr->txr_qid + sc->sc_base_queue; 2431 reg = I40E_GLLAN_TXPRE_QDIS(qid / 128); 2432 qid %= 128; 2433 2434 r = ixl_rd(sc, reg); 2435 CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK); 2436 SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 2437 SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK : 2438 I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK); 2439 ixl_wr(sc, reg, r); 2440} 2441 2442static void 2443ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2444{ 2445 struct ixl_hmc_txq txq; 2446 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch); 2447 void *hmc; 2448 2449 memset(&txq, 0, sizeof(txq)); 2450 txq.head = htole16(txr->txr_cons); 2451 txq.new_context = 1; 2452 txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT); 2453 txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB; 2454 txq.qlen = htole16(sc->sc_tx_ring_ndescs); 2455 txq.tphrdesc_ena = 0; 2456 txq.tphrpacket_ena = 0; 2457 txq.tphwdesc_ena = 0; 2458 txq.rdylist = data->qs_handle[0]; 2459 2460 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2461 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2462 ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq, 2463 __arraycount(ixl_hmc_pack_txq)); 2464} 2465 2466static void 2467ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2468{ 2469 void *hmc; 2470 2471 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid); 2472 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX)); 2473 txr->txr_cons = txr->txr_prod = 0; 2474} 2475 2476static void 2477ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2478{ 2479 struct ixl_tx_map *maps, *txm; 2480 bus_dmamap_t map; 2481 unsigned int i; 2482 2483 maps = txr->txr_maps; 2484 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2485 txm = &maps[i]; 2486 2487 if (txm->txm_m == NULL) 2488 continue; 2489 2490 map = txm->txm_map; 2491 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2492 BUS_DMASYNC_POSTWRITE); 2493 bus_dmamap_unload(sc->sc_dmat, map); 2494 2495 m_freem(txm->txm_m); 2496 txm->txm_m = NULL; 2497 } 2498} 2499 2500static int 2501ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2502{ 2503 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2504 uint32_t reg; 2505 int i; 2506 2507 for (i = 0; i < 10; i++) { 2508 reg = ixl_rd(sc, ena); 2509 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK)) 2510 return 0; 2511 2512 delaymsec(10); 2513 } 2514 2515 return ETIMEDOUT; 2516} 2517 2518static int 2519ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2520{ 2521 bus_size_t ena = I40E_QTX_ENA(txr->txr_qid); 2522 uint32_t reg; 2523 int i; 2524 2525 KASSERT(mutex_owned(&txr->txr_lock)); 2526 2527 for (i = 0; i < 10; i++) { 2528 reg = ixl_rd(sc, ena); 2529 if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2530 return 0; 2531 2532 delaymsec(10); 2533 } 2534 2535 return ETIMEDOUT; 2536} 2537 2538static void 2539ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr) 2540{ 2541 struct ixl_tx_map *maps, *txm; 2542 struct mbuf *m; 2543 unsigned int i; 2544 2545 softint_disestablish(txr->txr_si); 2546 2547 maps = txr->txr_maps; 2548 for (i = 0; i < sc->sc_tx_ring_ndescs; i++) { 2549 txm = &maps[i]; 2550 2551 bus_dmamap_destroy(sc->sc_dmat, txm->txm_map); 2552 } 2553 2554 while ((m = pcq_get(txr->txr_intrq)) != NULL) 2555 m_freem(m); 2556 pcq_destroy(txr->txr_intrq); 2557 2558 ixl_dmamem_free(sc, &txr->txr_mem); 2559 mutex_destroy(&txr->txr_lock); 2560 kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs); 2561 kmem_free(txr, sizeof(*txr)); 2562} 2563 2564static inline int 2565ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0, 2566 struct ixl_tx_ring *txr) 2567{ 2568 struct mbuf *m; 2569 int error; 2570 2571 KASSERT(mutex_owned(&txr->txr_lock)); 2572 2573 m = *m0; 2574 2575 error = bus_dmamap_load_mbuf(dmat, map, m, 2576 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2577 if (error != EFBIG) 2578 return error; 2579 2580 m = m_defrag(m, M_DONTWAIT); 2581 if (m != NULL) { 2582 *m0 = m; 2583 txr->txr_defragged.ev_count++; 2584 2585 error = bus_dmamap_load_mbuf(dmat, map, m, 2586 BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT); 2587 } else { 2588 txr->txr_defrag_failed.ev_count++; 2589 error = ENOBUFS; 2590 } 2591 2592 return error; 2593} 2594 2595static inline int 2596ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd) 2597{ 2598 struct ether_header *eh; 2599 size_t len; 2600 uint64_t cmd; 2601 2602 cmd = 0; 2603 2604 eh = mtod(m, struct ether_header *); 2605 switch (htons(eh->ether_type)) { 2606 case ETHERTYPE_IP: 2607 case ETHERTYPE_IPV6: 2608 len = ETHER_HDR_LEN; 2609 break; 2610 case ETHERTYPE_VLAN: 2611 len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2612 break; 2613 default: 2614 len = 0; 2615 } 2616 cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT); 2617 2618 if (m->m_pkthdr.csum_flags & 2619 (M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 2620 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4; 2621 } 2622 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) { 2623 cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM; 2624 } 2625 2626 if (m->m_pkthdr.csum_flags & 2627 (M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) { 2628 cmd |= IXL_TX_DESC_CMD_IIPT_IPV6; 2629 } 2630 2631 switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) { 2632 case IXL_TX_DESC_CMD_IIPT_IPV4: 2633 case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM: 2634 len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data); 2635 break; 2636 case IXL_TX_DESC_CMD_IIPT_IPV6: 2637 len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data); 2638 break; 2639 default: 2640 len = 0; 2641 } 2642 cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT); 2643 2644 if (m->m_pkthdr.csum_flags & 2645 (M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) { 2646 len = sizeof(struct tcphdr); 2647 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP; 2648 } else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) { 2649 len = sizeof(struct udphdr); 2650 cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP; 2651 } else { 2652 len = 0; 2653 } 2654 cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT); 2655 2656 *cmd_txd |= cmd; 2657 return 0; 2658} 2659 2660static void 2661ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr, 2662 bool is_transmit) 2663{ 2664 struct ixl_softc *sc = ifp->if_softc; 2665 struct ixl_tx_desc *ring, *txd; 2666 struct ixl_tx_map *txm; 2667 bus_dmamap_t map; 2668 struct mbuf *m; 2669 uint64_t cmd, cmd_txd; 2670 unsigned int prod, free, last, i; 2671 unsigned int mask; 2672 int post = 0; 2673 2674 KASSERT(mutex_owned(&txr->txr_lock)); 2675 2676 if (!ISSET(ifp->if_flags, IFF_RUNNING) 2677 || (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) { 2678 if (!is_transmit) 2679 IFQ_PURGE(&ifp->if_snd); 2680 return; 2681 } 2682 2683 prod = txr->txr_prod; 2684 free = txr->txr_cons; 2685 if (free <= prod) 2686 free += sc->sc_tx_ring_ndescs; 2687 free -= prod; 2688 2689 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2690 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE); 2691 2692 ring = IXL_DMA_KVA(&txr->txr_mem); 2693 mask = sc->sc_tx_ring_ndescs - 1; 2694 last = prod; 2695 cmd = 0; 2696 txd = NULL; 2697 2698 for (;;) { 2699 if (free <= IXL_TX_PKT_DESCS) { 2700 if (!is_transmit) 2701 SET(ifp->if_flags, IFF_OACTIVE); 2702 break; 2703 } 2704 2705 if (is_transmit) 2706 m = pcq_get(txr->txr_intrq); 2707 else 2708 IFQ_DEQUEUE(&ifp->if_snd, m); 2709 2710 if (m == NULL) 2711 break; 2712 2713 txm = &txr->txr_maps[prod]; 2714 map = txm->txm_map; 2715 2716 if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) { 2717 if_statinc(ifp, if_oerrors); 2718 m_freem(m); 2719 continue; 2720 } 2721 2722 cmd_txd = 0; 2723 if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) { 2724 ixl_tx_setup_offloads(m, &cmd_txd); 2725 } 2726 2727 if (vlan_has_tag(m)) { 2728 uint16_t vtag; 2729 vtag = htole16(vlan_get_tag(m)); 2730 cmd_txd |= (uint64_t)vtag << 2731 IXL_TX_DESC_L2TAG1_SHIFT; 2732 cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1; 2733 } 2734 2735 bus_dmamap_sync(sc->sc_dmat, map, 0, 2736 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2737 2738 for (i = 0; i < (unsigned int)map->dm_nsegs; i++) { 2739 txd = &ring[prod]; 2740 2741 cmd = (uint64_t)map->dm_segs[i].ds_len << 2742 IXL_TX_DESC_BSIZE_SHIFT; 2743 cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC; 2744 cmd |= cmd_txd; 2745 2746 txd->addr = htole64(map->dm_segs[i].ds_addr); 2747 txd->cmd = htole64(cmd); 2748 2749 last = prod; 2750 2751 prod++; 2752 prod &= mask; 2753 } 2754 cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS; 2755 txd->cmd = htole64(cmd); 2756 2757 txm->txm_m = m; 2758 txm->txm_eop = last; 2759 2760 bpf_mtap(ifp, m, BPF_D_OUT); 2761 2762 free -= i; 2763 post = 1; 2764 } 2765 2766 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2767 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE); 2768 2769 if (post) { 2770 txr->txr_prod = prod; 2771 ixl_wr(sc, txr->txr_tail, prod); 2772 } 2773} 2774 2775static int 2776ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit) 2777{ 2778 struct ifnet *ifp = &sc->sc_ec.ec_if; 2779 struct ixl_tx_desc *ring, *txd; 2780 struct ixl_tx_map *txm; 2781 struct mbuf *m; 2782 bus_dmamap_t map; 2783 unsigned int cons, prod, last; 2784 unsigned int mask; 2785 uint64_t dtype; 2786 int done = 0, more = 0; 2787 2788 KASSERT(mutex_owned(&txr->txr_lock)); 2789 2790 prod = txr->txr_prod; 2791 cons = txr->txr_cons; 2792 2793 if (cons == prod) 2794 return 0; 2795 2796 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2797 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD); 2798 2799 ring = IXL_DMA_KVA(&txr->txr_mem); 2800 mask = sc->sc_tx_ring_ndescs - 1; 2801 2802 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 2803 2804 do { 2805 if (txlimit-- <= 0) { 2806 more = 1; 2807 break; 2808 } 2809 2810 txm = &txr->txr_maps[cons]; 2811 last = txm->txm_eop; 2812 txd = &ring[last]; 2813 2814 dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK); 2815 if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE)) 2816 break; 2817 2818 map = txm->txm_map; 2819 2820 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 2821 BUS_DMASYNC_POSTWRITE); 2822 bus_dmamap_unload(sc->sc_dmat, map); 2823 2824 m = txm->txm_m; 2825 if (m != NULL) { 2826 if_statinc_ref(nsr, if_opackets); 2827 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 2828 if (ISSET(m->m_flags, M_MCAST)) 2829 if_statinc_ref(nsr, if_omcasts); 2830 m_freem(m); 2831 } 2832 2833 txm->txm_m = NULL; 2834 txm->txm_eop = -1; 2835 2836 cons = last + 1; 2837 cons &= mask; 2838 done = 1; 2839 } while (cons != prod); 2840 2841 IF_STAT_PUTREF(ifp); 2842 2843 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem), 2844 0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD); 2845 2846 txr->txr_cons = cons; 2847 2848 if (done) { 2849 softint_schedule(txr->txr_si); 2850 if (txr->txr_qid == 0) { 2851 CLR(ifp->if_flags, IFF_OACTIVE); 2852 if_schedule_deferred_start(ifp); 2853 } 2854 } 2855 2856 return more; 2857} 2858 2859static void 2860ixl_start(struct ifnet *ifp) 2861{ 2862 struct ixl_softc *sc; 2863 struct ixl_tx_ring *txr; 2864 2865 sc = ifp->if_softc; 2866 txr = sc->sc_qps[0].qp_txr; 2867 2868 mutex_enter(&txr->txr_lock); 2869 ixl_tx_common_locked(ifp, txr, false); 2870 mutex_exit(&txr->txr_lock); 2871} 2872 2873static inline unsigned int 2874ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m) 2875{ 2876 u_int cpuid; 2877 2878 cpuid = cpu_index(curcpu()); 2879 2880 return (unsigned int)(cpuid % sc->sc_nqueue_pairs); 2881} 2882 2883static int 2884ixl_transmit(struct ifnet *ifp, struct mbuf *m) 2885{ 2886 struct ixl_softc *sc; 2887 struct ixl_tx_ring *txr; 2888 unsigned int qid; 2889 2890 sc = ifp->if_softc; 2891 qid = ixl_select_txqueue(sc, m); 2892 2893 txr = sc->sc_qps[qid].qp_txr; 2894 2895 if (__predict_false(!pcq_put(txr->txr_intrq, m))) { 2896 mutex_enter(&txr->txr_lock); 2897 txr->txr_pcqdrop.ev_count++; 2898 mutex_exit(&txr->txr_lock); 2899 2900 m_freem(m); 2901 return ENOBUFS; 2902 } 2903 2904#ifdef IXL_ALWAYS_TXDEFER 2905 kpreempt_disable(); 2906 softint_schedule(txr->txr_si); 2907 kpreempt_enable(); 2908#else 2909 if (mutex_tryenter(&txr->txr_lock)) { 2910 ixl_tx_common_locked(ifp, txr, true); 2911 mutex_exit(&txr->txr_lock); 2912 } else { 2913 kpreempt_disable(); 2914 softint_schedule(txr->txr_si); 2915 kpreempt_enable(); 2916 } 2917#endif 2918 2919 return 0; 2920} 2921 2922static void 2923ixl_deferred_transmit(void *xtxr) 2924{ 2925 struct ixl_tx_ring *txr = xtxr; 2926 struct ixl_softc *sc = txr->txr_sc; 2927 struct ifnet *ifp = &sc->sc_ec.ec_if; 2928 2929 mutex_enter(&txr->txr_lock); 2930 txr->txr_transmitdef.ev_count++; 2931 if (pcq_peek(txr->txr_intrq) != NULL) 2932 ixl_tx_common_locked(ifp, txr, true); 2933 mutex_exit(&txr->txr_lock); 2934} 2935 2936static struct ixl_rx_ring * 2937ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid) 2938{ 2939 struct ixl_rx_ring *rxr = NULL; 2940 struct ixl_rx_map *maps = NULL, *rxm; 2941 unsigned int i; 2942 2943 rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP); 2944 maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs, 2945 KM_SLEEP); 2946 2947 if (ixl_dmamem_alloc(sc, &rxr->rxr_mem, 2948 sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs, 2949 IXL_RX_QUEUE_ALIGN) != 0) 2950 goto free; 2951 2952 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2953 rxm = &maps[i]; 2954 2955 if (bus_dmamap_create(sc->sc_dmat, 2956 IXL_MCLBYTES, 1, IXL_MCLBYTES, 0, 2957 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0) 2958 goto uncreate; 2959 2960 rxm->rxm_m = NULL; 2961 } 2962 2963 rxr->rxr_cons = rxr->rxr_prod = 0; 2964 rxr->rxr_m_head = NULL; 2965 rxr->rxr_m_tail = &rxr->rxr_m_head; 2966 rxr->rxr_maps = maps; 2967 2968 rxr->rxr_tail = I40E_QRX_TAIL(qid); 2969 rxr->rxr_qid = qid; 2970 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 2971 2972 return rxr; 2973 2974uncreate: 2975 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 2976 rxm = &maps[i]; 2977 2978 if (rxm->rxm_map == NULL) 2979 continue; 2980 2981 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 2982 } 2983 2984 ixl_dmamem_free(sc, &rxr->rxr_mem); 2985free: 2986 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 2987 kmem_free(rxr, sizeof(*rxr)); 2988 2989 return NULL; 2990} 2991 2992static void 2993ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 2994{ 2995 struct ixl_rx_map *maps, *rxm; 2996 bus_dmamap_t map; 2997 unsigned int i; 2998 2999 maps = rxr->rxr_maps; 3000 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3001 rxm = &maps[i]; 3002 3003 if (rxm->rxm_m == NULL) 3004 continue; 3005 3006 map = rxm->rxm_map; 3007 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3008 BUS_DMASYNC_POSTWRITE); 3009 bus_dmamap_unload(sc->sc_dmat, map); 3010 3011 m_freem(rxm->rxm_m); 3012 rxm->rxm_m = NULL; 3013 } 3014 3015 m_freem(rxr->rxr_m_head); 3016 rxr->rxr_m_head = NULL; 3017 rxr->rxr_m_tail = &rxr->rxr_m_head; 3018 3019 rxr->rxr_prod = rxr->rxr_cons = 0; 3020} 3021 3022static int 3023ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3024{ 3025 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3026 uint32_t reg; 3027 int i; 3028 3029 for (i = 0; i < 10; i++) { 3030 reg = ixl_rd(sc, ena); 3031 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK)) 3032 return 0; 3033 3034 delaymsec(10); 3035 } 3036 3037 return ETIMEDOUT; 3038} 3039 3040static int 3041ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3042{ 3043 bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid); 3044 uint32_t reg; 3045 int i; 3046 3047 KASSERT(mutex_owned(&rxr->rxr_lock)); 3048 3049 for (i = 0; i < 10; i++) { 3050 reg = ixl_rd(sc, ena); 3051 if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0) 3052 return 0; 3053 3054 delaymsec(10); 3055 } 3056 3057 return ETIMEDOUT; 3058} 3059 3060static void 3061ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3062{ 3063 struct ixl_hmc_rxq rxq; 3064 struct ifnet *ifp = &sc->sc_ec.ec_if; 3065 uint16_t rxmax; 3066 void *hmc; 3067 3068 memset(&rxq, 0, sizeof(rxq)); 3069 rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN; 3070 if (!ISSET(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_MTU)) 3071 rxmax -= ETHER_VLAN_ENCAP_LEN; 3072 3073 rxq.head = htole16(rxr->rxr_cons); 3074 rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT); 3075 rxq.qlen = htole16(sc->sc_rx_ring_ndescs); 3076 rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT); 3077 rxq.hbuff = 0; 3078 rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT; 3079 rxq.dsize = IXL_HMC_RXQ_DSIZE_32; 3080 rxq.crcstrip = 1; 3081 rxq.l2sel = 1; 3082 rxq.showiv = 1; 3083 rxq.rxmax = htole16(rxmax); 3084 rxq.tphrdesc_ena = 0; 3085 rxq.tphwdesc_ena = 0; 3086 rxq.tphdata_ena = 0; 3087 rxq.tphhead_ena = 0; 3088 rxq.lrxqthresh = 0; 3089 rxq.prefena = 1; 3090 3091 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3092 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3093 ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq, 3094 __arraycount(ixl_hmc_pack_rxq)); 3095} 3096 3097static void 3098ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3099{ 3100 void *hmc; 3101 3102 hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid); 3103 memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX)); 3104 rxr->rxr_cons = rxr->rxr_prod = 0; 3105} 3106 3107static void 3108ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3109{ 3110 struct ixl_rx_map *maps, *rxm; 3111 unsigned int i; 3112 3113 maps = rxr->rxr_maps; 3114 for (i = 0; i < sc->sc_rx_ring_ndescs; i++) { 3115 rxm = &maps[i]; 3116 3117 bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map); 3118 } 3119 3120 ixl_dmamem_free(sc, &rxr->rxr_mem); 3121 mutex_destroy(&rxr->rxr_lock); 3122 kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs); 3123 kmem_free(rxr, sizeof(*rxr)); 3124} 3125 3126static inline void 3127ixl_rx_csum(struct mbuf *m, uint64_t qword) 3128{ 3129 int flags_mask; 3130 3131 if (!ISSET(qword, IXL_RX_DESC_L3L4P)) { 3132 /* No L3 or L4 checksum was calculated */ 3133 return; 3134 } 3135 3136 switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) { 3137 case IXL_RX_DESC_PTYPE_IPV4FRAG: 3138 case IXL_RX_DESC_PTYPE_IPV4: 3139 case IXL_RX_DESC_PTYPE_SCTPV4: 3140 case IXL_RX_DESC_PTYPE_ICMPV4: 3141 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3142 break; 3143 case IXL_RX_DESC_PTYPE_TCPV4: 3144 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3145 flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD; 3146 break; 3147 case IXL_RX_DESC_PTYPE_UDPV4: 3148 flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD; 3149 flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD; 3150 break; 3151 case IXL_RX_DESC_PTYPE_TCPV6: 3152 flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD; 3153 break; 3154 case IXL_RX_DESC_PTYPE_UDPV6: 3155 flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD; 3156 break; 3157 default: 3158 flags_mask = 0; 3159 } 3160 3161 m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 | 3162 M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6)); 3163 3164 if (ISSET(qword, IXL_RX_DESC_IPE)) { 3165 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD); 3166 } 3167 3168 if (ISSET(qword, IXL_RX_DESC_L4E)) { 3169 m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD); 3170 } 3171} 3172 3173static int 3174ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit) 3175{ 3176 struct ifnet *ifp = &sc->sc_ec.ec_if; 3177 struct ixl_rx_wb_desc_32 *ring, *rxd; 3178 struct ixl_rx_map *rxm; 3179 bus_dmamap_t map; 3180 unsigned int cons, prod; 3181 struct mbuf *m; 3182 uint64_t word, word0; 3183 unsigned int len; 3184 unsigned int mask; 3185 int done = 0, more = 0; 3186 3187 KASSERT(mutex_owned(&rxr->rxr_lock)); 3188 3189 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3190 return 0; 3191 3192 prod = rxr->rxr_prod; 3193 cons = rxr->rxr_cons; 3194 3195 if (cons == prod) 3196 return 0; 3197 3198 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3199 0, IXL_DMA_LEN(&rxr->rxr_mem), 3200 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3201 3202 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3203 mask = sc->sc_rx_ring_ndescs - 1; 3204 3205 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 3206 3207 do { 3208 if (rxlimit-- <= 0) { 3209 more = 1; 3210 break; 3211 } 3212 3213 rxd = &ring[cons]; 3214 3215 word = le64toh(rxd->qword1); 3216 3217 if (!ISSET(word, IXL_RX_DESC_DD)) 3218 break; 3219 3220 rxm = &rxr->rxr_maps[cons]; 3221 3222 map = rxm->rxm_map; 3223 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3224 BUS_DMASYNC_POSTREAD); 3225 bus_dmamap_unload(sc->sc_dmat, map); 3226 3227 m = rxm->rxm_m; 3228 rxm->rxm_m = NULL; 3229 3230 KASSERT(m != NULL); 3231 3232 len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT; 3233 m->m_len = len; 3234 m->m_pkthdr.len = 0; 3235 3236 m->m_next = NULL; 3237 *rxr->rxr_m_tail = m; 3238 rxr->rxr_m_tail = &m->m_next; 3239 3240 m = rxr->rxr_m_head; 3241 m->m_pkthdr.len += len; 3242 3243 if (ISSET(word, IXL_RX_DESC_EOP)) { 3244 word0 = le64toh(rxd->qword0); 3245 3246 if (ISSET(word, IXL_RX_DESC_L2TAG1P)) { 3247 uint16_t vtag; 3248 vtag = __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK); 3249 vlan_set_tag(m, le16toh(vtag)); 3250 } 3251 3252 if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0) 3253 ixl_rx_csum(m, word); 3254 3255 if (!ISSET(word, 3256 IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) { 3257 m_set_rcvif(m, ifp); 3258 if_statinc_ref(nsr, if_ipackets); 3259 if_statadd_ref(nsr, if_ibytes, 3260 m->m_pkthdr.len); 3261 if_percpuq_enqueue(sc->sc_ipq, m); 3262 } else { 3263 if_statinc_ref(nsr, if_ierrors); 3264 m_freem(m); 3265 } 3266 3267 rxr->rxr_m_head = NULL; 3268 rxr->rxr_m_tail = &rxr->rxr_m_head; 3269 } 3270 3271 cons++; 3272 cons &= mask; 3273 3274 done = 1; 3275 } while (cons != prod); 3276 3277 if (done) { 3278 rxr->rxr_cons = cons; 3279 if (ixl_rxfill(sc, rxr) == -1) 3280 if_statinc_ref(nsr, if_iqdrops); 3281 } 3282 3283 IF_STAT_PUTREF(ifp); 3284 3285 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem), 3286 0, IXL_DMA_LEN(&rxr->rxr_mem), 3287 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3288 3289 return more; 3290} 3291 3292static int 3293ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr) 3294{ 3295 struct ixl_rx_rd_desc_32 *ring, *rxd; 3296 struct ixl_rx_map *rxm; 3297 bus_dmamap_t map; 3298 struct mbuf *m; 3299 unsigned int prod; 3300 unsigned int slots; 3301 unsigned int mask; 3302 int post = 0, error = 0; 3303 3304 KASSERT(mutex_owned(&rxr->rxr_lock)); 3305 3306 prod = rxr->rxr_prod; 3307 slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons, 3308 sc->sc_rx_ring_ndescs); 3309 3310 ring = IXL_DMA_KVA(&rxr->rxr_mem); 3311 mask = sc->sc_rx_ring_ndescs - 1; 3312 3313 if (__predict_false(slots <= 0)) 3314 return -1; 3315 3316 do { 3317 rxm = &rxr->rxr_maps[prod]; 3318 3319 MGETHDR(m, M_DONTWAIT, MT_DATA); 3320 if (m == NULL) { 3321 rxr->rxr_mgethdr_failed.ev_count++; 3322 error = -1; 3323 break; 3324 } 3325 3326 MCLGET(m, M_DONTWAIT); 3327 if (!ISSET(m->m_flags, M_EXT)) { 3328 rxr->rxr_mgetcl_failed.ev_count++; 3329 error = -1; 3330 m_freem(m); 3331 break; 3332 } 3333 3334 m->m_len = m->m_pkthdr.len = MCLBYTES; 3335 m_adj(m, ETHER_ALIGN); 3336 3337 map = rxm->rxm_map; 3338 3339 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 3340 BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) { 3341 rxr->rxr_mbuf_load_failed.ev_count++; 3342 error = -1; 3343 m_freem(m); 3344 break; 3345 } 3346 3347 rxm->rxm_m = m; 3348 3349 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 3350 BUS_DMASYNC_PREREAD); 3351 3352 rxd = &ring[prod]; 3353 3354 rxd->paddr = htole64(map->dm_segs[0].ds_addr); 3355 rxd->haddr = htole64(0); 3356 3357 prod++; 3358 prod &= mask; 3359 3360 post = 1; 3361 3362 } while (--slots); 3363 3364 if (post) { 3365 rxr->rxr_prod = prod; 3366 ixl_wr(sc, rxr->rxr_tail, prod); 3367 } 3368 3369 return error; 3370} 3371 3372static inline int 3373ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp, 3374 u_int txlimit, struct evcnt *txevcnt, 3375 u_int rxlimit, struct evcnt *rxevcnt) 3376{ 3377 struct ixl_tx_ring *txr = qp->qp_txr; 3378 struct ixl_rx_ring *rxr = qp->qp_rxr; 3379 int txmore, rxmore; 3380 int rv; 3381 3382 mutex_enter(&txr->txr_lock); 3383 txevcnt->ev_count++; 3384 txmore = ixl_txeof(sc, txr, txlimit); 3385 mutex_exit(&txr->txr_lock); 3386 3387 mutex_enter(&rxr->rxr_lock); 3388 rxevcnt->ev_count++; 3389 rxmore = ixl_rxeof(sc, rxr, rxlimit); 3390 mutex_exit(&rxr->rxr_lock); 3391 3392 rv = txmore | (rxmore << 1); 3393 3394 return rv; 3395} 3396 3397static void 3398ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp) 3399{ 3400 3401 if (qp->qp_workqueue) 3402 workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL); 3403 else 3404 softint_schedule(qp->qp_si); 3405} 3406 3407static int 3408ixl_intr(void *xsc) 3409{ 3410 struct ixl_softc *sc = xsc; 3411 struct ixl_tx_ring *txr; 3412 struct ixl_rx_ring *rxr; 3413 uint32_t icr, rxintr, txintr; 3414 int rv = 0; 3415 unsigned int i; 3416 3417 KASSERT(sc != NULL); 3418 3419 ixl_enable_other_intr(sc); 3420 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3421 3422 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3423 atomic_inc_64(&sc->sc_event_atq.ev_count); 3424 ixl_atq_done(sc); 3425 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3426 rv = 1; 3427 } 3428 3429 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3430 atomic_inc_64(&sc->sc_event_link.ev_count); 3431 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3432 rv = 1; 3433 } 3434 3435 rxintr = icr & I40E_INTR_NOTX_RX_MASK; 3436 txintr = icr & I40E_INTR_NOTX_TX_MASK; 3437 3438 if (txintr || rxintr) { 3439 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 3440 txr = sc->sc_qps[i].qp_txr; 3441 rxr = sc->sc_qps[i].qp_rxr; 3442 3443 ixl_handle_queue_common(sc, &sc->sc_qps[i], 3444 IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr, 3445 IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr); 3446 } 3447 rv = 1; 3448 } 3449 3450 return rv; 3451} 3452 3453static int 3454ixl_queue_intr(void *xqp) 3455{ 3456 struct ixl_queue_pair *qp = xqp; 3457 struct ixl_tx_ring *txr = qp->qp_txr; 3458 struct ixl_rx_ring *rxr = qp->qp_rxr; 3459 struct ixl_softc *sc = qp->qp_sc; 3460 u_int txlimit, rxlimit; 3461 int more; 3462 3463 txlimit = sc->sc_tx_intr_process_limit; 3464 rxlimit = sc->sc_rx_intr_process_limit; 3465 qp->qp_workqueue = sc->sc_txrx_workqueue; 3466 3467 more = ixl_handle_queue_common(sc, qp, 3468 txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr); 3469 3470 if (more != 0) { 3471 ixl_sched_handle_queue(sc, qp); 3472 } else { 3473 /* for ALTQ */ 3474 if (txr->txr_qid == 0) 3475 if_schedule_deferred_start(&sc->sc_ec.ec_if); 3476 softint_schedule(txr->txr_si); 3477 3478 ixl_enable_queue_intr(sc, qp); 3479 } 3480 3481 return 1; 3482} 3483 3484static void 3485ixl_handle_queue_wk(struct work *wk, void *xsc) 3486{ 3487 struct ixl_queue_pair *qp; 3488 3489 qp = container_of(wk, struct ixl_queue_pair, qp_work); 3490 ixl_handle_queue(qp); 3491} 3492 3493static void 3494ixl_handle_queue(void *xqp) 3495{ 3496 struct ixl_queue_pair *qp = xqp; 3497 struct ixl_softc *sc = qp->qp_sc; 3498 struct ixl_tx_ring *txr = qp->qp_txr; 3499 struct ixl_rx_ring *rxr = qp->qp_rxr; 3500 u_int txlimit, rxlimit; 3501 int more; 3502 3503 txlimit = sc->sc_tx_process_limit; 3504 rxlimit = sc->sc_rx_process_limit; 3505 3506 more = ixl_handle_queue_common(sc, qp, 3507 txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer); 3508 3509 if (more != 0) 3510 ixl_sched_handle_queue(sc, qp); 3511 else 3512 ixl_enable_queue_intr(sc, qp); 3513} 3514 3515static inline void 3516ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg) 3517{ 3518 uint32_t hmc_idx, hmc_isvf; 3519 uint32_t hmc_errtype, hmc_objtype, hmc_data; 3520 3521 hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK; 3522 hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT; 3523 hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK; 3524 hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT; 3525 hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK; 3526 hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT; 3527 hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK; 3528 hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT; 3529 hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA); 3530 3531 device_printf(sc->sc_dev, 3532 "HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n", 3533 hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data); 3534} 3535 3536static int 3537ixl_other_intr(void *xsc) 3538{ 3539 struct ixl_softc *sc = xsc; 3540 uint32_t icr, mask, reg; 3541 int rv; 3542 3543 icr = ixl_rd(sc, I40E_PFINT_ICR0); 3544 mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA); 3545 3546 if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) { 3547 atomic_inc_64(&sc->sc_event_atq.ev_count); 3548 ixl_atq_done(sc); 3549 ixl_work_add(sc->sc_workq, &sc->sc_arq_task); 3550 rv = 1; 3551 } 3552 3553 if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) { 3554 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3555 device_printf(sc->sc_dev, "link stat changed\n"); 3556 3557 atomic_inc_64(&sc->sc_event_link.ev_count); 3558 ixl_work_add(sc->sc_workq, &sc->sc_link_state_task); 3559 rv = 1; 3560 } 3561 3562 if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) { 3563 CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK); 3564 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 3565 reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK; 3566 reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3567 3568 device_printf(sc->sc_dev, "GRST: %s\n", 3569 reg == I40E_RESET_CORER ? "CORER" : 3570 reg == I40E_RESET_GLOBR ? "GLOBR" : 3571 reg == I40E_RESET_EMPR ? "EMPR" : 3572 "POR"); 3573 } 3574 3575 if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK)) 3576 atomic_inc_64(&sc->sc_event_ecc_err.ev_count); 3577 if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)) 3578 atomic_inc_64(&sc->sc_event_pci_exception.ev_count); 3579 if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK)) 3580 atomic_inc_64(&sc->sc_event_crit_err.ev_count); 3581 3582 if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) { 3583 CLR(mask, IXL_ICR0_CRIT_ERR_MASK); 3584 device_printf(sc->sc_dev, "critical error\n"); 3585 } 3586 3587 if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) { 3588 reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO); 3589 if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK)) 3590 ixl_print_hmc_error(sc, reg); 3591 ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0); 3592 } 3593 3594 ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask); 3595 ixl_flush(sc); 3596 ixl_enable_other_intr(sc); 3597 return rv; 3598} 3599 3600static void 3601ixl_get_link_status_done_work(void *xsc) 3602{ 3603 struct ixl_softc *sc = xsc; 3604 struct ixl_aq_desc *iaq, iaq_buf; 3605 3606 mutex_enter(&sc->sc_atq_lock); 3607 iaq = &sc->sc_link_state_atq.iatq_desc; 3608 iaq_buf = *iaq; 3609 mutex_exit(&sc->sc_atq_lock); 3610 3611 ixl_link_state_update(sc, &iaq_buf); 3612 3613 mutex_enter(&sc->sc_atq_lock); 3614 CLR(iaq->iaq_flags, htole16(IXL_AQ_DD)); 3615 ixl_wakeup(sc, iaq); 3616 mutex_exit(&sc->sc_atq_lock); 3617} 3618 3619static void 3620ixl_get_link_status_done(struct ixl_softc *sc, 3621 const struct ixl_aq_desc *iaq) 3622{ 3623 3624 ixl_work_add(sc->sc_workq, &sc->sc_link_state_done_task); 3625} 3626 3627static int 3628ixl_get_link_status(struct ixl_softc *sc, enum ixl_link_flags flags) 3629{ 3630 struct ixl_atq *iatq; 3631 struct ixl_aq_desc *iaq; 3632 struct ixl_aq_link_param *param; 3633 int error; 3634 3635 mutex_enter(&sc->sc_atq_lock); 3636 3637 iatq = &sc->sc_link_state_atq; 3638 iaq = &iatq->iatq_desc; 3639 3640 if (!sc->sc_link_state_atq.iatq_inuse && 3641 !ISSET(iaq->iaq_flags, htole16(IXL_AQ_DD))) { 3642 memset(iaq, 0, sizeof(*iaq)); 3643 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 3644 param = (struct ixl_aq_link_param *)iaq->iaq_param; 3645 param->notify = IXL_AQ_LINK_NOTIFY; 3646 3647 KASSERT(iatq->iatq_fn == ixl_get_link_status_done); 3648 error = ixl_atq_post_locked(sc, iatq); 3649 if (error != 0) 3650 goto out; 3651 } else { 3652 /* the previous command is not completed */ 3653 error = EBUSY; 3654 } 3655 3656 if (ISSET(flags, IXL_LINK_FLAG_WAITDONE)) { 3657 do { 3658 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3659 IXL_ATQ_EXEC_TIMEOUT); 3660 if (error == EWOULDBLOCK) 3661 break; 3662 } while (iatq->iatq_inuse || 3663 ISSET(iaq->iaq_flags, htole16(IXL_AQ_DD))); 3664 } 3665 3666out: 3667 mutex_exit(&sc->sc_atq_lock); 3668 3669 return error; 3670} 3671 3672static void 3673ixl_get_link_status_work(void *xsc) 3674{ 3675 struct ixl_softc *sc = xsc; 3676 3677 /* 3678 * IXL_LINK_FLAG_WAITDONE causes deadlock 3679 * because of doing ixl_gt_link_status_done_work() 3680 * in the same workqueue. 3681 */ 3682 (void)ixl_get_link_status(sc, IXL_LINK_NOFLAGS); 3683} 3684 3685static void 3686ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3687{ 3688 struct ifnet *ifp = &sc->sc_ec.ec_if; 3689 int link_state; 3690 3691 mutex_enter(&sc->sc_cfg_lock); 3692 link_state = ixl_set_link_status_locked(sc, iaq); 3693 mutex_exit(&sc->sc_cfg_lock); 3694 3695 if (ifp->if_link_state != link_state) 3696 if_link_state_change(ifp, link_state); 3697 3698 if (link_state != LINK_STATE_DOWN) { 3699 kpreempt_disable(); 3700 if_schedule_deferred_start(ifp); 3701 kpreempt_enable(); 3702 } 3703} 3704 3705static void 3706ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq, 3707 const char *msg) 3708{ 3709 char buf[512]; 3710 size_t len; 3711 3712 len = sizeof(buf); 3713 buf[--len] = '\0'; 3714 3715 device_printf(sc->sc_dev, "%s\n", msg); 3716 snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags)); 3717 device_printf(sc->sc_dev, "flags %s opcode %04x\n", 3718 buf, le16toh(iaq->iaq_opcode)); 3719 device_printf(sc->sc_dev, "datalen %u retval %u\n", 3720 le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval)); 3721 device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie); 3722 device_printf(sc->sc_dev, "%08x %08x %08x %08x\n", 3723 le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]), 3724 le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3])); 3725} 3726 3727static void 3728ixl_arq(void *xsc) 3729{ 3730 struct ixl_softc *sc = xsc; 3731 struct ixl_aq_desc *arq, *iaq; 3732 struct ixl_aq_buf *aqb; 3733 unsigned int cons = sc->sc_arq_cons; 3734 unsigned int prod; 3735 int done = 0; 3736 3737 prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) & 3738 sc->sc_aq_regs->arq_head_mask; 3739 3740 if (cons == prod) 3741 goto done; 3742 3743 arq = IXL_DMA_KVA(&sc->sc_arq); 3744 3745 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3746 0, IXL_DMA_LEN(&sc->sc_arq), 3747 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3748 3749 do { 3750 iaq = &arq[cons]; 3751 aqb = sc->sc_arq_live[cons]; 3752 3753 KASSERT(aqb != NULL); 3754 3755 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN, 3756 BUS_DMASYNC_POSTREAD); 3757 3758 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3759 ixl_aq_dump(sc, iaq, "arq event"); 3760 3761 switch (iaq->iaq_opcode) { 3762 case htole16(IXL_AQ_OP_PHY_LINK_STATUS): 3763 ixl_link_state_update(sc, iaq); 3764 break; 3765 } 3766 3767 memset(iaq, 0, sizeof(*iaq)); 3768 sc->sc_arq_live[cons] = NULL; 3769 SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry); 3770 3771 cons++; 3772 cons &= IXL_AQ_MASK; 3773 3774 done = 1; 3775 } while (cons != prod); 3776 3777 if (done) { 3778 sc->sc_arq_cons = cons; 3779 ixl_arq_fill(sc); 3780 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq), 3781 0, IXL_DMA_LEN(&sc->sc_arq), 3782 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3783 } 3784 3785done: 3786 ixl_enable_other_intr(sc); 3787} 3788 3789static void 3790ixl_atq_set(struct ixl_atq *iatq, 3791 void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *)) 3792{ 3793 3794 iatq->iatq_fn = fn; 3795} 3796 3797static int 3798ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3799{ 3800 struct ixl_aq_desc *atq, *slot; 3801 unsigned int prod, cons, prod_next; 3802 3803 /* assert locked */ 3804 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3805 3806 atq = IXL_DMA_KVA(&sc->sc_atq); 3807 prod = sc->sc_atq_prod; 3808 cons = sc->sc_atq_cons; 3809 prod_next = (prod +1) & IXL_AQ_MASK; 3810 3811 if (cons == prod_next) 3812 return ENOMEM; 3813 3814 slot = &atq[prod]; 3815 3816 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3817 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3818 3819 KASSERT(iatq->iatq_fn != NULL); 3820 *slot = iatq->iatq_desc; 3821 slot->iaq_cookie = (uint64_t)((intptr_t)iatq); 3822 3823 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3824 ixl_aq_dump(sc, slot, "atq command"); 3825 3826 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3827 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3828 3829 sc->sc_atq_prod = prod_next; 3830 ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod); 3831 iatq->iatq_inuse = true; 3832 3833 return 0; 3834} 3835 3836static void 3837ixl_atq_done_locked(struct ixl_softc *sc) 3838{ 3839 struct ixl_aq_desc *atq, *slot; 3840 struct ixl_atq *iatq; 3841 unsigned int cons; 3842 unsigned int prod; 3843 3844 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3845 3846 prod = sc->sc_atq_prod; 3847 cons = sc->sc_atq_cons; 3848 3849 if (prod == cons) 3850 return; 3851 3852 atq = IXL_DMA_KVA(&sc->sc_atq); 3853 3854 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3855 0, IXL_DMA_LEN(&sc->sc_atq), 3856 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 3857 3858 do { 3859 slot = &atq[cons]; 3860 if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD))) 3861 break; 3862 3863 iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie); 3864 iatq->iatq_desc = *slot; 3865 iatq->iatq_inuse = false; 3866 3867 memset(slot, 0, sizeof(*slot)); 3868 3869 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG)) 3870 ixl_aq_dump(sc, &iatq->iatq_desc, "atq response"); 3871 3872 (*iatq->iatq_fn)(sc, &iatq->iatq_desc); 3873 3874 cons++; 3875 cons &= IXL_AQ_MASK; 3876 } while (cons != prod); 3877 3878 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3879 0, IXL_DMA_LEN(&sc->sc_atq), 3880 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 3881 3882 sc->sc_atq_cons = cons; 3883} 3884 3885static void 3886ixl_atq_done(struct ixl_softc *sc) 3887{ 3888 3889 mutex_enter(&sc->sc_atq_lock); 3890 ixl_atq_done_locked(sc); 3891 mutex_exit(&sc->sc_atq_lock); 3892} 3893 3894static void 3895ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 3896{ 3897 3898 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3899 3900 cv_broadcast(&sc->sc_atq_cv); 3901} 3902 3903static int 3904ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq) 3905{ 3906 int error; 3907 3908 mutex_enter(&sc->sc_atq_lock); 3909 error = ixl_atq_exec_locked(sc, iatq); 3910 mutex_exit(&sc->sc_atq_lock); 3911 3912 return error; 3913} 3914 3915static int 3916ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq) 3917{ 3918 int error; 3919 3920 KASSERT(mutex_owned(&sc->sc_atq_lock)); 3921 KASSERT(iatq->iatq_desc.iaq_cookie == 0); 3922 3923 ixl_atq_set(iatq, ixl_wakeup); 3924 3925 error = ixl_atq_post_locked(sc, iatq); 3926 if (error) 3927 return error; 3928 3929 do { 3930 error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock, 3931 IXL_ATQ_EXEC_TIMEOUT); 3932 if (error == EWOULDBLOCK) 3933 break; 3934 } while (iatq->iatq_inuse); 3935 3936 return error; 3937} 3938 3939static int 3940ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm) 3941{ 3942 struct ixl_aq_desc *atq, *slot; 3943 unsigned int prod; 3944 unsigned int t = 0; 3945 3946 mutex_enter(&sc->sc_atq_lock); 3947 3948 atq = IXL_DMA_KVA(&sc->sc_atq); 3949 prod = sc->sc_atq_prod; 3950 slot = atq + prod; 3951 3952 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3953 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE); 3954 3955 *slot = *iaq; 3956 slot->iaq_flags |= htole16(IXL_AQ_SI); 3957 3958 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3959 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE); 3960 3961 prod++; 3962 prod &= IXL_AQ_MASK; 3963 sc->sc_atq_prod = prod; 3964 ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod); 3965 3966 while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) { 3967 delaymsec(1); 3968 3969 if (t++ > tm) { 3970 mutex_exit(&sc->sc_atq_lock); 3971 return ETIMEDOUT; 3972 } 3973 } 3974 3975 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3976 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD); 3977 *iaq = *slot; 3978 memset(slot, 0, sizeof(*slot)); 3979 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq), 3980 0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD); 3981 3982 sc->sc_atq_cons = prod; 3983 3984 mutex_exit(&sc->sc_atq_lock); 3985 3986 return 0; 3987} 3988 3989static int 3990ixl_get_version(struct ixl_softc *sc) 3991{ 3992 struct ixl_aq_desc iaq; 3993 uint32_t fwbuild, fwver, apiver; 3994 uint16_t api_maj_ver, api_min_ver; 3995 3996 memset(&iaq, 0, sizeof(iaq)); 3997 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION); 3998 3999 iaq.iaq_retval = le16toh(23); 4000 4001 if (ixl_atq_poll(sc, &iaq, 2000) != 0) 4002 return ETIMEDOUT; 4003 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) 4004 return EIO; 4005 4006 fwbuild = le32toh(iaq.iaq_param[1]); 4007 fwver = le32toh(iaq.iaq_param[2]); 4008 apiver = le32toh(iaq.iaq_param[3]); 4009 4010 api_maj_ver = (uint16_t)apiver; 4011 api_min_ver = (uint16_t)(apiver >> 16); 4012 4013 aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver, 4014 (uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver); 4015 4016 if (sc->sc_mac_type == I40E_MAC_X722) { 4017 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK | 4018 IXL_SC_AQ_FLAG_NVMREAD); 4019 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4020 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS); 4021 } 4022 4023#define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min)) 4024 if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) { 4025 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL); 4026 SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK); 4027 } 4028#undef IXL_API_VER 4029 4030 return 0; 4031} 4032 4033static int 4034ixl_get_nvm_version(struct ixl_softc *sc) 4035{ 4036 uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo; 4037 uint32_t eetrack, oem; 4038 uint16_t nvm_maj_ver, nvm_min_ver, oem_build; 4039 uint8_t oem_ver, oem_patch; 4040 4041 nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0; 4042 ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver); 4043 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 4044 ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 4045 ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 4046 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi); 4047 ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo); 4048 4049 nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK); 4050 nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK); 4051 eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo; 4052 oem = ((uint32_t)oem_hi << 16) | oem_lo; 4053 oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK); 4054 oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK); 4055 oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK); 4056 4057 aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d", 4058 nvm_maj_ver, nvm_min_ver, eetrack, 4059 oem_ver, oem_build, oem_patch); 4060 4061 return 0; 4062} 4063 4064static int 4065ixl_pxe_clear(struct ixl_softc *sc) 4066{ 4067 struct ixl_aq_desc iaq; 4068 int rv; 4069 4070 memset(&iaq, 0, sizeof(iaq)); 4071 iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE); 4072 iaq.iaq_param[0] = htole32(0x2); 4073 4074 rv = ixl_atq_poll(sc, &iaq, 250); 4075 4076 ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1); 4077 4078 if (rv != 0) 4079 return ETIMEDOUT; 4080 4081 switch (iaq.iaq_retval) { 4082 case htole16(IXL_AQ_RC_OK): 4083 case htole16(IXL_AQ_RC_EEXIST): 4084 break; 4085 default: 4086 return EIO; 4087 } 4088 4089 return 0; 4090} 4091 4092static int 4093ixl_lldp_shut(struct ixl_softc *sc) 4094{ 4095 struct ixl_aq_desc iaq; 4096 4097 memset(&iaq, 0, sizeof(iaq)); 4098 iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT); 4099 iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN); 4100 4101 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4102 aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n"); 4103 return -1; 4104 } 4105 4106 switch (iaq.iaq_retval) { 4107 case htole16(IXL_AQ_RC_EMODE): 4108 case htole16(IXL_AQ_RC_EPERM): 4109 /* ignore silently */ 4110 default: 4111 break; 4112 } 4113 4114 return 0; 4115} 4116 4117static void 4118ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap) 4119{ 4120 uint16_t id; 4121 uint32_t number, logical_id; 4122 4123 id = le16toh(cap->cap_id); 4124 number = le32toh(cap->number); 4125 logical_id = le32toh(cap->logical_id); 4126 4127 switch (id) { 4128 case IXL_AQ_CAP_RSS: 4129 sc->sc_rss_table_size = number; 4130 sc->sc_rss_table_entry_width = logical_id; 4131 break; 4132 case IXL_AQ_CAP_RXQ: 4133 case IXL_AQ_CAP_TXQ: 4134 sc->sc_nqueue_pairs_device = MIN(number, 4135 sc->sc_nqueue_pairs_device); 4136 break; 4137 } 4138} 4139 4140static int 4141ixl_get_hw_capabilities(struct ixl_softc *sc) 4142{ 4143 struct ixl_dmamem idm; 4144 struct ixl_aq_desc iaq; 4145 struct ixl_aq_capability *caps; 4146 size_t i, ncaps; 4147 bus_size_t caps_size; 4148 uint16_t status; 4149 int rv; 4150 4151 caps_size = sizeof(caps[0]) * 40; 4152 memset(&iaq, 0, sizeof(iaq)); 4153 iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP); 4154 4155 do { 4156 if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) { 4157 return -1; 4158 } 4159 4160 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4161 (caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4162 iaq.iaq_datalen = htole16(caps_size); 4163 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4164 4165 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4166 IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD); 4167 4168 rv = ixl_atq_poll(sc, &iaq, 250); 4169 4170 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, 4171 IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD); 4172 4173 if (rv != 0) { 4174 aprint_error(", HW capabilities timeout\n"); 4175 goto done; 4176 } 4177 4178 status = le16toh(iaq.iaq_retval); 4179 4180 if (status == IXL_AQ_RC_ENOMEM) { 4181 caps_size = le16toh(iaq.iaq_datalen); 4182 ixl_dmamem_free(sc, &idm); 4183 } 4184 } while (status == IXL_AQ_RC_ENOMEM); 4185 4186 if (status != IXL_AQ_RC_OK) { 4187 aprint_error(", HW capabilities error\n"); 4188 goto done; 4189 } 4190 4191 caps = IXL_DMA_KVA(&idm); 4192 ncaps = le16toh(iaq.iaq_param[1]); 4193 4194 for (i = 0; i < ncaps; i++) { 4195 ixl_parse_hw_capability(sc, &caps[i]); 4196 } 4197 4198done: 4199 ixl_dmamem_free(sc, &idm); 4200 return rv; 4201} 4202 4203static int 4204ixl_get_mac(struct ixl_softc *sc) 4205{ 4206 struct ixl_dmamem idm; 4207 struct ixl_aq_desc iaq; 4208 struct ixl_aq_mac_addresses *addrs; 4209 int rv; 4210 4211 if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) { 4212 aprint_error(", unable to allocate mac addresses\n"); 4213 return -1; 4214 } 4215 4216 memset(&iaq, 0, sizeof(iaq)); 4217 iaq.iaq_flags = htole16(IXL_AQ_BUF); 4218 iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ); 4219 iaq.iaq_datalen = htole16(sizeof(*addrs)); 4220 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4221 4222 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4223 BUS_DMASYNC_PREREAD); 4224 4225 rv = ixl_atq_poll(sc, &iaq, 250); 4226 4227 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4228 BUS_DMASYNC_POSTREAD); 4229 4230 if (rv != 0) { 4231 aprint_error(", MAC ADDRESS READ timeout\n"); 4232 rv = -1; 4233 goto done; 4234 } 4235 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4236 aprint_error(", MAC ADDRESS READ error\n"); 4237 rv = -1; 4238 goto done; 4239 } 4240 4241 addrs = IXL_DMA_KVA(&idm); 4242 if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) { 4243 printf(", port address is not valid\n"); 4244 goto done; 4245 } 4246 4247 memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN); 4248 rv = 0; 4249 4250done: 4251 ixl_dmamem_free(sc, &idm); 4252 return rv; 4253} 4254 4255static int 4256ixl_get_switch_config(struct ixl_softc *sc) 4257{ 4258 struct ixl_dmamem idm; 4259 struct ixl_aq_desc iaq; 4260 struct ixl_aq_switch_config *hdr; 4261 struct ixl_aq_switch_config_element *elms, *elm; 4262 unsigned int nelm, i; 4263 int rv; 4264 4265 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4266 aprint_error_dev(sc->sc_dev, 4267 "unable to allocate switch config buffer\n"); 4268 return -1; 4269 } 4270 4271 memset(&iaq, 0, sizeof(iaq)); 4272 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4273 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4274 iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG); 4275 iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN); 4276 ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm)); 4277 4278 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4279 BUS_DMASYNC_PREREAD); 4280 4281 rv = ixl_atq_poll(sc, &iaq, 250); 4282 4283 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm), 4284 BUS_DMASYNC_POSTREAD); 4285 4286 if (rv != 0) { 4287 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n"); 4288 rv = -1; 4289 goto done; 4290 } 4291 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4292 aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n"); 4293 rv = -1; 4294 goto done; 4295 } 4296 4297 hdr = IXL_DMA_KVA(&idm); 4298 elms = (struct ixl_aq_switch_config_element *)(hdr + 1); 4299 4300 nelm = le16toh(hdr->num_reported); 4301 if (nelm < 1) { 4302 aprint_error_dev(sc->sc_dev, "no switch config available\n"); 4303 rv = -1; 4304 goto done; 4305 } 4306 4307 for (i = 0; i < nelm; i++) { 4308 elm = &elms[i]; 4309 4310 aprint_debug_dev(sc->sc_dev, 4311 "type %x revision %u seid %04x\n", 4312 elm->type, elm->revision, le16toh(elm->seid)); 4313 aprint_debug_dev(sc->sc_dev, 4314 "uplink %04x downlink %04x\n", 4315 le16toh(elm->uplink_seid), 4316 le16toh(elm->downlink_seid)); 4317 aprint_debug_dev(sc->sc_dev, 4318 "conntype %x scheduler %04x extra %04x\n", 4319 elm->connection_type, 4320 le16toh(elm->scheduler_id), 4321 le16toh(elm->element_info)); 4322 } 4323 4324 elm = &elms[0]; 4325 4326 sc->sc_uplink_seid = elm->uplink_seid; 4327 sc->sc_downlink_seid = elm->downlink_seid; 4328 sc->sc_seid = elm->seid; 4329 4330 if ((sc->sc_uplink_seid == htole16(0)) != 4331 (sc->sc_downlink_seid == htole16(0))) { 4332 aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n"); 4333 rv = -1; 4334 goto done; 4335 } 4336 4337done: 4338 ixl_dmamem_free(sc, &idm); 4339 return rv; 4340} 4341 4342static int 4343ixl_phy_mask_ints(struct ixl_softc *sc) 4344{ 4345 struct ixl_aq_desc iaq; 4346 4347 memset(&iaq, 0, sizeof(iaq)); 4348 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK); 4349 iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK & 4350 ~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL | 4351 IXL_AQ_PHY_EV_MEDIA_NA)); 4352 4353 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4354 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n"); 4355 return -1; 4356 } 4357 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4358 aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n"); 4359 return -1; 4360 } 4361 4362 return 0; 4363} 4364 4365static int 4366ixl_get_phy_abilities(struct ixl_softc *sc, struct ixl_dmamem *idm) 4367{ 4368 struct ixl_aq_desc iaq; 4369 int rv; 4370 4371 memset(&iaq, 0, sizeof(iaq)); 4372 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4373 (IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4374 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES); 4375 iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm)); 4376 iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT); 4377 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 4378 4379 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4380 BUS_DMASYNC_PREREAD); 4381 4382 rv = ixl_atq_poll(sc, &iaq, 250); 4383 4384 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 4385 BUS_DMASYNC_POSTREAD); 4386 4387 if (rv != 0) 4388 return -1; 4389 4390 return le16toh(iaq.iaq_retval); 4391} 4392 4393static int 4394ixl_get_phy_info(struct ixl_softc *sc) 4395{ 4396 struct ixl_dmamem idm; 4397 struct ixl_aq_phy_abilities *phy; 4398 int rv; 4399 4400 if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) { 4401 aprint_error_dev(sc->sc_dev, 4402 "unable to allocate phy abilities buffer\n"); 4403 return -1; 4404 } 4405 4406 rv = ixl_get_phy_abilities(sc, &idm); 4407 switch (rv) { 4408 case -1: 4409 aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n"); 4410 goto done; 4411 case IXL_AQ_RC_OK: 4412 break; 4413 case IXL_AQ_RC_EIO: 4414 aprint_error_dev(sc->sc_dev,"unable to query phy types\n"); 4415 goto done; 4416 default: 4417 aprint_error_dev(sc->sc_dev, 4418 "GET PHY ABILITIES error %u\n", rv); 4419 goto done; 4420 } 4421 4422 phy = IXL_DMA_KVA(&idm); 4423 4424 sc->sc_phy_types = le32toh(phy->phy_type); 4425 sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32; 4426 4427 sc->sc_phy_abilities = phy->abilities; 4428 sc->sc_phy_linkspeed = phy->link_speed; 4429 sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info & 4430 (IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS | 4431 IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS); 4432 sc->sc_eee_cap = phy->eee_capability; 4433 sc->sc_eeer_val = phy->eeer_val; 4434 sc->sc_d3_lpan = phy->d3_lpan; 4435 4436 rv = 0; 4437 4438done: 4439 ixl_dmamem_free(sc, &idm); 4440 return rv; 4441} 4442 4443static int 4444ixl_set_phy_config(struct ixl_softc *sc, 4445 uint8_t link_speed, uint8_t abilities, bool polling) 4446{ 4447 struct ixl_aq_phy_param *param; 4448 struct ixl_atq iatq; 4449 struct ixl_aq_desc *iaq; 4450 int error; 4451 4452 memset(&iatq, 0, sizeof(iatq)); 4453 4454 iaq = &iatq.iatq_desc; 4455 iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG); 4456 param = (struct ixl_aq_phy_param *)&iaq->iaq_param; 4457 param->phy_types = htole32((uint32_t)sc->sc_phy_types); 4458 param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32); 4459 param->link_speed = link_speed; 4460 param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK; 4461 param->fec_cfg = sc->sc_phy_fec_cfg; 4462 param->eee_capability = sc->sc_eee_cap; 4463 param->eeer_val = sc->sc_eeer_val; 4464 param->d3_lpan = sc->sc_d3_lpan; 4465 4466 if (polling) 4467 error = ixl_atq_poll(sc, iaq, 250); 4468 else 4469 error = ixl_atq_exec(sc, &iatq); 4470 4471 if (error != 0) 4472 return error; 4473 4474 switch (le16toh(iaq->iaq_retval)) { 4475 case IXL_AQ_RC_OK: 4476 break; 4477 case IXL_AQ_RC_EPERM: 4478 return EPERM; 4479 default: 4480 return EIO; 4481 } 4482 4483 return 0; 4484} 4485 4486static int 4487ixl_set_phy_autoselect(struct ixl_softc *sc) 4488{ 4489 uint8_t link_speed, abilities; 4490 4491 link_speed = sc->sc_phy_linkspeed; 4492 abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO; 4493 4494 return ixl_set_phy_config(sc, link_speed, abilities, true); 4495} 4496 4497static int 4498ixl_get_link_status_poll(struct ixl_softc *sc, int *l) 4499{ 4500 struct ixl_aq_desc iaq; 4501 struct ixl_aq_link_param *param; 4502 int link; 4503 4504 memset(&iaq, 0, sizeof(iaq)); 4505 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS); 4506 param = (struct ixl_aq_link_param *)iaq.iaq_param; 4507 param->notify = IXL_AQ_LINK_NOTIFY; 4508 4509 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4510 return ETIMEDOUT; 4511 } 4512 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4513 return EIO; 4514 } 4515 4516 /* It is unnecessary to hold lock */ 4517 link = ixl_set_link_status_locked(sc, &iaq); 4518 4519 if (l != NULL) 4520 *l = link; 4521 4522 return 0; 4523} 4524 4525static int 4526ixl_get_vsi(struct ixl_softc *sc) 4527{ 4528 struct ixl_dmamem *vsi = &sc->sc_scratch; 4529 struct ixl_aq_desc iaq; 4530 struct ixl_aq_vsi_param *param; 4531 struct ixl_aq_vsi_reply *reply; 4532 struct ixl_aq_vsi_data *data; 4533 int rv; 4534 4535 /* grumble, vsi info isn't "known" at compile time */ 4536 4537 memset(&iaq, 0, sizeof(iaq)); 4538 iaq.iaq_flags = htole16(IXL_AQ_BUF | 4539 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4540 iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS); 4541 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4542 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4543 4544 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4545 param->uplink_seid = sc->sc_seid; 4546 4547 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4548 BUS_DMASYNC_PREREAD); 4549 4550 rv = ixl_atq_poll(sc, &iaq, 250); 4551 4552 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4553 BUS_DMASYNC_POSTREAD); 4554 4555 if (rv != 0) { 4556 return ETIMEDOUT; 4557 } 4558 4559 switch (le16toh(iaq.iaq_retval)) { 4560 case IXL_AQ_RC_OK: 4561 break; 4562 case IXL_AQ_RC_ENOENT: 4563 return ENOENT; 4564 case IXL_AQ_RC_EACCES: 4565 return EACCES; 4566 default: 4567 return EIO; 4568 } 4569 4570 reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param; 4571 sc->sc_vsi_number = le16toh(reply->vsi_number); 4572 data = IXL_DMA_KVA(vsi); 4573 sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx); 4574 4575 return 0; 4576} 4577 4578static int 4579ixl_set_vsi(struct ixl_softc *sc) 4580{ 4581 struct ixl_dmamem *vsi = &sc->sc_scratch; 4582 struct ixl_aq_desc iaq; 4583 struct ixl_aq_vsi_param *param; 4584 struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi); 4585 unsigned int qnum; 4586 uint16_t val; 4587 int rv; 4588 4589 qnum = sc->sc_nqueue_pairs - 1; 4590 4591 data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP | 4592 IXL_AQ_VSI_VALID_VLAN); 4593 4594 CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK)); 4595 SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG)); 4596 data->queue_mapping[0] = htole16(0); 4597 data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) | 4598 (qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT)); 4599 4600 val = le16toh(data->port_vlan_flags); 4601 CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK); 4602 SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL); 4603 4604 if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) { 4605 SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH); 4606 } else { 4607 SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING); 4608 } 4609 4610 data->port_vlan_flags = htole16(val); 4611 4612 /* grumble, vsi info isn't "known" at compile time */ 4613 4614 memset(&iaq, 0, sizeof(iaq)); 4615 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4616 (IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4617 iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS); 4618 iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi)); 4619 ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi)); 4620 4621 param = (struct ixl_aq_vsi_param *)iaq.iaq_param; 4622 param->uplink_seid = sc->sc_seid; 4623 4624 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4625 BUS_DMASYNC_PREWRITE); 4626 4627 rv = ixl_atq_poll(sc, &iaq, 250); 4628 4629 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi), 4630 BUS_DMASYNC_POSTWRITE); 4631 4632 if (rv != 0) { 4633 return ETIMEDOUT; 4634 } 4635 4636 switch (le16toh(iaq.iaq_retval)) { 4637 case IXL_AQ_RC_OK: 4638 break; 4639 case IXL_AQ_RC_ENOENT: 4640 return ENOENT; 4641 case IXL_AQ_RC_EACCES: 4642 return EACCES; 4643 default: 4644 return EIO; 4645 } 4646 4647 return 0; 4648} 4649 4650static void 4651ixl_set_filter_control(struct ixl_softc *sc) 4652{ 4653 uint32_t reg; 4654 4655 reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0); 4656 4657 CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK); 4658 SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT); 4659 4660 SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK); 4661 SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK); 4662 SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK); 4663 4664 ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg); 4665} 4666 4667static inline void 4668ixl_get_default_rss_key(uint32_t *buf, size_t len) 4669{ 4670 size_t cplen; 4671 uint8_t rss_seed[RSS_KEYSIZE]; 4672 4673 rss_getkey(rss_seed); 4674 memset(buf, 0, len); 4675 4676 cplen = MIN(len, sizeof(rss_seed)); 4677 memcpy(buf, rss_seed, cplen); 4678} 4679 4680static int 4681ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen) 4682{ 4683 struct ixl_dmamem *idm; 4684 struct ixl_atq iatq; 4685 struct ixl_aq_desc *iaq; 4686 struct ixl_aq_rss_key_param *param; 4687 struct ixl_aq_rss_key_data *data; 4688 size_t len, datalen, stdlen, extlen; 4689 uint16_t vsi_id; 4690 int rv; 4691 4692 memset(&iatq, 0, sizeof(iatq)); 4693 iaq = &iatq.iatq_desc; 4694 idm = &sc->sc_aqbuf; 4695 4696 datalen = sizeof(*data); 4697 4698 /*XXX The buf size has to be less than the size of the register */ 4699 datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen); 4700 4701 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4702 (datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4703 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY); 4704 iaq->iaq_datalen = htole16(datalen); 4705 4706 param = (struct ixl_aq_rss_key_param *)iaq->iaq_param; 4707 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) | 4708 IXL_AQ_RSSKEY_VSI_VALID; 4709 param->vsi_id = htole16(vsi_id); 4710 4711 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4712 data = IXL_DMA_KVA(idm); 4713 4714 len = MIN(keylen, datalen); 4715 stdlen = MIN(sizeof(data->standard_rss_key), len); 4716 memcpy(data->standard_rss_key, key, stdlen); 4717 len = (len > stdlen) ? (len - stdlen) : 0; 4718 4719 extlen = MIN(sizeof(data->extended_hash_key), len); 4720 extlen = (stdlen < keylen) ? 0 : keylen - stdlen; 4721 memcpy(data->extended_hash_key, key + stdlen, extlen); 4722 4723 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4724 4725 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4726 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4727 4728 rv = ixl_atq_exec(sc, &iatq); 4729 4730 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4731 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4732 4733 if (rv != 0) { 4734 return ETIMEDOUT; 4735 } 4736 4737 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4738 return EIO; 4739 } 4740 4741 return 0; 4742} 4743 4744static int 4745ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen) 4746{ 4747 struct ixl_dmamem *idm; 4748 struct ixl_atq iatq; 4749 struct ixl_aq_desc *iaq; 4750 struct ixl_aq_rss_lut_param *param; 4751 uint16_t vsi_id; 4752 uint8_t *data; 4753 size_t dmalen; 4754 int rv; 4755 4756 memset(&iatq, 0, sizeof(iatq)); 4757 iaq = &iatq.iatq_desc; 4758 idm = &sc->sc_aqbuf; 4759 4760 dmalen = MIN(lutlen, IXL_DMA_LEN(idm)); 4761 4762 iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD | 4763 (dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0)); 4764 iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT); 4765 iaq->iaq_datalen = htole16(dmalen); 4766 4767 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 4768 data = IXL_DMA_KVA(idm); 4769 memcpy(data, lut, dmalen); 4770 ixl_aq_dva(iaq, IXL_DMA_DVA(idm)); 4771 4772 param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param; 4773 vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) | 4774 IXL_AQ_RSSLUT_VSI_VALID; 4775 param->vsi_id = htole16(vsi_id); 4776 param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF << 4777 IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT); 4778 4779 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4780 IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE); 4781 4782 rv = ixl_atq_exec(sc, &iatq); 4783 4784 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, 4785 IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE); 4786 4787 if (rv != 0) { 4788 return ETIMEDOUT; 4789 } 4790 4791 if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) { 4792 return EIO; 4793 } 4794 4795 return 0; 4796} 4797 4798static int 4799ixl_register_rss_key(struct ixl_softc *sc) 4800{ 4801 uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG]; 4802 int rv; 4803 size_t i; 4804 4805 ixl_get_default_rss_key(rss_seed, sizeof(rss_seed)); 4806 4807 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4808 rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed, 4809 sizeof(rss_seed)); 4810 } else { 4811 rv = 0; 4812 for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { 4813 ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]); 4814 } 4815 } 4816 4817 return rv; 4818} 4819 4820static void 4821ixl_register_rss_pctype(struct ixl_softc *sc) 4822{ 4823 uint64_t set_hena = 0; 4824 uint32_t hena0, hena1; 4825 4826 /* 4827 * We use TCP/UDP with IPv4/IPv6 by default. 4828 * Note: the device can not use just IP header in each 4829 * TCP/UDP packets for the RSS hash calculation. 4830 */ 4831 if (sc->sc_mac_type == I40E_MAC_X722) 4832 set_hena = IXL_RSS_HENA_DEFAULT_X722; 4833 else 4834 set_hena = IXL_RSS_HENA_DEFAULT_XL710; 4835 4836 hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0)); 4837 hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1)); 4838 4839 SET(hena0, set_hena); 4840 SET(hena1, set_hena >> 32); 4841 4842 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0); 4843 ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1); 4844} 4845 4846static int 4847ixl_register_rss_hlut(struct ixl_softc *sc) 4848{ 4849 unsigned int qid; 4850 uint8_t hlut_buf[512], lut_mask; 4851 uint32_t *hluts; 4852 size_t i, hluts_num; 4853 int rv; 4854 4855 lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1; 4856 4857 for (i = 0; i < sc->sc_rss_table_size; i++) { 4858 qid = i % sc->sc_nqueue_pairs; 4859 hlut_buf[i] = qid & lut_mask; 4860 } 4861 4862 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) { 4863 rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf)); 4864 } else { 4865 rv = 0; 4866 hluts = (uint32_t *)hlut_buf; 4867 hluts_num = sc->sc_rss_table_size >> 2; 4868 for (i = 0; i < hluts_num; i++) { 4869 ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]); 4870 } 4871 ixl_flush(sc); 4872 } 4873 4874 return rv; 4875} 4876 4877static void 4878ixl_config_rss(struct ixl_softc *sc) 4879{ 4880 4881 KASSERT(mutex_owned(&sc->sc_cfg_lock)); 4882 4883 ixl_register_rss_key(sc); 4884 ixl_register_rss_pctype(sc); 4885 ixl_register_rss_hlut(sc); 4886} 4887 4888static const struct ixl_phy_type * 4889ixl_search_phy_type(uint8_t phy_type) 4890{ 4891 const struct ixl_phy_type *itype; 4892 uint64_t mask; 4893 unsigned int i; 4894 4895 if (phy_type >= 64) 4896 return NULL; 4897 4898 mask = 1ULL << phy_type; 4899 4900 for (i = 0; i < __arraycount(ixl_phy_type_map); i++) { 4901 itype = &ixl_phy_type_map[i]; 4902 4903 if (ISSET(itype->phy_type, mask)) 4904 return itype; 4905 } 4906 4907 return NULL; 4908} 4909 4910static uint64_t 4911ixl_search_link_speed(uint8_t link_speed) 4912{ 4913 const struct ixl_speed_type *type; 4914 unsigned int i; 4915 4916 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4917 type = &ixl_speed_type_map[i]; 4918 4919 if (ISSET(type->dev_speed, link_speed)) 4920 return type->net_speed; 4921 } 4922 4923 return 0; 4924} 4925 4926static uint8_t 4927ixl_search_baudrate(uint64_t baudrate) 4928{ 4929 const struct ixl_speed_type *type; 4930 unsigned int i; 4931 4932 for (i = 0; i < __arraycount(ixl_speed_type_map); i++) { 4933 type = &ixl_speed_type_map[i]; 4934 4935 if (type->net_speed == baudrate) { 4936 return type->dev_speed; 4937 } 4938 } 4939 4940 return 0; 4941} 4942 4943static int 4944ixl_restart_an(struct ixl_softc *sc) 4945{ 4946 struct ixl_aq_desc iaq; 4947 4948 memset(&iaq, 0, sizeof(iaq)); 4949 iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN); 4950 iaq.iaq_param[0] = 4951 htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE); 4952 4953 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4954 aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n"); 4955 return -1; 4956 } 4957 if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) { 4958 aprint_error_dev(sc->sc_dev, "RESTART AN error\n"); 4959 return -1; 4960 } 4961 4962 return 0; 4963} 4964 4965static int 4966ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 4967 uint16_t vlan, uint16_t flags) 4968{ 4969 struct ixl_aq_desc iaq; 4970 struct ixl_aq_add_macvlan *param; 4971 struct ixl_aq_add_macvlan_elem *elem; 4972 4973 memset(&iaq, 0, sizeof(iaq)); 4974 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 4975 iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN); 4976 iaq.iaq_datalen = htole16(sizeof(*elem)); 4977 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 4978 4979 param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param; 4980 param->num_addrs = htole16(1); 4981 param->seid0 = htole16(0x8000) | sc->sc_seid; 4982 param->seid1 = 0; 4983 param->seid2 = 0; 4984 4985 elem = IXL_DMA_KVA(&sc->sc_scratch); 4986 memset(elem, 0, sizeof(*elem)); 4987 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 4988 elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags); 4989 elem->vlan = htole16(vlan); 4990 4991 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 4992 return IXL_AQ_RC_EINVAL; 4993 } 4994 4995 switch (le16toh(iaq.iaq_retval)) { 4996 case IXL_AQ_RC_OK: 4997 break; 4998 case IXL_AQ_RC_ENOSPC: 4999 return ENOSPC; 5000 case IXL_AQ_RC_ENOENT: 5001 return ENOENT; 5002 case IXL_AQ_RC_EACCES: 5003 return EACCES; 5004 case IXL_AQ_RC_EEXIST: 5005 return EEXIST; 5006 case IXL_AQ_RC_EINVAL: 5007 return EINVAL; 5008 default: 5009 return EIO; 5010 } 5011 5012 return 0; 5013} 5014 5015static int 5016ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr, 5017 uint16_t vlan, uint16_t flags) 5018{ 5019 struct ixl_aq_desc iaq; 5020 struct ixl_aq_remove_macvlan *param; 5021 struct ixl_aq_remove_macvlan_elem *elem; 5022 5023 memset(&iaq, 0, sizeof(iaq)); 5024 iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD); 5025 iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN); 5026 iaq.iaq_datalen = htole16(sizeof(*elem)); 5027 ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch)); 5028 5029 param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param; 5030 param->num_addrs = htole16(1); 5031 param->seid0 = htole16(0x8000) | sc->sc_seid; 5032 param->seid1 = 0; 5033 param->seid2 = 0; 5034 5035 elem = IXL_DMA_KVA(&sc->sc_scratch); 5036 memset(elem, 0, sizeof(*elem)); 5037 memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN); 5038 elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags); 5039 elem->vlan = htole16(vlan); 5040 5041 if (ixl_atq_poll(sc, &iaq, 250) != 0) { 5042 return EINVAL; 5043 } 5044 5045 switch (le16toh(iaq.iaq_retval)) { 5046 case IXL_AQ_RC_OK: 5047 break; 5048 case IXL_AQ_RC_ENOENT: 5049 return ENOENT; 5050 case IXL_AQ_RC_EACCES: 5051 return EACCES; 5052 case IXL_AQ_RC_EINVAL: 5053 return EINVAL; 5054 default: 5055 return EIO; 5056 } 5057 5058 return 0; 5059} 5060 5061static int 5062ixl_hmc(struct ixl_softc *sc) 5063{ 5064 struct { 5065 uint32_t count; 5066 uint32_t minsize; 5067 bus_size_t objsiz; 5068 bus_size_t setoff; 5069 bus_size_t setcnt; 5070 } regs[] = { 5071 { 5072 0, 5073 IXL_HMC_TXQ_MINSIZE, 5074 I40E_GLHMC_LANTXOBJSZ, 5075 I40E_GLHMC_LANTXBASE(sc->sc_pf_id), 5076 I40E_GLHMC_LANTXCNT(sc->sc_pf_id), 5077 }, 5078 { 5079 0, 5080 IXL_HMC_RXQ_MINSIZE, 5081 I40E_GLHMC_LANRXOBJSZ, 5082 I40E_GLHMC_LANRXBASE(sc->sc_pf_id), 5083 I40E_GLHMC_LANRXCNT(sc->sc_pf_id), 5084 }, 5085 { 5086 0, 5087 0, 5088 I40E_GLHMC_FCOEDDPOBJSZ, 5089 I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id), 5090 I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id), 5091 }, 5092 { 5093 0, 5094 0, 5095 I40E_GLHMC_FCOEFOBJSZ, 5096 I40E_GLHMC_FCOEFBASE(sc->sc_pf_id), 5097 I40E_GLHMC_FCOEFCNT(sc->sc_pf_id), 5098 }, 5099 }; 5100 struct ixl_hmc_entry *e; 5101 uint64_t size, dva; 5102 uint8_t *kva; 5103 uint64_t *sdpage; 5104 unsigned int i; 5105 int npages, tables; 5106 uint32_t reg; 5107 5108 CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries)); 5109 5110 regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count = 5111 ixl_rd(sc, I40E_GLHMC_LANQMAX); 5112 5113 size = 0; 5114 for (i = 0; i < __arraycount(regs); i++) { 5115 e = &sc->sc_hmc_entries[i]; 5116 5117 e->hmc_count = regs[i].count; 5118 reg = ixl_rd(sc, regs[i].objsiz); 5119 e->hmc_size = IXL_BIT_ULL(0x3F & reg); 5120 e->hmc_base = size; 5121 5122 if ((e->hmc_size * 8) < regs[i].minsize) { 5123 aprint_error_dev(sc->sc_dev, 5124 "kernel hmc entry is too big\n"); 5125 return -1; 5126 } 5127 5128 size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP); 5129 } 5130 size = roundup(size, IXL_HMC_PGSIZE); 5131 npages = size / IXL_HMC_PGSIZE; 5132 5133 tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ; 5134 5135 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) { 5136 aprint_error_dev(sc->sc_dev, 5137 "unable to allocate hmc pd memory\n"); 5138 return -1; 5139 } 5140 5141 if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE, 5142 IXL_HMC_PGSIZE) != 0) { 5143 aprint_error_dev(sc->sc_dev, 5144 "unable to allocate hmc sd memory\n"); 5145 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5146 return -1; 5147 } 5148 5149 kva = IXL_DMA_KVA(&sc->sc_hmc_pd); 5150 memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd)); 5151 5152 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd), 5153 0, IXL_DMA_LEN(&sc->sc_hmc_pd), 5154 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5155 5156 dva = IXL_DMA_DVA(&sc->sc_hmc_pd); 5157 sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd); 5158 memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd)); 5159 5160 for (i = 0; (int)i < npages; i++) { 5161 *sdpage = htole64(dva | IXL_HMC_PDVALID); 5162 sdpage++; 5163 5164 dva += IXL_HMC_PGSIZE; 5165 } 5166 5167 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd), 5168 0, IXL_DMA_LEN(&sc->sc_hmc_sd), 5169 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 5170 5171 dva = IXL_DMA_DVA(&sc->sc_hmc_sd); 5172 for (i = 0; (int)i < tables; i++) { 5173 uint32_t count; 5174 5175 KASSERT(npages >= 0); 5176 5177 count = ((unsigned int)npages > IXL_HMC_PGS) ? 5178 IXL_HMC_PGS : (unsigned int)npages; 5179 5180 ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32); 5181 ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva | 5182 (count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) | 5183 (1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)); 5184 ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE); 5185 ixl_wr(sc, I40E_PFHMC_SDCMD, 5186 (1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i); 5187 5188 npages -= IXL_HMC_PGS; 5189 dva += IXL_HMC_PGSIZE; 5190 } 5191 5192 for (i = 0; i < __arraycount(regs); i++) { 5193 e = &sc->sc_hmc_entries[i]; 5194 5195 ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP); 5196 ixl_wr(sc, regs[i].setcnt, e->hmc_count); 5197 } 5198 5199 return 0; 5200} 5201 5202static void 5203ixl_hmc_free(struct ixl_softc *sc) 5204{ 5205 ixl_dmamem_free(sc, &sc->sc_hmc_sd); 5206 ixl_dmamem_free(sc, &sc->sc_hmc_pd); 5207} 5208 5209static void 5210ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing, 5211 unsigned int npacking) 5212{ 5213 uint8_t *dst = d; 5214 const uint8_t *src = s; 5215 unsigned int i; 5216 5217 for (i = 0; i < npacking; i++) { 5218 const struct ixl_hmc_pack *pack = &packing[i]; 5219 unsigned int offset = pack->lsb / 8; 5220 unsigned int align = pack->lsb % 8; 5221 const uint8_t *in = src + pack->offset; 5222 uint8_t *out = dst + offset; 5223 int width = pack->width; 5224 unsigned int inbits = 0; 5225 5226 if (align) { 5227 inbits = (*in++) << align; 5228 *out++ |= (inbits & 0xff); 5229 inbits >>= 8; 5230 5231 width -= 8 - align; 5232 } 5233 5234 while (width >= 8) { 5235 inbits |= (*in++) << align; 5236 *out++ = (inbits & 0xff); 5237 inbits >>= 8; 5238 5239 width -= 8; 5240 } 5241 5242 if (width > 0) { 5243 inbits |= (*in) << align; 5244 *out |= (inbits & ((1 << width) - 1)); 5245 } 5246 } 5247} 5248 5249static struct ixl_aq_buf * 5250ixl_aqb_alloc(struct ixl_softc *sc) 5251{ 5252 struct ixl_aq_buf *aqb; 5253 5254 aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP); 5255 5256 aqb->aqb_size = IXL_AQ_BUFLEN; 5257 5258 if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1, 5259 aqb->aqb_size, 0, 5260 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0) 5261 goto free; 5262 if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size, 5263 IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs, 5264 BUS_DMA_WAITOK) != 0) 5265 goto destroy; 5266 if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs, 5267 aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0) 5268 goto dma_free; 5269 if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data, 5270 aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0) 5271 goto unmap; 5272 5273 return aqb; 5274unmap: 5275 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5276dma_free: 5277 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5278destroy: 5279 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5280free: 5281 kmem_free(aqb, sizeof(*aqb)); 5282 5283 return NULL; 5284} 5285 5286static void 5287ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb) 5288{ 5289 5290 bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map); 5291 bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size); 5292 bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1); 5293 bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map); 5294 kmem_free(aqb, sizeof(*aqb)); 5295} 5296 5297static int 5298ixl_arq_fill(struct ixl_softc *sc) 5299{ 5300 struct ixl_aq_buf *aqb; 5301 struct ixl_aq_desc *arq, *iaq; 5302 unsigned int prod = sc->sc_arq_prod; 5303 unsigned int n; 5304 int post = 0; 5305 5306 n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons, 5307 IXL_AQ_NUM); 5308 arq = IXL_DMA_KVA(&sc->sc_arq); 5309 5310 if (__predict_false(n <= 0)) 5311 return 0; 5312 5313 do { 5314 aqb = sc->sc_arq_live[prod]; 5315 iaq = &arq[prod]; 5316 5317 if (aqb == NULL) { 5318 aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle); 5319 if (aqb != NULL) { 5320 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5321 ixl_aq_buf, aqb_entry); 5322 } else if ((aqb = ixl_aqb_alloc(sc)) == NULL) { 5323 break; 5324 } 5325 5326 sc->sc_arq_live[prod] = aqb; 5327 memset(aqb->aqb_data, 0, aqb->aqb_size); 5328 5329 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, 5330 aqb->aqb_size, BUS_DMASYNC_PREREAD); 5331 5332 iaq->iaq_flags = htole16(IXL_AQ_BUF | 5333 (IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? 5334 IXL_AQ_LB : 0)); 5335 iaq->iaq_opcode = 0; 5336 iaq->iaq_datalen = htole16(aqb->aqb_size); 5337 iaq->iaq_retval = 0; 5338 iaq->iaq_cookie = 0; 5339 iaq->iaq_param[0] = 0; 5340 iaq->iaq_param[1] = 0; 5341 ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr); 5342 } 5343 5344 prod++; 5345 prod &= IXL_AQ_MASK; 5346 5347 post = 1; 5348 5349 } while (--n); 5350 5351 if (post) { 5352 sc->sc_arq_prod = prod; 5353 ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod); 5354 } 5355 5356 return post; 5357} 5358 5359static void 5360ixl_arq_unfill(struct ixl_softc *sc) 5361{ 5362 struct ixl_aq_buf *aqb; 5363 unsigned int i; 5364 5365 for (i = 0; i < __arraycount(sc->sc_arq_live); i++) { 5366 aqb = sc->sc_arq_live[i]; 5367 if (aqb == NULL) 5368 continue; 5369 5370 sc->sc_arq_live[i] = NULL; 5371 bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size, 5372 BUS_DMASYNC_POSTREAD); 5373 ixl_aqb_free(sc, aqb); 5374 } 5375 5376 while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) { 5377 SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb, 5378 ixl_aq_buf, aqb_entry); 5379 ixl_aqb_free(sc, aqb); 5380 } 5381} 5382 5383static void 5384ixl_clear_hw(struct ixl_softc *sc) 5385{ 5386 uint32_t num_queues, base_queue; 5387 uint32_t num_pf_int; 5388 uint32_t num_vf_int; 5389 uint32_t num_vfs; 5390 uint32_t i, j; 5391 uint32_t val; 5392 uint32_t eol = 0x7ff; 5393 5394 /* get number of interrupts, queues, and vfs */ 5395 val = ixl_rd(sc, I40E_GLPCI_CNF2); 5396 num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 5397 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 5398 num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >> 5399 I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT; 5400 5401 val = ixl_rd(sc, I40E_PFLAN_QALLOC); 5402 base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >> 5403 I40E_PFLAN_QALLOC_FIRSTQ_SHIFT; 5404 j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >> 5405 I40E_PFLAN_QALLOC_LASTQ_SHIFT; 5406 if (val & I40E_PFLAN_QALLOC_VALID_MASK) 5407 num_queues = (j - base_queue) + 1; 5408 else 5409 num_queues = 0; 5410 5411 val = ixl_rd(sc, I40E_PF_VT_PFALLOC); 5412 i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >> 5413 I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT; 5414 j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >> 5415 I40E_PF_VT_PFALLOC_LASTVF_SHIFT; 5416 if (val & I40E_PF_VT_PFALLOC_VALID_MASK) 5417 num_vfs = (j - i) + 1; 5418 else 5419 num_vfs = 0; 5420 5421 /* stop all the interrupts */ 5422 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5423 ixl_flush(sc); 5424 val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 5425 for (i = 0; i < num_pf_int - 2; i++) 5426 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val); 5427 ixl_flush(sc); 5428 5429 /* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */ 5430 val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5431 ixl_wr(sc, I40E_PFINT_LNKLST0, val); 5432 for (i = 0; i < num_pf_int - 2; i++) 5433 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val); 5434 val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5435 for (i = 0; i < num_vfs; i++) 5436 ixl_wr(sc, I40E_VPINT_LNKLST0(i), val); 5437 for (i = 0; i < num_vf_int - 2; i++) 5438 ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val); 5439 5440 /* warn the HW of the coming Tx disables */ 5441 for (i = 0; i < num_queues; i++) { 5442 uint32_t abs_queue_idx = base_queue + i; 5443 uint32_t reg_block = 0; 5444 5445 if (abs_queue_idx >= 128) { 5446 reg_block = abs_queue_idx / 128; 5447 abs_queue_idx %= 128; 5448 } 5449 5450 val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block)); 5451 val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK; 5452 val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT); 5453 val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK; 5454 5455 ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val); 5456 } 5457 delaymsec(400); 5458 5459 /* stop all the queues */ 5460 for (i = 0; i < num_queues; i++) { 5461 ixl_wr(sc, I40E_QINT_TQCTL(i), 0); 5462 ixl_wr(sc, I40E_QTX_ENA(i), 0); 5463 ixl_wr(sc, I40E_QINT_RQCTL(i), 0); 5464 ixl_wr(sc, I40E_QRX_ENA(i), 0); 5465 } 5466 5467 /* short wait for all queue disables to settle */ 5468 delaymsec(50); 5469} 5470 5471static int 5472ixl_pf_reset(struct ixl_softc *sc) 5473{ 5474 uint32_t cnt = 0; 5475 uint32_t cnt1 = 0; 5476 uint32_t reg = 0, reg0 = 0; 5477 uint32_t grst_del; 5478 5479 /* 5480 * Poll for Global Reset steady state in case of recent GRST. 5481 * The grst delay value is in 100ms units, and we'll wait a 5482 * couple counts longer to be sure we don't just miss the end. 5483 */ 5484 grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL); 5485 grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK; 5486 grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; 5487 5488 grst_del = grst_del * 20; 5489 5490 for (cnt = 0; cnt < grst_del; cnt++) { 5491 reg = ixl_rd(sc, I40E_GLGEN_RSTAT); 5492 if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK)) 5493 break; 5494 delaymsec(100); 5495 } 5496 if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5497 aprint_error(", Global reset polling failed to complete\n"); 5498 return -1; 5499 } 5500 5501 /* Now Wait for the FW to be ready */ 5502 for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) { 5503 reg = ixl_rd(sc, I40E_GLNVM_ULD); 5504 reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5505 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK); 5506 if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5507 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK)) 5508 break; 5509 5510 delaymsec(10); 5511 } 5512 if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK | 5513 I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) { 5514 aprint_error(", wait for FW Reset complete timed out " 5515 "(I40E_GLNVM_ULD = 0x%x)\n", reg); 5516 return -1; 5517 } 5518 5519 /* 5520 * If there was a Global Reset in progress when we got here, 5521 * we don't need to do the PF Reset 5522 */ 5523 if (cnt == 0) { 5524 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5525 ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK); 5526 for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) { 5527 reg = ixl_rd(sc, I40E_PFGEN_CTRL); 5528 if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) 5529 break; 5530 delaymsec(1); 5531 5532 reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT); 5533 if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { 5534 aprint_error(", Core reset upcoming." 5535 " Skipping PF reset reset request\n"); 5536 return -1; 5537 } 5538 } 5539 if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { 5540 aprint_error(", PF reset polling failed to complete" 5541 "(I40E_PFGEN_CTRL= 0x%x)\n", reg); 5542 return -1; 5543 } 5544 } 5545 5546 return 0; 5547} 5548 5549static int 5550ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm, 5551 bus_size_t size, bus_size_t align) 5552{ 5553 ixm->ixm_size = size; 5554 5555 if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1, 5556 ixm->ixm_size, 0, 5557 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 5558 &ixm->ixm_map) != 0) 5559 return 1; 5560 if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size, 5561 align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs, 5562 BUS_DMA_WAITOK) != 0) 5563 goto destroy; 5564 if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs, 5565 ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0) 5566 goto free; 5567 if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva, 5568 ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0) 5569 goto unmap; 5570 5571 memset(ixm->ixm_kva, 0, ixm->ixm_size); 5572 5573 return 0; 5574unmap: 5575 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5576free: 5577 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5578destroy: 5579 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5580 return 1; 5581} 5582 5583static void 5584ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm) 5585{ 5586 bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map); 5587 bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size); 5588 bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1); 5589 bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map); 5590} 5591 5592static int 5593ixl_setup_vlan_hwfilter(struct ixl_softc *sc) 5594{ 5595 struct ethercom *ec = &sc->sc_ec; 5596 struct vlanid_list *vlanidp; 5597 int rv; 5598 5599 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5600 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5601 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5602 IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN); 5603 5604 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5605 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5606 if (rv != 0) 5607 return rv; 5608 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5609 IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5610 if (rv != 0) 5611 return rv; 5612 5613 ETHER_LOCK(ec); 5614 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5615 rv = ixl_add_macvlan(sc, sc->sc_enaddr, 5616 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5617 if (rv != 0) 5618 break; 5619 rv = ixl_add_macvlan(sc, etherbroadcastaddr, 5620 vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH); 5621 if (rv != 0) 5622 break; 5623 } 5624 ETHER_UNLOCK(ec); 5625 5626 return rv; 5627} 5628 5629static void 5630ixl_teardown_vlan_hwfilter(struct ixl_softc *sc) 5631{ 5632 struct vlanid_list *vlanidp; 5633 struct ethercom *ec = &sc->sc_ec; 5634 5635 ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 5636 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5637 ixl_remove_macvlan(sc, etherbroadcastaddr, 0, 5638 IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5639 5640 ETHER_LOCK(ec); 5641 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 5642 ixl_remove_macvlan(sc, sc->sc_enaddr, 5643 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5644 ixl_remove_macvlan(sc, etherbroadcastaddr, 5645 vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH); 5646 } 5647 ETHER_UNLOCK(ec); 5648 5649 ixl_add_macvlan(sc, sc->sc_enaddr, 0, 5650 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5651 ixl_add_macvlan(sc, etherbroadcastaddr, 0, 5652 IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN); 5653} 5654 5655static int 5656ixl_update_macvlan(struct ixl_softc *sc) 5657{ 5658 int rv = 0; 5659 int next_ec_capenable = sc->sc_ec.ec_capenable; 5660 5661 if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) { 5662 rv = ixl_setup_vlan_hwfilter(sc); 5663 if (rv != 0) 5664 ixl_teardown_vlan_hwfilter(sc); 5665 } else { 5666 ixl_teardown_vlan_hwfilter(sc); 5667 } 5668 5669 return rv; 5670} 5671 5672static int 5673ixl_ifflags_cb(struct ethercom *ec) 5674{ 5675 struct ifnet *ifp = &ec->ec_if; 5676 struct ixl_softc *sc = ifp->if_softc; 5677 int rv, change, reset_bits; 5678 5679 mutex_enter(&sc->sc_cfg_lock); 5680 5681 change = ec->ec_capenable ^ sc->sc_cur_ec_capenable; 5682 reset_bits = change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU); 5683 if (reset_bits != 0) { 5684 sc->sc_cur_ec_capenable ^= reset_bits; 5685 rv = ENETRESET; 5686 goto out; 5687 } 5688 5689 if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) { 5690 rv = ixl_update_macvlan(sc); 5691 if (rv == 0) { 5692 sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER; 5693 } else { 5694 CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER); 5695 CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER); 5696 } 5697 } 5698 5699 rv = ixl_iff(sc); 5700out: 5701 mutex_exit(&sc->sc_cfg_lock); 5702 5703 return rv; 5704} 5705 5706static int 5707ixl_set_link_status_locked(struct ixl_softc *sc, const struct ixl_aq_desc *iaq) 5708{ 5709 const struct ixl_aq_link_status *status; 5710 const struct ixl_phy_type *itype; 5711 5712 uint64_t ifm_active = IFM_ETHER; 5713 uint64_t ifm_status = IFM_AVALID; 5714 int link_state = LINK_STATE_DOWN; 5715 uint64_t baudrate = 0; 5716 5717 status = (const struct ixl_aq_link_status *)iaq->iaq_param; 5718 if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) { 5719 ifm_active |= IFM_NONE; 5720 goto done; 5721 } 5722 5723 ifm_active |= IFM_FDX; 5724 ifm_status |= IFM_ACTIVE; 5725 link_state = LINK_STATE_UP; 5726 5727 itype = ixl_search_phy_type(status->phy_type); 5728 if (itype != NULL) 5729 ifm_active |= itype->ifm_type; 5730 5731 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX)) 5732 ifm_active |= IFM_ETH_TXPAUSE; 5733 if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX)) 5734 ifm_active |= IFM_ETH_RXPAUSE; 5735 5736 baudrate = ixl_search_link_speed(status->link_speed); 5737 5738done: 5739 /* sc->sc_cfg_lock held expect during attach */ 5740 sc->sc_media_active = ifm_active; 5741 sc->sc_media_status = ifm_status; 5742 5743 sc->sc_ec.ec_if.if_baudrate = baudrate; 5744 5745 return link_state; 5746} 5747 5748static int 5749ixl_establish_intx(struct ixl_softc *sc) 5750{ 5751 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5752 pci_intr_handle_t *intr; 5753 char xnamebuf[32]; 5754 char intrbuf[PCI_INTRSTR_LEN]; 5755 char const *intrstr; 5756 5757 KASSERT(sc->sc_nintrs == 1); 5758 5759 intr = &sc->sc_ihp[0]; 5760 5761 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 5762 snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy", 5763 device_xname(sc->sc_dev)); 5764 5765 sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr, 5766 sc, xnamebuf); 5767 5768 if (sc->sc_ihs[0] == NULL) { 5769 aprint_error_dev(sc->sc_dev, 5770 "unable to establish interrupt at %s\n", intrstr); 5771 return -1; 5772 } 5773 5774 aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 5775 return 0; 5776} 5777 5778static int 5779ixl_establish_msix(struct ixl_softc *sc) 5780{ 5781 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 5782 kcpuset_t *affinity; 5783 unsigned int vector = 0; 5784 unsigned int i; 5785 int affinity_to, r; 5786 char xnamebuf[32]; 5787 char intrbuf[PCI_INTRSTR_LEN]; 5788 char const *intrstr; 5789 5790 kcpuset_create(&affinity, false); 5791 5792 /* the "other" intr is mapped to vector 0 */ 5793 vector = 0; 5794 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5795 intrbuf, sizeof(intrbuf)); 5796 snprintf(xnamebuf, sizeof(xnamebuf), "%s others", 5797 device_xname(sc->sc_dev)); 5798 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5799 sc->sc_ihp[vector], IPL_NET, ixl_other_intr, 5800 sc, xnamebuf); 5801 if (sc->sc_ihs[vector] == NULL) { 5802 aprint_error_dev(sc->sc_dev, 5803 "unable to establish interrupt at %s\n", intrstr); 5804 goto fail; 5805 } 5806 5807 aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr); 5808 5809 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5810 affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu; 5811 5812 kcpuset_zero(affinity); 5813 kcpuset_set(affinity, affinity_to); 5814 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5815 if (r == 0) { 5816 aprint_normal(", affinity to %u", affinity_to); 5817 } 5818 aprint_normal("\n"); 5819 vector++; 5820 5821 sc->sc_msix_vector_queue = vector; 5822 affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0; 5823 5824 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 5825 intrstr = pci_intr_string(pc, sc->sc_ihp[vector], 5826 intrbuf, sizeof(intrbuf)); 5827 snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d", 5828 device_xname(sc->sc_dev), i); 5829 5830 sc->sc_ihs[vector] = pci_intr_establish_xname(pc, 5831 sc->sc_ihp[vector], IPL_NET, ixl_queue_intr, 5832 (void *)&sc->sc_qps[i], xnamebuf); 5833 5834 if (sc->sc_ihs[vector] == NULL) { 5835 aprint_error_dev(sc->sc_dev, 5836 "unable to establish interrupt at %s\n", intrstr); 5837 goto fail; 5838 } 5839 5840 aprint_normal_dev(sc->sc_dev, 5841 "for TXRX%d interrupt at %s", i, intrstr); 5842 5843 kcpuset_zero(affinity); 5844 kcpuset_set(affinity, affinity_to); 5845 r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL); 5846 if (r == 0) { 5847 aprint_normal(", affinity to %u", affinity_to); 5848 affinity_to = (affinity_to + 1) % ncpu; 5849 } 5850 aprint_normal("\n"); 5851 vector++; 5852 } 5853 5854 kcpuset_destroy(affinity); 5855 5856 return 0; 5857fail: 5858 for (i = 0; i < vector; i++) { 5859 pci_intr_disestablish(pc, sc->sc_ihs[i]); 5860 } 5861 5862 sc->sc_msix_vector_queue = 0; 5863 sc->sc_msix_vector_queue = 0; 5864 kcpuset_destroy(affinity); 5865 5866 return -1; 5867} 5868 5869static void 5870ixl_config_queue_intr(struct ixl_softc *sc) 5871{ 5872 unsigned int i, vector; 5873 5874 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5875 vector = sc->sc_msix_vector_queue; 5876 } else { 5877 vector = I40E_INTR_NOTX_INTR; 5878 5879 ixl_wr(sc, I40E_PFINT_LNKLST0, 5880 (I40E_INTR_NOTX_QUEUE << 5881 I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5882 (I40E_QUEUE_TYPE_RX << 5883 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5884 } 5885 5886 for (i = 0; i < sc->sc_nqueue_pairs; i++) { 5887 ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0); 5888 ixl_flush(sc); 5889 5890 ixl_wr(sc, I40E_PFINT_LNKLSTN(i), 5891 ((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 5892 (I40E_QUEUE_TYPE_RX << 5893 I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); 5894 5895 ixl_wr(sc, I40E_QINT_RQCTL(i), 5896 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5897 (I40E_ITR_INDEX_RX << 5898 I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5899 (I40E_INTR_NOTX_RX_QUEUE << 5900 I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5901 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5902 (I40E_QUEUE_TYPE_TX << 5903 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5904 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5905 5906 ixl_wr(sc, I40E_QINT_TQCTL(i), 5907 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5908 (I40E_ITR_INDEX_TX << 5909 I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5910 (I40E_INTR_NOTX_TX_QUEUE << 5911 I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5912 (I40E_QUEUE_TYPE_EOL << 5913 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5914 (I40E_QUEUE_TYPE_RX << 5915 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 5916 I40E_QINT_TQCTL_CAUSE_ENA_MASK); 5917 5918 if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) { 5919 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_RX, i), 5920 sc->sc_itr_rx); 5921 ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_TX, i), 5922 sc->sc_itr_tx); 5923 vector++; 5924 } 5925 } 5926 ixl_flush(sc); 5927 5928 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), sc->sc_itr_rx); 5929 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), sc->sc_itr_tx); 5930 ixl_flush(sc); 5931} 5932 5933static void 5934ixl_config_other_intr(struct ixl_softc *sc) 5935{ 5936 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0); 5937 (void)ixl_rd(sc, I40E_PFINT_ICR0); 5938 5939 ixl_wr(sc, I40E_PFINT_ICR0_ENA, 5940 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 5941 I40E_PFINT_ICR0_ENA_GRST_MASK | 5942 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 5943 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 5944 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 5945 I40E_PFINT_ICR0_ENA_VFLR_MASK | 5946 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | 5947 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 5948 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK); 5949 5950 ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF); 5951 ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0); 5952 ixl_wr(sc, I40E_PFINT_STAT_CTL0, 5953 (I40E_ITR_INDEX_OTHER << 5954 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT)); 5955 ixl_flush(sc); 5956} 5957 5958static int 5959ixl_setup_interrupts(struct ixl_softc *sc) 5960{ 5961 struct pci_attach_args *pa = &sc->sc_pa; 5962 pci_intr_type_t max_type, intr_type; 5963 int counts[PCI_INTR_TYPE_SIZE]; 5964 int error; 5965 unsigned int i; 5966 bool retry; 5967 5968 memset(counts, 0, sizeof(counts)); 5969 max_type = PCI_INTR_TYPE_MSIX; 5970 /* QPs + other interrupt */ 5971 counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1; 5972 counts[PCI_INTR_TYPE_INTX] = 1; 5973 5974 if (ixl_param_nomsix) 5975 counts[PCI_INTR_TYPE_MSIX] = 0; 5976 5977 do { 5978 retry = false; 5979 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 5980 if (error != 0) { 5981 aprint_error_dev(sc->sc_dev, 5982 "couldn't map interrupt\n"); 5983 break; 5984 } 5985 5986 intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]); 5987 sc->sc_nintrs = counts[intr_type]; 5988 KASSERT(sc->sc_nintrs > 0); 5989 5990 for (i = 0; i < sc->sc_nintrs; i++) { 5991 pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i], 5992 PCI_INTR_MPSAFE, true); 5993 } 5994 5995 sc->sc_ihs = kmem_zalloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs, 5996 KM_SLEEP); 5997 5998 if (intr_type == PCI_INTR_TYPE_MSIX) { 5999 error = ixl_establish_msix(sc); 6000 if (error) { 6001 counts[PCI_INTR_TYPE_MSIX] = 0; 6002 retry = true; 6003 } 6004 } else if (intr_type == PCI_INTR_TYPE_INTX) { 6005 error = ixl_establish_intx(sc); 6006 } else { 6007 error = -1; 6008 } 6009 6010 if (error) { 6011 kmem_free(sc->sc_ihs, 6012 sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6013 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6014 } else { 6015 sc->sc_intrtype = intr_type; 6016 } 6017 } while (retry); 6018 6019 return error; 6020} 6021 6022static void 6023ixl_teardown_interrupts(struct ixl_softc *sc) 6024{ 6025 struct pci_attach_args *pa = &sc->sc_pa; 6026 unsigned int i; 6027 6028 for (i = 0; i < sc->sc_nintrs; i++) { 6029 pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]); 6030 } 6031 6032 pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs); 6033 6034 kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs); 6035 sc->sc_ihs = NULL; 6036 sc->sc_nintrs = 0; 6037} 6038 6039static int 6040ixl_setup_stats(struct ixl_softc *sc) 6041{ 6042 struct ixl_queue_pair *qp; 6043 struct ixl_tx_ring *txr; 6044 struct ixl_rx_ring *rxr; 6045 struct ixl_stats_counters *isc; 6046 unsigned int i; 6047 6048 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6049 qp = &sc->sc_qps[i]; 6050 txr = qp->qp_txr; 6051 rxr = qp->qp_rxr; 6052 6053 evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC, 6054 NULL, qp->qp_name, "m_defrag successed"); 6055 evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC, 6056 NULL, qp->qp_name, "m_defrag_failed"); 6057 evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC, 6058 NULL, qp->qp_name, "Dropped in pcq"); 6059 evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC, 6060 NULL, qp->qp_name, "Deferred transmit"); 6061 evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR, 6062 NULL, qp->qp_name, "Interrupt on queue"); 6063 evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC, 6064 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6065 6066 evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC, 6067 NULL, qp->qp_name, "MGETHDR failed"); 6068 evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC, 6069 NULL, qp->qp_name, "MCLGET failed"); 6070 evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed, 6071 EVCNT_TYPE_MISC, NULL, qp->qp_name, 6072 "bus_dmamap_load_mbuf failed"); 6073 evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR, 6074 NULL, qp->qp_name, "Interrupt on queue"); 6075 evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC, 6076 NULL, qp->qp_name, "Handled queue in softint/workqueue"); 6077 } 6078 6079 evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR, 6080 NULL, device_xname(sc->sc_dev), "Interrupt for other events"); 6081 evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC, 6082 NULL, device_xname(sc->sc_dev), "Link status event"); 6083 evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC, 6084 NULL, device_xname(sc->sc_dev), "ECC error"); 6085 evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC, 6086 NULL, device_xname(sc->sc_dev), "PCI exception"); 6087 evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC, 6088 NULL, device_xname(sc->sc_dev), "Critical error"); 6089 6090 isc = &sc->sc_stats_counters; 6091 evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC, 6092 NULL, device_xname(sc->sc_dev), "CRC errors"); 6093 evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC, 6094 NULL, device_xname(sc->sc_dev), "Illegal bytes"); 6095 evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC, 6096 NULL, device_xname(sc->sc_dev), "Mac local faults"); 6097 evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC, 6098 NULL, device_xname(sc->sc_dev), "Mac remote faults"); 6099 evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC, 6100 NULL, device_xname(sc->sc_dev), "Rx xon"); 6101 evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC, 6102 NULL, device_xname(sc->sc_dev), "Tx xon"); 6103 evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC, 6104 NULL, device_xname(sc->sc_dev), "Rx xoff"); 6105 evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC, 6106 NULL, device_xname(sc->sc_dev), "Tx xoff"); 6107 evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC, 6108 NULL, device_xname(sc->sc_dev), "Rx fragments"); 6109 evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC, 6110 NULL, device_xname(sc->sc_dev), "Rx jabber"); 6111 6112 evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC, 6113 NULL, device_xname(sc->sc_dev), "Rx size 64"); 6114 evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC, 6115 NULL, device_xname(sc->sc_dev), "Rx size 127"); 6116 evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC, 6117 NULL, device_xname(sc->sc_dev), "Rx size 255"); 6118 evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC, 6119 NULL, device_xname(sc->sc_dev), "Rx size 511"); 6120 evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC, 6121 NULL, device_xname(sc->sc_dev), "Rx size 1023"); 6122 evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC, 6123 NULL, device_xname(sc->sc_dev), "Rx size 1522"); 6124 evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC, 6125 NULL, device_xname(sc->sc_dev), "Rx jumbo packets"); 6126 evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC, 6127 NULL, device_xname(sc->sc_dev), "Rx under size"); 6128 evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC, 6129 NULL, device_xname(sc->sc_dev), "Rx over size"); 6130 6131 evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC, 6132 NULL, device_xname(sc->sc_dev), "Rx bytes / port"); 6133 evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC, 6134 NULL, device_xname(sc->sc_dev), "Rx discards / port"); 6135 evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC, 6136 NULL, device_xname(sc->sc_dev), "Rx unicast / port"); 6137 evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC, 6138 NULL, device_xname(sc->sc_dev), "Rx multicast / port"); 6139 evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC, 6140 NULL, device_xname(sc->sc_dev), "Rx broadcast / port"); 6141 6142 evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC, 6143 NULL, device_xname(sc->sc_dev), "Rx bytes / vsi"); 6144 evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC, 6145 NULL, device_xname(sc->sc_dev), "Rx discards / vsi"); 6146 evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC, 6147 NULL, device_xname(sc->sc_dev), "Rx unicast / vsi"); 6148 evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC, 6149 NULL, device_xname(sc->sc_dev), "Rx multicast / vsi"); 6150 evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC, 6151 NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi"); 6152 6153 evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC, 6154 NULL, device_xname(sc->sc_dev), "Tx size 64"); 6155 evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC, 6156 NULL, device_xname(sc->sc_dev), "Tx size 127"); 6157 evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC, 6158 NULL, device_xname(sc->sc_dev), "Tx size 255"); 6159 evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC, 6160 NULL, device_xname(sc->sc_dev), "Tx size 511"); 6161 evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC, 6162 NULL, device_xname(sc->sc_dev), "Tx size 1023"); 6163 evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC, 6164 NULL, device_xname(sc->sc_dev), "Tx size 1522"); 6165 evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC, 6166 NULL, device_xname(sc->sc_dev), "Tx jumbo packets"); 6167 6168 evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC, 6169 NULL, device_xname(sc->sc_dev), "Tx bytes / port"); 6170 evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC, 6171 NULL, device_xname(sc->sc_dev), 6172 "Tx dropped due to link down / port"); 6173 evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC, 6174 NULL, device_xname(sc->sc_dev), "Tx unicast / port"); 6175 evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC, 6176 NULL, device_xname(sc->sc_dev), "Tx multicast / port"); 6177 evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC, 6178 NULL, device_xname(sc->sc_dev), "Tx broadcast / port"); 6179 6180 evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC, 6181 NULL, device_xname(sc->sc_dev), "Tx bytes / vsi"); 6182 evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC, 6183 NULL, device_xname(sc->sc_dev), "Tx errors / vsi"); 6184 evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC, 6185 NULL, device_xname(sc->sc_dev), "Tx unicast / vsi"); 6186 evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC, 6187 NULL, device_xname(sc->sc_dev), "Tx multicast / vsi"); 6188 evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC, 6189 NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi"); 6190 6191 sc->sc_stats_intval = ixl_param_stats_interval; 6192 callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE); 6193 callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc); 6194 ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc); 6195 6196 return 0; 6197} 6198 6199static void 6200ixl_teardown_stats(struct ixl_softc *sc) 6201{ 6202 struct ixl_tx_ring *txr; 6203 struct ixl_rx_ring *rxr; 6204 struct ixl_stats_counters *isc; 6205 unsigned int i; 6206 6207 for (i = 0; i < sc->sc_nqueue_pairs_max; i++) { 6208 txr = sc->sc_qps[i].qp_txr; 6209 rxr = sc->sc_qps[i].qp_rxr; 6210 6211 evcnt_detach(&txr->txr_defragged); 6212 evcnt_detach(&txr->txr_defrag_failed); 6213 evcnt_detach(&txr->txr_pcqdrop); 6214 evcnt_detach(&txr->txr_transmitdef); 6215 evcnt_detach(&txr->txr_intr); 6216 evcnt_detach(&txr->txr_defer); 6217 6218 evcnt_detach(&rxr->rxr_mgethdr_failed); 6219 evcnt_detach(&rxr->rxr_mgetcl_failed); 6220 evcnt_detach(&rxr->rxr_mbuf_load_failed); 6221 evcnt_detach(&rxr->rxr_intr); 6222 evcnt_detach(&rxr->rxr_defer); 6223 } 6224 6225 isc = &sc->sc_stats_counters; 6226 evcnt_detach(&isc->isc_crc_errors); 6227 evcnt_detach(&isc->isc_illegal_bytes); 6228 evcnt_detach(&isc->isc_mac_local_faults); 6229 evcnt_detach(&isc->isc_mac_remote_faults); 6230 evcnt_detach(&isc->isc_link_xon_rx); 6231 evcnt_detach(&isc->isc_link_xon_tx); 6232 evcnt_detach(&isc->isc_link_xoff_rx); 6233 evcnt_detach(&isc->isc_link_xoff_tx); 6234 evcnt_detach(&isc->isc_rx_fragments); 6235 evcnt_detach(&isc->isc_rx_jabber); 6236 evcnt_detach(&isc->isc_rx_bytes); 6237 evcnt_detach(&isc->isc_rx_discards); 6238 evcnt_detach(&isc->isc_rx_unicast); 6239 evcnt_detach(&isc->isc_rx_multicast); 6240 evcnt_detach(&isc->isc_rx_broadcast); 6241 evcnt_detach(&isc->isc_rx_size_64); 6242 evcnt_detach(&isc->isc_rx_size_127); 6243 evcnt_detach(&isc->isc_rx_size_255); 6244 evcnt_detach(&isc->isc_rx_size_511); 6245 evcnt_detach(&isc->isc_rx_size_1023); 6246 evcnt_detach(&isc->isc_rx_size_1522); 6247 evcnt_detach(&isc->isc_rx_size_big); 6248 evcnt_detach(&isc->isc_rx_undersize); 6249 evcnt_detach(&isc->isc_rx_oversize); 6250 evcnt_detach(&isc->isc_tx_bytes); 6251 evcnt_detach(&isc->isc_tx_dropped_link_down); 6252 evcnt_detach(&isc->isc_tx_unicast); 6253 evcnt_detach(&isc->isc_tx_multicast); 6254 evcnt_detach(&isc->isc_tx_broadcast); 6255 evcnt_detach(&isc->isc_tx_size_64); 6256 evcnt_detach(&isc->isc_tx_size_127); 6257 evcnt_detach(&isc->isc_tx_size_255); 6258 evcnt_detach(&isc->isc_tx_size_511); 6259 evcnt_detach(&isc->isc_tx_size_1023); 6260 evcnt_detach(&isc->isc_tx_size_1522); 6261 evcnt_detach(&isc->isc_tx_size_big); 6262 evcnt_detach(&isc->isc_vsi_rx_discards); 6263 evcnt_detach(&isc->isc_vsi_rx_bytes); 6264 evcnt_detach(&isc->isc_vsi_rx_unicast); 6265 evcnt_detach(&isc->isc_vsi_rx_multicast); 6266 evcnt_detach(&isc->isc_vsi_rx_broadcast); 6267 evcnt_detach(&isc->isc_vsi_tx_errors); 6268 evcnt_detach(&isc->isc_vsi_tx_bytes); 6269 evcnt_detach(&isc->isc_vsi_tx_unicast); 6270 evcnt_detach(&isc->isc_vsi_tx_multicast); 6271 evcnt_detach(&isc->isc_vsi_tx_broadcast); 6272 6273 evcnt_detach(&sc->sc_event_atq); 6274 evcnt_detach(&sc->sc_event_link); 6275 evcnt_detach(&sc->sc_event_ecc_err); 6276 evcnt_detach(&sc->sc_event_pci_exception); 6277 evcnt_detach(&sc->sc_event_crit_err); 6278 6279 callout_destroy(&sc->sc_stats_callout); 6280} 6281 6282static void 6283ixl_stats_callout(void *xsc) 6284{ 6285 struct ixl_softc *sc = xsc; 6286 6287 ixl_work_add(sc->sc_workq, &sc->sc_stats_task); 6288 callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval)); 6289} 6290 6291static uint64_t 6292ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo, 6293 uint64_t *offset, bool has_offset) 6294{ 6295 uint64_t value, delta; 6296 int bitwidth; 6297 6298 bitwidth = reg_hi == 0 ? 32 : 48; 6299 6300 value = ixl_rd(sc, reg_lo); 6301 6302 if (bitwidth > 32) { 6303 value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32); 6304 } 6305 6306 if (__predict_true(has_offset)) { 6307 delta = value; 6308 if (value < *offset) 6309 delta += ((uint64_t)1 << bitwidth); 6310 delta -= *offset; 6311 } else { 6312 delta = 0; 6313 } 6314 atomic_swap_64(offset, value); 6315 6316 return delta; 6317} 6318 6319static void 6320ixl_stats_update(void *xsc) 6321{ 6322 struct ixl_softc *sc = xsc; 6323 struct ixl_stats_counters *isc; 6324 uint64_t delta; 6325 6326 isc = &sc->sc_stats_counters; 6327 6328 /* errors */ 6329 delta = ixl_stat_delta(sc, 6330 0, I40E_GLPRT_CRCERRS(sc->sc_port), 6331 &isc->isc_crc_errors_offset, isc->isc_has_offset); 6332 atomic_add_64(&isc->isc_crc_errors.ev_count, delta); 6333 6334 delta = ixl_stat_delta(sc, 6335 0, I40E_GLPRT_ILLERRC(sc->sc_port), 6336 &isc->isc_illegal_bytes_offset, isc->isc_has_offset); 6337 atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta); 6338 6339 /* rx */ 6340 delta = ixl_stat_delta(sc, 6341 I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port), 6342 &isc->isc_rx_bytes_offset, isc->isc_has_offset); 6343 atomic_add_64(&isc->isc_rx_bytes.ev_count, delta); 6344 6345 delta = ixl_stat_delta(sc, 6346 0, I40E_GLPRT_RDPC(sc->sc_port), 6347 &isc->isc_rx_discards_offset, isc->isc_has_offset); 6348 atomic_add_64(&isc->isc_rx_discards.ev_count, delta); 6349 6350 delta = ixl_stat_delta(sc, 6351 I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port), 6352 &isc->isc_rx_unicast_offset, isc->isc_has_offset); 6353 atomic_add_64(&isc->isc_rx_unicast.ev_count, delta); 6354 6355 delta = ixl_stat_delta(sc, 6356 I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port), 6357 &isc->isc_rx_multicast_offset, isc->isc_has_offset); 6358 atomic_add_64(&isc->isc_rx_multicast.ev_count, delta); 6359 6360 delta = ixl_stat_delta(sc, 6361 I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port), 6362 &isc->isc_rx_broadcast_offset, isc->isc_has_offset); 6363 atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta); 6364 6365 /* Packet size stats rx */ 6366 delta = ixl_stat_delta(sc, 6367 I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port), 6368 &isc->isc_rx_size_64_offset, isc->isc_has_offset); 6369 atomic_add_64(&isc->isc_rx_size_64.ev_count, delta); 6370 6371 delta = ixl_stat_delta(sc, 6372 I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port), 6373 &isc->isc_rx_size_127_offset, isc->isc_has_offset); 6374 atomic_add_64(&isc->isc_rx_size_127.ev_count, delta); 6375 6376 delta = ixl_stat_delta(sc, 6377 I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port), 6378 &isc->isc_rx_size_255_offset, isc->isc_has_offset); 6379 atomic_add_64(&isc->isc_rx_size_255.ev_count, delta); 6380 6381 delta = ixl_stat_delta(sc, 6382 I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port), 6383 &isc->isc_rx_size_511_offset, isc->isc_has_offset); 6384 atomic_add_64(&isc->isc_rx_size_511.ev_count, delta); 6385 6386 delta = ixl_stat_delta(sc, 6387 I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port), 6388 &isc->isc_rx_size_1023_offset, isc->isc_has_offset); 6389 atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta); 6390 6391 delta = ixl_stat_delta(sc, 6392 I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port), 6393 &isc->isc_rx_size_1522_offset, isc->isc_has_offset); 6394 atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta); 6395 6396 delta = ixl_stat_delta(sc, 6397 I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port), 6398 &isc->isc_rx_size_big_offset, isc->isc_has_offset); 6399 atomic_add_64(&isc->isc_rx_size_big.ev_count, delta); 6400 6401 delta = ixl_stat_delta(sc, 6402 0, I40E_GLPRT_RUC(sc->sc_port), 6403 &isc->isc_rx_undersize_offset, isc->isc_has_offset); 6404 atomic_add_64(&isc->isc_rx_undersize.ev_count, delta); 6405 6406 delta = ixl_stat_delta(sc, 6407 0, I40E_GLPRT_ROC(sc->sc_port), 6408 &isc->isc_rx_oversize_offset, isc->isc_has_offset); 6409 atomic_add_64(&isc->isc_rx_oversize.ev_count, delta); 6410 6411 /* tx */ 6412 delta = ixl_stat_delta(sc, 6413 I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port), 6414 &isc->isc_tx_bytes_offset, isc->isc_has_offset); 6415 atomic_add_64(&isc->isc_tx_bytes.ev_count, delta); 6416 6417 delta = ixl_stat_delta(sc, 6418 0, I40E_GLPRT_TDOLD(sc->sc_port), 6419 &isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset); 6420 atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta); 6421 6422 delta = ixl_stat_delta(sc, 6423 I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port), 6424 &isc->isc_tx_unicast_offset, isc->isc_has_offset); 6425 atomic_add_64(&isc->isc_tx_unicast.ev_count, delta); 6426 6427 delta = ixl_stat_delta(sc, 6428 I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port), 6429 &isc->isc_tx_multicast_offset, isc->isc_has_offset); 6430 atomic_add_64(&isc->isc_tx_multicast.ev_count, delta); 6431 6432 delta = ixl_stat_delta(sc, 6433 I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port), 6434 &isc->isc_tx_broadcast_offset, isc->isc_has_offset); 6435 atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta); 6436 6437 /* Packet size stats tx */ 6438 delta = ixl_stat_delta(sc, 6439 I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port), 6440 &isc->isc_tx_size_64_offset, isc->isc_has_offset); 6441 atomic_add_64(&isc->isc_tx_size_64.ev_count, delta); 6442 6443 delta = ixl_stat_delta(sc, 6444 I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port), 6445 &isc->isc_tx_size_127_offset, isc->isc_has_offset); 6446 atomic_add_64(&isc->isc_tx_size_127.ev_count, delta); 6447 6448 delta = ixl_stat_delta(sc, 6449 I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port), 6450 &isc->isc_tx_size_255_offset, isc->isc_has_offset); 6451 atomic_add_64(&isc->isc_tx_size_255.ev_count, delta); 6452 6453 delta = ixl_stat_delta(sc, 6454 I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port), 6455 &isc->isc_tx_size_511_offset, isc->isc_has_offset); 6456 atomic_add_64(&isc->isc_tx_size_511.ev_count, delta); 6457 6458 delta = ixl_stat_delta(sc, 6459 I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port), 6460 &isc->isc_tx_size_1023_offset, isc->isc_has_offset); 6461 atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta); 6462 6463 delta = ixl_stat_delta(sc, 6464 I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port), 6465 &isc->isc_tx_size_1522_offset, isc->isc_has_offset); 6466 atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta); 6467 6468 delta = ixl_stat_delta(sc, 6469 I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port), 6470 &isc->isc_tx_size_big_offset, isc->isc_has_offset); 6471 atomic_add_64(&isc->isc_tx_size_big.ev_count, delta); 6472 6473 /* mac faults */ 6474 delta = ixl_stat_delta(sc, 6475 0, I40E_GLPRT_MLFC(sc->sc_port), 6476 &isc->isc_mac_local_faults_offset, isc->isc_has_offset); 6477 atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta); 6478 6479 delta = ixl_stat_delta(sc, 6480 0, I40E_GLPRT_MRFC(sc->sc_port), 6481 &isc->isc_mac_remote_faults_offset, isc->isc_has_offset); 6482 atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta); 6483 6484 /* Flow control (LFC) stats */ 6485 delta = ixl_stat_delta(sc, 6486 0, I40E_GLPRT_LXONRXC(sc->sc_port), 6487 &isc->isc_link_xon_rx_offset, isc->isc_has_offset); 6488 atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta); 6489 6490 delta = ixl_stat_delta(sc, 6491 0, I40E_GLPRT_LXONTXC(sc->sc_port), 6492 &isc->isc_link_xon_tx_offset, isc->isc_has_offset); 6493 atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta); 6494 6495 delta = ixl_stat_delta(sc, 6496 0, I40E_GLPRT_LXOFFRXC(sc->sc_port), 6497 &isc->isc_link_xoff_rx_offset, isc->isc_has_offset); 6498 atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta); 6499 6500 delta = ixl_stat_delta(sc, 6501 0, I40E_GLPRT_LXOFFTXC(sc->sc_port), 6502 &isc->isc_link_xoff_tx_offset, isc->isc_has_offset); 6503 atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta); 6504 6505 /* fragments */ 6506 delta = ixl_stat_delta(sc, 6507 0, I40E_GLPRT_RFC(sc->sc_port), 6508 &isc->isc_rx_fragments_offset, isc->isc_has_offset); 6509 atomic_add_64(&isc->isc_rx_fragments.ev_count, delta); 6510 6511 delta = ixl_stat_delta(sc, 6512 0, I40E_GLPRT_RJC(sc->sc_port), 6513 &isc->isc_rx_jabber_offset, isc->isc_has_offset); 6514 atomic_add_64(&isc->isc_rx_jabber.ev_count, delta); 6515 6516 /* VSI rx counters */ 6517 delta = ixl_stat_delta(sc, 6518 0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx), 6519 &isc->isc_vsi_rx_discards_offset, isc->isc_has_offset); 6520 atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta); 6521 6522 delta = ixl_stat_delta(sc, 6523 I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx), 6524 I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx), 6525 &isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset); 6526 atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta); 6527 6528 delta = ixl_stat_delta(sc, 6529 I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx), 6530 I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx), 6531 &isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset); 6532 atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta); 6533 6534 delta = ixl_stat_delta(sc, 6535 I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx), 6536 I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx), 6537 &isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset); 6538 atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta); 6539 6540 delta = ixl_stat_delta(sc, 6541 I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx), 6542 I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx), 6543 &isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset); 6544 atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta); 6545 6546 /* VSI tx counters */ 6547 delta = ixl_stat_delta(sc, 6548 0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx), 6549 &isc->isc_vsi_tx_errors_offset, isc->isc_has_offset); 6550 atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta); 6551 6552 delta = ixl_stat_delta(sc, 6553 I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx), 6554 I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx), 6555 &isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset); 6556 atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta); 6557 6558 delta = ixl_stat_delta(sc, 6559 I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx), 6560 I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx), 6561 &isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset); 6562 atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta); 6563 6564 delta = ixl_stat_delta(sc, 6565 I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx), 6566 I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx), 6567 &isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset); 6568 atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta); 6569 6570 delta = ixl_stat_delta(sc, 6571 I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx), 6572 I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx), 6573 &isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset); 6574 atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta); 6575} 6576 6577static int 6578ixl_setup_sysctls(struct ixl_softc *sc) 6579{ 6580 const char *devname; 6581 struct sysctllog **log; 6582 const struct sysctlnode *rnode, *rxnode, *txnode; 6583 int error; 6584 6585 log = &sc->sc_sysctllog; 6586 devname = device_xname(sc->sc_dev); 6587 6588 error = sysctl_createv(log, 0, NULL, &rnode, 6589 0, CTLTYPE_NODE, devname, 6590 SYSCTL_DESCR("ixl information and settings"), 6591 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 6592 if (error) 6593 goto out; 6594 6595 error = sysctl_createv(log, 0, &rnode, NULL, 6596 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 6597 SYSCTL_DESCR("Use workqueue for packet processing"), 6598 NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL); 6599 if (error) 6600 goto out; 6601 6602 error = sysctl_createv(log, 0, &rnode, NULL, 6603 CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval", 6604 SYSCTL_DESCR("Statistics collection interval in milliseconds"), 6605 NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL); 6606 6607 error = sysctl_createv(log, 0, &rnode, &rxnode, 6608 0, CTLTYPE_NODE, "rx", 6609 SYSCTL_DESCR("ixl information and settings for Rx"), 6610 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6611 if (error) 6612 goto out; 6613 6614 error = sysctl_createv(log, 0, &rxnode, NULL, 6615 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6616 SYSCTL_DESCR("Interrupt Throttling"), 6617 ixl_sysctl_itr_handler, 0, 6618 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6619 if (error) 6620 goto out; 6621 6622 error = sysctl_createv(log, 0, &rxnode, NULL, 6623 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6624 SYSCTL_DESCR("the number of rx descriptors"), 6625 NULL, 0, &sc->sc_rx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6626 if (error) 6627 goto out; 6628 6629 error = sysctl_createv(log, 0, &rxnode, NULL, 6630 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6631 SYSCTL_DESCR("max number of Rx packets" 6632 " to process for interrupt processing"), 6633 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6634 if (error) 6635 goto out; 6636 6637 error = sysctl_createv(log, 0, &rxnode, NULL, 6638 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6639 SYSCTL_DESCR("max number of Rx packets" 6640 " to process for deferred processing"), 6641 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 6642 if (error) 6643 goto out; 6644 6645 error = sysctl_createv(log, 0, &rnode, &txnode, 6646 0, CTLTYPE_NODE, "tx", 6647 SYSCTL_DESCR("ixl information and settings for Tx"), 6648 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 6649 if (error) 6650 goto out; 6651 6652 error = sysctl_createv(log, 0, &txnode, NULL, 6653 CTLFLAG_READWRITE, CTLTYPE_INT, "itr", 6654 SYSCTL_DESCR("Interrupt Throttling"), 6655 ixl_sysctl_itr_handler, 0, 6656 (void *)sc, 0, CTL_CREATE, CTL_EOL); 6657 if (error) 6658 goto out; 6659 6660 error = sysctl_createv(log, 0, &txnode, NULL, 6661 CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num", 6662 SYSCTL_DESCR("the number of tx descriptors"), 6663 NULL, 0, &sc->sc_tx_ring_ndescs, 0, CTL_CREATE, CTL_EOL); 6664 if (error) 6665 goto out; 6666 6667 error = sysctl_createv(log, 0, &txnode, NULL, 6668 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 6669 SYSCTL_DESCR("max number of Tx packets" 6670 " to process for interrupt processing"), 6671 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 6672 if (error) 6673 goto out; 6674 6675 error = sysctl_createv(log, 0, &txnode, NULL, 6676 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 6677 SYSCTL_DESCR("max number of Tx packets" 6678 " to process for deferred processing"), 6679 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 6680 if (error) 6681 goto out; 6682 6683out: 6684 if (error) { 6685 aprint_error_dev(sc->sc_dev, 6686 "unable to create sysctl node\n"); 6687 sysctl_teardown(log); 6688 } 6689 6690 return error; 6691} 6692 6693static void 6694ixl_teardown_sysctls(struct ixl_softc *sc) 6695{ 6696 6697 sysctl_teardown(&sc->sc_sysctllog); 6698} 6699 6700static bool 6701ixl_sysctlnode_is_rx(struct sysctlnode *node) 6702{ 6703 6704 if (strstr(node->sysctl_parent->sysctl_name, "rx") != NULL) 6705 return true; 6706 6707 return false; 6708} 6709 6710static int 6711ixl_sysctl_itr_handler(SYSCTLFN_ARGS) 6712{ 6713 struct sysctlnode node = *rnode; 6714 struct ixl_softc *sc = (struct ixl_softc *)node.sysctl_data; 6715 struct ifnet *ifp = &sc->sc_ec.ec_if; 6716 uint32_t newitr, *itrptr; 6717 int error; 6718 6719 if (ixl_sysctlnode_is_rx(&node)) { 6720 itrptr = &sc->sc_itr_rx; 6721 } else { 6722 itrptr = &sc->sc_itr_tx; 6723 } 6724 6725 newitr = *itrptr; 6726 node.sysctl_data = &newitr; 6727 node.sysctl_size = sizeof(newitr); 6728 6729 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6730 6731 if (error || newp == NULL) 6732 return error; 6733 6734 /* ITRs are applied in ixl_init() for simple implementation */ 6735 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6736 return EBUSY; 6737 6738 if (newitr > 0x07ff) 6739 return EINVAL; 6740 6741 *itrptr = newitr; 6742 6743 return 0; 6744} 6745 6746static struct workqueue * 6747ixl_workq_create(const char *name, pri_t prio, int ipl, int flags) 6748{ 6749 struct workqueue *wq; 6750 int error; 6751 6752 error = workqueue_create(&wq, name, ixl_workq_work, NULL, 6753 prio, ipl, flags); 6754 6755 if (error) 6756 return NULL; 6757 6758 return wq; 6759} 6760 6761static void 6762ixl_workq_destroy(struct workqueue *wq) 6763{ 6764 6765 workqueue_destroy(wq); 6766} 6767 6768static void 6769ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg) 6770{ 6771 6772 memset(work, 0, sizeof(*work)); 6773 work->ixw_func = func; 6774 work->ixw_arg = arg; 6775} 6776 6777static void 6778ixl_work_add(struct workqueue *wq, struct ixl_work *work) 6779{ 6780 if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0) 6781 return; 6782 6783 kpreempt_disable(); 6784 workqueue_enqueue(wq, &work->ixw_cookie, NULL); 6785 kpreempt_enable(); 6786} 6787 6788static void 6789ixl_work_wait(struct workqueue *wq, struct ixl_work *work) 6790{ 6791 6792 workqueue_wait(wq, &work->ixw_cookie); 6793} 6794 6795static void 6796ixl_workq_work(struct work *wk, void *context) 6797{ 6798 struct ixl_work *work; 6799 6800 work = container_of(wk, struct ixl_work, ixw_cookie); 6801 6802 atomic_swap_uint(&work->ixw_added, 0); 6803 work->ixw_func(work->ixw_arg); 6804} 6805 6806static int 6807ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv) 6808{ 6809 struct ixl_aq_desc iaq; 6810 6811 memset(&iaq, 0, sizeof(iaq)); 6812 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ); 6813 iaq.iaq_param[1] = htole32(reg); 6814 6815 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6816 return ETIMEDOUT; 6817 6818 switch (htole16(iaq.iaq_retval)) { 6819 case IXL_AQ_RC_OK: 6820 /* success */ 6821 break; 6822 case IXL_AQ_RC_EACCES: 6823 return EPERM; 6824 case IXL_AQ_RC_EAGAIN: 6825 return EAGAIN; 6826 default: 6827 return EIO; 6828 } 6829 6830 *rv = htole32(iaq.iaq_param[3]); 6831 return 0; 6832} 6833 6834static uint32_t 6835ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg) 6836{ 6837 uint32_t val; 6838 int rv, retry, retry_limit; 6839 6840 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6841 retry_limit = 5; 6842 } else { 6843 retry_limit = 0; 6844 } 6845 6846 for (retry = 0; retry < retry_limit; retry++) { 6847 rv = ixl_rx_ctl_read(sc, reg, &val); 6848 if (rv == 0) 6849 return val; 6850 else if (rv == EAGAIN) 6851 delaymsec(1); 6852 else 6853 break; 6854 } 6855 6856 val = ixl_rd(sc, reg); 6857 6858 return val; 6859} 6860 6861static int 6862ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6863{ 6864 struct ixl_aq_desc iaq; 6865 6866 memset(&iaq, 0, sizeof(iaq)); 6867 iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE); 6868 iaq.iaq_param[1] = htole32(reg); 6869 iaq.iaq_param[3] = htole32(value); 6870 6871 if (ixl_atq_poll(sc, &iaq, 250) != 0) 6872 return ETIMEDOUT; 6873 6874 switch (htole16(iaq.iaq_retval)) { 6875 case IXL_AQ_RC_OK: 6876 /* success */ 6877 break; 6878 case IXL_AQ_RC_EACCES: 6879 return EPERM; 6880 case IXL_AQ_RC_EAGAIN: 6881 return EAGAIN; 6882 default: 6883 return EIO; 6884 } 6885 6886 return 0; 6887} 6888 6889static void 6890ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value) 6891{ 6892 int rv, retry, retry_limit; 6893 6894 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) { 6895 retry_limit = 5; 6896 } else { 6897 retry_limit = 0; 6898 } 6899 6900 for (retry = 0; retry < retry_limit; retry++) { 6901 rv = ixl_rx_ctl_write(sc, reg, value); 6902 if (rv == 0) 6903 return; 6904 else if (rv == EAGAIN) 6905 delaymsec(1); 6906 else 6907 break; 6908 } 6909 6910 ixl_wr(sc, reg, value); 6911} 6912 6913static int 6914ixl_nvm_lock(struct ixl_softc *sc, char rw) 6915{ 6916 struct ixl_aq_desc iaq; 6917 struct ixl_aq_req_resource_param *param; 6918 int rv; 6919 6920 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6921 return 0; 6922 6923 memset(&iaq, 0, sizeof(iaq)); 6924 iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE); 6925 6926 param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param; 6927 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6928 if (rw == 'R') { 6929 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ); 6930 } else { 6931 param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE); 6932 } 6933 6934 rv = ixl_atq_poll(sc, &iaq, 250); 6935 6936 if (rv != 0) 6937 return ETIMEDOUT; 6938 6939 switch (le16toh(iaq.iaq_retval)) { 6940 case IXL_AQ_RC_OK: 6941 break; 6942 case IXL_AQ_RC_EACCES: 6943 return EACCES; 6944 case IXL_AQ_RC_EBUSY: 6945 return EBUSY; 6946 case IXL_AQ_RC_EPERM: 6947 return EPERM; 6948 } 6949 6950 return 0; 6951} 6952 6953static int 6954ixl_nvm_unlock(struct ixl_softc *sc) 6955{ 6956 struct ixl_aq_desc iaq; 6957 struct ixl_aq_rel_resource_param *param; 6958 int rv; 6959 6960 if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK)) 6961 return 0; 6962 6963 memset(&iaq, 0, sizeof(iaq)); 6964 iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE); 6965 6966 param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param; 6967 param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM); 6968 6969 rv = ixl_atq_poll(sc, &iaq, 250); 6970 6971 if (rv != 0) 6972 return ETIMEDOUT; 6973 6974 switch (le16toh(iaq.iaq_retval)) { 6975 case IXL_AQ_RC_OK: 6976 break; 6977 default: 6978 return EIO; 6979 } 6980 return 0; 6981} 6982 6983static int 6984ixl_srdone_poll(struct ixl_softc *sc) 6985{ 6986 int wait_count; 6987 uint32_t reg; 6988 6989 for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS; 6990 wait_count++) { 6991 reg = ixl_rd(sc, I40E_GLNVM_SRCTL); 6992 if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK)) 6993 break; 6994 6995 delaymsec(5); 6996 } 6997 6998 if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS) 6999 return -1; 7000 7001 return 0; 7002} 7003 7004static int 7005ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7006{ 7007 uint32_t reg; 7008 7009 if (ixl_srdone_poll(sc) != 0) 7010 return ETIMEDOUT; 7011 7012 reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) | 7013 __BIT(I40E_GLNVM_SRCTL_START_SHIFT); 7014 ixl_wr(sc, I40E_GLNVM_SRCTL, reg); 7015 7016 if (ixl_srdone_poll(sc) != 0) { 7017 aprint_debug("NVM read error: couldn't access " 7018 "Shadow RAM address: 0x%x\n", offset); 7019 return ETIMEDOUT; 7020 } 7021 7022 reg = ixl_rd(sc, I40E_GLNVM_SRDATA); 7023 *data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK); 7024 7025 return 0; 7026} 7027 7028static int 7029ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word, 7030 void *data, size_t len) 7031{ 7032 struct ixl_dmamem *idm; 7033 struct ixl_aq_desc iaq; 7034 struct ixl_aq_nvm_param *param; 7035 uint32_t offset_bytes; 7036 int rv; 7037 7038 idm = &sc->sc_aqbuf; 7039 if (len > IXL_DMA_LEN(idm)) 7040 return ENOMEM; 7041 7042 memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm)); 7043 memset(&iaq, 0, sizeof(iaq)); 7044 iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ); 7045 iaq.iaq_flags = htole16(IXL_AQ_BUF | 7046 ((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0)); 7047 iaq.iaq_datalen = htole16(len); 7048 ixl_aq_dva(&iaq, IXL_DMA_DVA(idm)); 7049 7050 param = (struct ixl_aq_nvm_param *)iaq.iaq_param; 7051 param->command_flags = IXL_AQ_NVM_LAST_CMD; 7052 param->module_pointer = 0; 7053 param->length = htole16(len); 7054 offset_bytes = (uint32_t)offset_word * 2; 7055 offset_bytes &= 0x00FFFFFF; 7056 param->offset = htole32(offset_bytes); 7057 7058 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7059 BUS_DMASYNC_PREREAD); 7060 7061 rv = ixl_atq_poll(sc, &iaq, 250); 7062 7063 bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm), 7064 BUS_DMASYNC_POSTREAD); 7065 7066 if (rv != 0) { 7067 return ETIMEDOUT; 7068 } 7069 7070 switch (le16toh(iaq.iaq_retval)) { 7071 case IXL_AQ_RC_OK: 7072 break; 7073 case IXL_AQ_RC_EPERM: 7074 return EPERM; 7075 case IXL_AQ_RC_EINVAL: 7076 return EINVAL; 7077 case IXL_AQ_RC_EBUSY: 7078 return EBUSY; 7079 case IXL_AQ_RC_EIO: 7080 default: 7081 return EIO; 7082 } 7083 7084 memcpy(data, IXL_DMA_KVA(idm), len); 7085 7086 return 0; 7087} 7088 7089static int 7090ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data) 7091{ 7092 int error; 7093 uint16_t buf; 7094 7095 error = ixl_nvm_lock(sc, 'R'); 7096 if (error) 7097 return error; 7098 7099 if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) { 7100 error = ixl_nvm_read_aq(sc, offset, 7101 &buf, sizeof(buf)); 7102 if (error == 0) 7103 *data = le16toh(buf); 7104 } else { 7105 error = ixl_nvm_read_srctl(sc, offset, &buf); 7106 if (error == 0) 7107 *data = buf; 7108 } 7109 7110 ixl_nvm_unlock(sc); 7111 7112 return error; 7113} 7114 7115MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci"); 7116 7117#ifdef _MODULE 7118#include "ioconf.c" 7119#endif 7120 7121#ifdef _MODULE 7122static void 7123ixl_parse_modprop(prop_dictionary_t dict) 7124{ 7125 prop_object_t obj; 7126 int64_t val; 7127 uint64_t uval; 7128 7129 if (dict == NULL) 7130 return; 7131 7132 obj = prop_dictionary_get(dict, "nomsix"); 7133 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) { 7134 ixl_param_nomsix = prop_bool_true((prop_bool_t)obj); 7135 } 7136 7137 obj = prop_dictionary_get(dict, "stats_interval"); 7138 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7139 val = prop_number_signed_value((prop_number_t)obj); 7140 7141 /* the range has no reason */ 7142 if (100 < val && val < 180000) { 7143 ixl_param_stats_interval = val; 7144 } 7145 } 7146 7147 obj = prop_dictionary_get(dict, "nqps_limit"); 7148 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7149 val = prop_number_signed_value((prop_number_t)obj); 7150 7151 if (val <= INT32_MAX) 7152 ixl_param_nqps_limit = val; 7153 } 7154 7155 obj = prop_dictionary_get(dict, "rx_ndescs"); 7156 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7157 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7158 7159 if (uval > 8) 7160 ixl_param_rx_ndescs = uval; 7161 } 7162 7163 obj = prop_dictionary_get(dict, "tx_ndescs"); 7164 if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) { 7165 uval = prop_number_unsigned_integer_value((prop_number_t)obj); 7166 7167 if (uval > IXL_TX_PKT_DESCS) 7168 ixl_param_tx_ndescs = uval; 7169 } 7170 7171} 7172#endif 7173 7174static int 7175if_ixl_modcmd(modcmd_t cmd, void *opaque) 7176{ 7177 int error = 0; 7178 7179#ifdef _MODULE 7180 switch (cmd) { 7181 case MODULE_CMD_INIT: 7182 ixl_parse_modprop((prop_dictionary_t)opaque); 7183 error = config_init_component(cfdriver_ioconf_if_ixl, 7184 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7185 break; 7186 case MODULE_CMD_FINI: 7187 error = config_fini_component(cfdriver_ioconf_if_ixl, 7188 cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl); 7189 break; 7190 default: 7191 error = ENOTTY; 7192 break; 7193 } 7194#endif 7195 7196 return error; 7197} 7198