1325618Ssbruno/* 2325618Ssbruno * BSD LICENSE 3325618Ssbruno * 4325618Ssbruno * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5325618Ssbruno * All rights reserved. 6325618Ssbruno * 7325618Ssbruno * Redistribution and use in source and binary forms, with or without 8325618Ssbruno * modification, are permitted provided that the following conditions 9325618Ssbruno * are met: 10325618Ssbruno * 11325618Ssbruno * * Redistributions of source code must retain the above copyright 12325618Ssbruno * notice, this list of conditions and the following disclaimer. 13325618Ssbruno * * Redistributions in binary form must reproduce the above copyright 14325618Ssbruno * notice, this list of conditions and the following disclaimer in 15325618Ssbruno * the documentation and/or other materials provided with the 16325618Ssbruno * distribution. 17325618Ssbruno * * Neither the name of Cavium, Inc. nor the names of its 18325618Ssbruno * contributors may be used to endorse or promote products derived 19325618Ssbruno * from this software without specific prior written permission. 20325618Ssbruno * 21325618Ssbruno * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22325618Ssbruno * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23325618Ssbruno * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24325618Ssbruno * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25325618Ssbruno * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26325618Ssbruno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27325618Ssbruno * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28325618Ssbruno * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29325618Ssbruno * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30325618Ssbruno * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31325618Ssbruno * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32325618Ssbruno */ 33325618Ssbruno/*$FreeBSD: stable/11/sys/dev/liquidio/lio_main.c 335293 2018-06-17 17:38:24Z dim $*/ 34325618Ssbruno 35325618Ssbruno#include "lio_bsd.h" 36325618Ssbruno#include "lio_common.h" 37325618Ssbruno 38325618Ssbruno#include "lio_droq.h" 39325618Ssbruno#include "lio_iq.h" 40325618Ssbruno#include "lio_response_manager.h" 41325618Ssbruno#include "lio_device.h" 42325618Ssbruno#include "lio_ctrl.h" 43325618Ssbruno#include "lio_main.h" 44325618Ssbruno#include "lio_network.h" 45325618Ssbruno#include "cn23xx_pf_device.h" 46325618Ssbruno#include "lio_image.h" 47325618Ssbruno#include "lio_ioctl.h" 48325618Ssbruno#include "lio_rxtx.h" 49325618Ssbruno#include "lio_rss.h" 50325618Ssbruno 51325618Ssbruno/* Number of milliseconds to wait for DDR initialization */ 52325618Ssbruno#define LIO_DDR_TIMEOUT 10000 53325618Ssbruno#define LIO_MAX_FW_TYPE_LEN 8 54325618Ssbruno 55325618Ssbrunostatic char fw_type[LIO_MAX_FW_TYPE_LEN]; 56325618SsbrunoTUNABLE_STR("hw.lio.fw_type", fw_type, sizeof(fw_type)); 57325618Ssbruno 58325618Ssbruno/* 59325618Ssbruno * Integers that specify number of queues per PF. 60325618Ssbruno * Valid range is 0 to 64. 61325618Ssbruno * Use 0 to derive from CPU count. 62325618Ssbruno */ 63325618Ssbrunostatic int num_queues_per_pf0; 64325618Ssbrunostatic int num_queues_per_pf1; 65325618SsbrunoTUNABLE_INT("hw.lio.num_queues_per_pf0", &num_queues_per_pf0); 66325618SsbrunoTUNABLE_INT("hw.lio.num_queues_per_pf1", &num_queues_per_pf1); 67325618Ssbruno 68325618Ssbruno#ifdef RSS 69325618Ssbrunostatic int lio_rss = 1; 70325618SsbrunoTUNABLE_INT("hw.lio.rss", &lio_rss); 71325618Ssbruno#endif /* RSS */ 72325618Ssbruno 73325618Ssbruno/* Hardware LRO */ 74325618Ssbrunounsigned int lio_hwlro = 0; 75325618SsbrunoTUNABLE_INT("hw.lio.hwlro", &lio_hwlro); 76325618Ssbruno 77325618Ssbruno/* 78325618Ssbruno * Bitmask indicating which consoles have debug 79325618Ssbruno * output redirected to syslog. 80325618Ssbruno */ 81325618Ssbrunostatic unsigned long console_bitmask; 82325618SsbrunoTUNABLE_ULONG("hw.lio.console_bitmask", &console_bitmask); 83325618Ssbruno 84325618Ssbruno/* 85325618Ssbruno * \brief determines if a given console has debug enabled. 86325618Ssbruno * @param console console to check 87325618Ssbruno * @returns 1 = enabled. 0 otherwise 88325618Ssbruno */ 89325618Ssbrunoint 90325618Ssbrunolio_console_debug_enabled(uint32_t console) 91325618Ssbruno{ 92325618Ssbruno 93325618Ssbruno return (console_bitmask >> (console)) & 0x1; 94325618Ssbruno} 95325618Ssbruno 96325618Ssbrunostatic int lio_detach(device_t dev); 97325618Ssbruno 98325618Ssbrunostatic int lio_device_init(struct octeon_device *octeon_dev); 99325618Ssbrunostatic int lio_chip_specific_setup(struct octeon_device *oct); 100325618Ssbrunostatic void lio_watchdog(void *param); 101325618Ssbrunostatic int lio_load_firmware(struct octeon_device *oct); 102325618Ssbrunostatic int lio_nic_starter(struct octeon_device *oct); 103325618Ssbrunostatic int lio_init_nic_module(struct octeon_device *oct); 104325618Ssbrunostatic int lio_setup_nic_devices(struct octeon_device *octeon_dev); 105325618Ssbrunostatic int lio_link_info(struct lio_recv_info *recv_info, void *ptr); 106325618Ssbrunostatic void lio_if_cfg_callback(struct octeon_device *oct, uint32_t status, 107325618Ssbruno void *buf); 108325618Ssbrunostatic int lio_set_rxcsum_command(struct ifnet *ifp, int command, 109325618Ssbruno uint8_t rx_cmd); 110325618Ssbrunostatic int lio_setup_glists(struct octeon_device *oct, struct lio *lio, 111325618Ssbruno int num_iqs); 112325618Ssbrunostatic void lio_destroy_nic_device(struct octeon_device *oct, int ifidx); 113325618Ssbrunostatic inline void lio_update_link_status(struct ifnet *ifp, 114325618Ssbruno union octeon_link_status *ls); 115325618Ssbrunostatic void lio_send_rx_ctrl_cmd(struct lio *lio, int start_stop); 116325618Ssbrunostatic int lio_stop_nic_module(struct octeon_device *oct); 117325618Ssbrunostatic void lio_destroy_resources(struct octeon_device *oct); 118325618Ssbrunostatic int lio_setup_rx_oom_poll_fn(struct ifnet *ifp); 119325618Ssbruno 120325618Ssbrunostatic void lio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid); 121325618Ssbrunostatic void lio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, 122325618Ssbruno uint16_t vid); 123325618Ssbrunostatic struct octeon_device * 124325618Ssbruno lio_get_other_octeon_device(struct octeon_device *oct); 125325618Ssbruno 126325618Ssbrunostatic int lio_wait_for_oq_pkts(struct octeon_device *oct); 127325618Ssbruno 128325618Ssbrunoint lio_send_rss_param(struct lio *lio); 129325618Ssbrunostatic int lio_dbg_console_print(struct octeon_device *oct, 130325618Ssbruno uint32_t console_num, char *prefix, 131325618Ssbruno char *suffix); 132325618Ssbruno 133325618Ssbruno/* Polling interval for determining when NIC application is alive */ 134325618Ssbruno#define LIO_STARTER_POLL_INTERVAL_MS 100 135325618Ssbruno 136325618Ssbruno/* 137325618Ssbruno * vendor_info_array. 138325618Ssbruno * This array contains the list of IDs on which the driver should load. 139325618Ssbruno */ 140325618Ssbrunostruct lio_vendor_info { 141325618Ssbruno uint16_t vendor_id; 142325618Ssbruno uint16_t device_id; 143325618Ssbruno uint16_t subdevice_id; 144325618Ssbruno uint8_t revision_id; 145325618Ssbruno uint8_t index; 146325618Ssbruno}; 147325618Ssbruno 148325618Ssbrunostatic struct lio_vendor_info lio_pci_tbl[] = { 149325618Ssbruno /* CN2350 10G */ 150325618Ssbruno {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE, 151325618Ssbruno 0x02, 0}, 152325618Ssbruno 153325618Ssbruno /* CN2350 10G */ 154325618Ssbruno {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE1, 155325618Ssbruno 0x02, 0}, 156325618Ssbruno 157325618Ssbruno /* CN2360 10G */ 158325618Ssbruno {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_10G_SUBDEVICE, 159325618Ssbruno 0x02, 1}, 160325618Ssbruno 161325618Ssbruno /* CN2350 25G */ 162325618Ssbruno {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_25G_SUBDEVICE, 163325618Ssbruno 0x02, 2}, 164325618Ssbruno 165325618Ssbruno /* CN2360 25G */ 166325618Ssbruno {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_25G_SUBDEVICE, 167325618Ssbruno 0x02, 3}, 168325618Ssbruno 169325618Ssbruno {0, 0, 0, 0, 0} 170325618Ssbruno}; 171325618Ssbruno 172325618Ssbrunostatic char *lio_strings[] = { 173325618Ssbruno "LiquidIO 2350 10GbE Server Adapter", 174325618Ssbruno "LiquidIO 2360 10GbE Server Adapter", 175325618Ssbruno "LiquidIO 2350 25GbE Server Adapter", 176325618Ssbruno "LiquidIO 2360 25GbE Server Adapter", 177325618Ssbruno}; 178325618Ssbruno 179325618Ssbrunostruct lio_if_cfg_resp { 180325618Ssbruno uint64_t rh; 181325618Ssbruno struct octeon_if_cfg_info cfg_info; 182325618Ssbruno uint64_t status; 183325618Ssbruno}; 184325618Ssbruno 185325618Ssbrunostruct lio_if_cfg_context { 186325618Ssbruno int octeon_id; 187325618Ssbruno volatile int cond; 188325618Ssbruno}; 189325618Ssbruno 190325618Ssbrunostruct lio_rx_ctl_context { 191325618Ssbruno int octeon_id; 192325618Ssbruno volatile int cond; 193325618Ssbruno}; 194325618Ssbruno 195325618Ssbrunostatic int 196325618Ssbrunolio_probe(device_t dev) 197325618Ssbruno{ 198325618Ssbruno struct lio_vendor_info *tbl; 199325618Ssbruno 200325618Ssbruno uint16_t vendor_id; 201325618Ssbruno uint16_t device_id; 202325618Ssbruno uint16_t subdevice_id; 203325618Ssbruno uint8_t revision_id; 204325618Ssbruno char device_ver[256]; 205325618Ssbruno 206325618Ssbruno vendor_id = pci_get_vendor(dev); 207325618Ssbruno if (vendor_id != PCI_VENDOR_ID_CAVIUM) 208325618Ssbruno return (ENXIO); 209325618Ssbruno 210325618Ssbruno device_id = pci_get_device(dev); 211325618Ssbruno subdevice_id = pci_get_subdevice(dev); 212325618Ssbruno revision_id = pci_get_revid(dev); 213325618Ssbruno 214325618Ssbruno tbl = lio_pci_tbl; 215325618Ssbruno while (tbl->vendor_id) { 216325618Ssbruno if ((vendor_id == tbl->vendor_id) && 217325618Ssbruno (device_id == tbl->device_id) && 218325618Ssbruno (subdevice_id == tbl->subdevice_id) && 219325618Ssbruno (revision_id == tbl->revision_id)) { 220325618Ssbruno sprintf(device_ver, "%s, Version - %s", 221325618Ssbruno lio_strings[tbl->index], LIO_VERSION); 222325618Ssbruno device_set_desc_copy(dev, device_ver); 223325618Ssbruno return (BUS_PROBE_DEFAULT); 224325618Ssbruno } 225325618Ssbruno 226325618Ssbruno tbl++; 227325618Ssbruno } 228325618Ssbruno 229325618Ssbruno return (ENXIO); 230325618Ssbruno} 231325618Ssbruno 232325618Ssbrunostatic int 233325618Ssbrunolio_attach(device_t device) 234325618Ssbruno{ 235325618Ssbruno struct octeon_device *oct_dev = NULL; 236325618Ssbruno uint64_t scratch1; 237325618Ssbruno uint32_t error; 238325618Ssbruno int timeout, ret = 1; 239325618Ssbruno uint8_t bus, dev, function; 240325618Ssbruno 241325618Ssbruno oct_dev = lio_allocate_device(device); 242325618Ssbruno if (oct_dev == NULL) { 243325618Ssbruno device_printf(device, "Error: Unable to allocate device\n"); 244325618Ssbruno return (-ENOMEM); 245325618Ssbruno } 246325618Ssbruno 247325618Ssbruno oct_dev->tx_budget = LIO_DEFAULT_TX_PKTS_PROCESS_BUDGET; 248325618Ssbruno oct_dev->rx_budget = LIO_DEFAULT_RX_PKTS_PROCESS_BUDGET; 249325618Ssbruno oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED; 250325618Ssbruno 251325618Ssbruno oct_dev->device = device; 252325618Ssbruno bus = pci_get_bus(device); 253325618Ssbruno dev = pci_get_slot(device); 254325618Ssbruno function = pci_get_function(device); 255325618Ssbruno 256325618Ssbruno lio_dev_info(oct_dev, "Initializing device %x:%x %02x:%02x.%01x\n", 257325618Ssbruno pci_get_vendor(device), pci_get_device(device), bus, dev, 258325618Ssbruno function); 259325618Ssbruno 260325618Ssbruno if (lio_device_init(oct_dev)) { 261325618Ssbruno lio_dev_err(oct_dev, "Failed to init device\n"); 262325618Ssbruno lio_detach(device); 263325618Ssbruno return (-ENOMEM); 264325618Ssbruno } 265325618Ssbruno 266325618Ssbruno scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1); 267325618Ssbruno if (!(scratch1 & 4ULL)) { 268325618Ssbruno /* 269325618Ssbruno * Bit 2 of SLI_SCRATCH_1 is a flag that indicates that 270325618Ssbruno * the lio watchdog kernel thread is running for this 271325618Ssbruno * NIC. Each NIC gets one watchdog kernel thread. 272325618Ssbruno */ 273325618Ssbruno scratch1 |= 4ULL; 274325618Ssbruno lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1); 275325618Ssbruno 276325618Ssbruno error = kproc_create(lio_watchdog, oct_dev, 277325618Ssbruno &oct_dev->watchdog_task, 0, 0, 278325618Ssbruno "liowd/%02hhx:%02hhx.%hhx", bus, 279325618Ssbruno dev, function); 280325618Ssbruno if (!error) { 281325618Ssbruno kproc_resume(oct_dev->watchdog_task); 282325618Ssbruno } else { 283325618Ssbruno oct_dev->watchdog_task = NULL; 284325618Ssbruno lio_dev_err(oct_dev, 285325618Ssbruno "failed to create kernel_thread\n"); 286325618Ssbruno lio_detach(device); 287325618Ssbruno return (-1); 288325618Ssbruno } 289325618Ssbruno } 290325618Ssbruno oct_dev->rx_pause = 1; 291325618Ssbruno oct_dev->tx_pause = 1; 292325618Ssbruno 293325618Ssbruno timeout = 0; 294325618Ssbruno while (timeout < LIO_NIC_STARTER_TIMEOUT) { 295325618Ssbruno lio_mdelay(LIO_STARTER_POLL_INTERVAL_MS); 296325618Ssbruno timeout += LIO_STARTER_POLL_INTERVAL_MS; 297325618Ssbruno 298325618Ssbruno /* 299325618Ssbruno * During the boot process interrupts are not available. 300325618Ssbruno * So polling for first control message from FW. 301325618Ssbruno */ 302325618Ssbruno if (cold) 303325618Ssbruno lio_droq_bh(oct_dev->droq[0], 0); 304325618Ssbruno 305325618Ssbruno if (atomic_load_acq_int(&oct_dev->status) == LIO_DEV_CORE_OK) { 306325618Ssbruno ret = lio_nic_starter(oct_dev); 307325618Ssbruno break; 308325618Ssbruno } 309325618Ssbruno } 310325618Ssbruno 311325618Ssbruno if (ret) { 312325618Ssbruno lio_dev_err(oct_dev, "Firmware failed to start\n"); 313325618Ssbruno lio_detach(device); 314325618Ssbruno return (-EIO); 315325618Ssbruno } 316325618Ssbruno 317325618Ssbruno lio_dev_dbg(oct_dev, "Device is ready\n"); 318325618Ssbruno 319325618Ssbruno return (0); 320325618Ssbruno} 321325618Ssbruno 322325618Ssbrunostatic int 323325618Ssbrunolio_detach(device_t dev) 324325618Ssbruno{ 325325618Ssbruno struct octeon_device *oct_dev = device_get_softc(dev); 326325618Ssbruno 327325618Ssbruno lio_dev_dbg(oct_dev, "Stopping device\n"); 328325618Ssbruno if (oct_dev->watchdog_task) { 329325618Ssbruno uint64_t scratch1; 330325618Ssbruno 331325618Ssbruno kproc_suspend(oct_dev->watchdog_task, 0); 332325618Ssbruno 333325618Ssbruno scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1); 334325618Ssbruno scratch1 &= ~4ULL; 335325618Ssbruno lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1); 336325618Ssbruno } 337325618Ssbruno 338325618Ssbruno if (oct_dev->app_mode && (oct_dev->app_mode == LIO_DRV_NIC_APP)) 339325618Ssbruno lio_stop_nic_module(oct_dev); 340325618Ssbruno 341325618Ssbruno /* 342325618Ssbruno * Reset the octeon device and cleanup all memory allocated for 343325618Ssbruno * the octeon device by driver. 344325618Ssbruno */ 345325618Ssbruno lio_destroy_resources(oct_dev); 346325618Ssbruno 347325618Ssbruno lio_dev_info(oct_dev, "Device removed\n"); 348325618Ssbruno 349325618Ssbruno /* 350325618Ssbruno * This octeon device has been removed. Update the global 351325618Ssbruno * data structure to reflect this. Free the device structure. 352325618Ssbruno */ 353325618Ssbruno lio_free_device_mem(oct_dev); 354325618Ssbruno return (0); 355325618Ssbruno} 356325618Ssbruno 357325618Ssbrunostatic int 358325618Ssbrunolio_shutdown(device_t dev) 359325618Ssbruno{ 360325618Ssbruno struct octeon_device *oct_dev = device_get_softc(dev); 361325618Ssbruno struct lio *lio = if_getsoftc(oct_dev->props.ifp); 362325618Ssbruno 363325618Ssbruno lio_send_rx_ctrl_cmd(lio, 0); 364325618Ssbruno 365325618Ssbruno return (0); 366325618Ssbruno} 367325618Ssbruno 368325618Ssbrunostatic int 369325618Ssbrunolio_suspend(device_t dev) 370325618Ssbruno{ 371325618Ssbruno 372325618Ssbruno return (ENXIO); 373325618Ssbruno} 374325618Ssbruno 375325618Ssbrunostatic int 376325618Ssbrunolio_resume(device_t dev) 377325618Ssbruno{ 378325618Ssbruno 379325618Ssbruno return (ENXIO); 380325618Ssbruno} 381325618Ssbruno 382325618Ssbrunostatic int 383325618Ssbrunolio_event(struct module *mod, int event, void *junk) 384325618Ssbruno{ 385325618Ssbruno 386325618Ssbruno switch (event) { 387325618Ssbruno case MOD_LOAD: 388325618Ssbruno lio_init_device_list(LIO_CFG_TYPE_DEFAULT); 389325618Ssbruno break; 390325618Ssbruno default: 391325618Ssbruno break; 392325618Ssbruno } 393325618Ssbruno 394325618Ssbruno return (0); 395325618Ssbruno} 396325618Ssbruno 397325618Ssbruno/********************************************************************* 398325618Ssbruno * FreeBSD Device Interface Entry Points 399325618Ssbruno * *******************************************************************/ 400325618Ssbrunostatic device_method_t lio_methods[] = { 401325618Ssbruno /* Device interface */ 402325618Ssbruno DEVMETHOD(device_probe, lio_probe), 403325618Ssbruno DEVMETHOD(device_attach, lio_attach), 404325618Ssbruno DEVMETHOD(device_detach, lio_detach), 405325618Ssbruno DEVMETHOD(device_shutdown, lio_shutdown), 406325618Ssbruno DEVMETHOD(device_suspend, lio_suspend), 407325618Ssbruno DEVMETHOD(device_resume, lio_resume), 408325618Ssbruno DEVMETHOD_END 409325618Ssbruno}; 410325618Ssbruno 411325618Ssbrunostatic driver_t lio_driver = { 412325618Ssbruno LIO_DRV_NAME, lio_methods, sizeof(struct octeon_device), 413325618Ssbruno}; 414325618Ssbruno 415325618Ssbrunodevclass_t lio_devclass; 416325618SsbrunoDRIVER_MODULE(lio, pci, lio_driver, lio_devclass, lio_event, 0); 417325618Ssbruno 418325618SsbrunoMODULE_DEPEND(lio, pci, 1, 1, 1); 419325618SsbrunoMODULE_DEPEND(lio, ether, 1, 1, 1); 420325618SsbrunoMODULE_DEPEND(lio, firmware, 1, 1, 1); 421325618Ssbruno 422325618Ssbrunostatic bool 423325618Ssbrunofw_type_is_none(void) 424325618Ssbruno{ 425325618Ssbruno return strncmp(fw_type, LIO_FW_NAME_TYPE_NONE, 426325618Ssbruno sizeof(LIO_FW_NAME_TYPE_NONE)) == 0; 427325618Ssbruno} 428325618Ssbruno 429325618Ssbruno/* 430325618Ssbruno * \brief Device initialization for each Octeon device that is probed 431325618Ssbruno * @param octeon_dev octeon device 432325618Ssbruno */ 433325618Ssbrunostatic int 434325618Ssbrunolio_device_init(struct octeon_device *octeon_dev) 435325618Ssbruno{ 436325618Ssbruno unsigned long ddr_timeout = LIO_DDR_TIMEOUT; 437325618Ssbruno char *dbg_enb = NULL; 438325618Ssbruno int fw_loaded = 0; 439325618Ssbruno int i, j, ret; 440325618Ssbruno uint8_t bus, dev, function; 441325618Ssbruno char bootcmd[] = "\n"; 442325618Ssbruno 443325618Ssbruno bus = pci_get_bus(octeon_dev->device); 444325618Ssbruno dev = pci_get_slot(octeon_dev->device); 445325618Ssbruno function = pci_get_function(octeon_dev->device); 446325618Ssbruno 447325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_BEGIN_STATE); 448325618Ssbruno 449325618Ssbruno /* Enable access to the octeon device */ 450325618Ssbruno if (pci_enable_busmaster(octeon_dev->device)) { 451325618Ssbruno lio_dev_err(octeon_dev, "pci_enable_device failed\n"); 452325618Ssbruno return (1); 453325618Ssbruno } 454325618Ssbruno 455325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_ENABLE_DONE); 456325618Ssbruno 457325618Ssbruno /* Identify the Octeon type and map the BAR address space. */ 458325618Ssbruno if (lio_chip_specific_setup(octeon_dev)) { 459325618Ssbruno lio_dev_err(octeon_dev, "Chip specific setup failed\n"); 460325618Ssbruno return (1); 461325618Ssbruno } 462325618Ssbruno 463325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_MAP_DONE); 464325618Ssbruno 465325618Ssbruno /* 466325618Ssbruno * Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE', 467325618Ssbruno * since that is what is required for the reference to be removed 468325618Ssbruno * during de-initialization (see 'octeon_destroy_resources'). 469325618Ssbruno */ 470325618Ssbruno lio_register_device(octeon_dev, bus, dev, function, true); 471325618Ssbruno 472325618Ssbruno 473325618Ssbruno octeon_dev->app_mode = LIO_DRV_INVALID_APP; 474325618Ssbruno 475325618Ssbruno if (!lio_cn23xx_pf_fw_loaded(octeon_dev) && !fw_type_is_none()) { 476325618Ssbruno fw_loaded = 0; 477325618Ssbruno /* Do a soft reset of the Octeon device. */ 478325618Ssbruno if (octeon_dev->fn_list.soft_reset(octeon_dev)) 479325618Ssbruno return (1); 480325618Ssbruno 481325618Ssbruno /* things might have changed */ 482325618Ssbruno if (!lio_cn23xx_pf_fw_loaded(octeon_dev)) 483325618Ssbruno fw_loaded = 0; 484325618Ssbruno else 485325618Ssbruno fw_loaded = 1; 486325618Ssbruno } else { 487325618Ssbruno fw_loaded = 1; 488325618Ssbruno } 489325618Ssbruno 490325618Ssbruno /* 491325618Ssbruno * Initialize the dispatch mechanism used to push packets arriving on 492325618Ssbruno * Octeon Output queues. 493325618Ssbruno */ 494325618Ssbruno if (lio_init_dispatch_list(octeon_dev)) 495325618Ssbruno return (1); 496325618Ssbruno 497325618Ssbruno lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC, 498325618Ssbruno LIO_OPCODE_NIC_CORE_DRV_ACTIVE, 499325618Ssbruno lio_core_drv_init, octeon_dev); 500325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DISPATCH_INIT_DONE); 501325618Ssbruno 502325618Ssbruno ret = octeon_dev->fn_list.setup_device_regs(octeon_dev); 503325618Ssbruno if (ret) { 504325618Ssbruno lio_dev_err(octeon_dev, 505325618Ssbruno "Failed to configure device registers\n"); 506325618Ssbruno return (ret); 507325618Ssbruno } 508325618Ssbruno 509325618Ssbruno /* Initialize soft command buffer pool */ 510325618Ssbruno if (lio_setup_sc_buffer_pool(octeon_dev)) { 511325618Ssbruno lio_dev_err(octeon_dev, "sc buffer pool allocation failed\n"); 512325618Ssbruno return (1); 513325618Ssbruno } 514325618Ssbruno 515325618Ssbruno atomic_store_rel_int(&octeon_dev->status, 516325618Ssbruno LIO_DEV_SC_BUFF_POOL_INIT_DONE); 517325618Ssbruno 518325618Ssbruno if (lio_allocate_ioq_vector(octeon_dev)) { 519325618Ssbruno lio_dev_err(octeon_dev, 520325618Ssbruno "IOQ vector allocation failed\n"); 521325618Ssbruno return (1); 522325618Ssbruno } 523325618Ssbruno 524325618Ssbruno atomic_store_rel_int(&octeon_dev->status, 525325618Ssbruno LIO_DEV_MSIX_ALLOC_VECTOR_DONE); 526325618Ssbruno 527325618Ssbruno for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) { 528325618Ssbruno octeon_dev->instr_queue[i] = 529325618Ssbruno malloc(sizeof(struct lio_instr_queue), 530325618Ssbruno M_DEVBUF, M_NOWAIT | M_ZERO); 531325618Ssbruno if (octeon_dev->instr_queue[i] == NULL) 532325618Ssbruno return (1); 533325618Ssbruno } 534325618Ssbruno 535325618Ssbruno /* Setup the data structures that manage this Octeon's Input queues. */ 536325618Ssbruno if (lio_setup_instr_queue0(octeon_dev)) { 537325618Ssbruno lio_dev_err(octeon_dev, 538325618Ssbruno "Instruction queue initialization failed\n"); 539325618Ssbruno return (1); 540325618Ssbruno } 541325618Ssbruno 542325618Ssbruno atomic_store_rel_int(&octeon_dev->status, 543325618Ssbruno LIO_DEV_INSTR_QUEUE_INIT_DONE); 544325618Ssbruno 545325618Ssbruno /* 546325618Ssbruno * Initialize lists to manage the requests of different types that 547325618Ssbruno * arrive from user & kernel applications for this octeon device. 548325618Ssbruno */ 549325618Ssbruno 550325618Ssbruno if (lio_setup_response_list(octeon_dev)) { 551325618Ssbruno lio_dev_err(octeon_dev, "Response list allocation failed\n"); 552325618Ssbruno return (1); 553325618Ssbruno } 554325618Ssbruno 555325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_RESP_LIST_INIT_DONE); 556325618Ssbruno 557325618Ssbruno for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) { 558325618Ssbruno octeon_dev->droq[i] = malloc(sizeof(*octeon_dev->droq[i]), 559325618Ssbruno M_DEVBUF, M_NOWAIT | M_ZERO); 560325618Ssbruno if (octeon_dev->droq[i] == NULL) 561325618Ssbruno return (1); 562325618Ssbruno } 563325618Ssbruno 564325618Ssbruno if (lio_setup_output_queue0(octeon_dev)) { 565325618Ssbruno lio_dev_err(octeon_dev, "Output queue initialization failed\n"); 566325618Ssbruno return (1); 567325618Ssbruno } 568325618Ssbruno 569325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DROQ_INIT_DONE); 570325618Ssbruno 571325618Ssbruno /* 572325618Ssbruno * Setup the interrupt handler and record the INT SUM register address 573325618Ssbruno */ 574325618Ssbruno if (lio_setup_interrupt(octeon_dev, 575325618Ssbruno octeon_dev->sriov_info.num_pf_rings)) 576325618Ssbruno return (1); 577325618Ssbruno 578325618Ssbruno /* Enable Octeon device interrupts */ 579325618Ssbruno octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR); 580325618Ssbruno 581325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_INTR_SET_DONE); 582325618Ssbruno 583325618Ssbruno /* 584325618Ssbruno * Send Credit for Octeon Output queues. Credits are always sent BEFORE 585325618Ssbruno * the output queue is enabled. 586325618Ssbruno * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in 587325618Ssbruno * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0. 588325618Ssbruno * Otherwise, it is possible that the DRV_ACTIVE message will be sent 589325618Ssbruno * before any credits have been issued, causing the ring to be reset 590325618Ssbruno * (and the f/w appear to never have started). 591325618Ssbruno */ 592325618Ssbruno for (j = 0; j < octeon_dev->num_oqs; j++) 593325618Ssbruno lio_write_csr32(octeon_dev, 594325618Ssbruno octeon_dev->droq[j]->pkts_credit_reg, 595325618Ssbruno octeon_dev->droq[j]->max_count); 596325618Ssbruno 597325618Ssbruno /* Enable the input and output queues for this Octeon device */ 598325618Ssbruno ret = octeon_dev->fn_list.enable_io_queues(octeon_dev); 599325618Ssbruno if (ret) { 600325618Ssbruno lio_dev_err(octeon_dev, "Failed to enable input/output queues"); 601325618Ssbruno return (ret); 602325618Ssbruno } 603325618Ssbruno 604325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_IO_QUEUES_DONE); 605325618Ssbruno 606325618Ssbruno if (!fw_loaded) { 607325618Ssbruno lio_dev_dbg(octeon_dev, "Waiting for DDR initialization...\n"); 608325618Ssbruno if (!ddr_timeout) { 609325618Ssbruno lio_dev_info(octeon_dev, 610325618Ssbruno "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n"); 611325618Ssbruno } 612325618Ssbruno 613325618Ssbruno lio_sleep_timeout(LIO_RESET_MSECS); 614325618Ssbruno 615325618Ssbruno /* 616325618Ssbruno * Wait for the octeon to initialize DDR after the 617325618Ssbruno * soft-reset. 618325618Ssbruno */ 619325618Ssbruno while (!ddr_timeout) { 620325618Ssbruno if (pause("-", lio_ms_to_ticks(100))) { 621325618Ssbruno /* user probably pressed Control-C */ 622325618Ssbruno return (1); 623325618Ssbruno } 624325618Ssbruno } 625325618Ssbruno 626325618Ssbruno ret = lio_wait_for_ddr_init(octeon_dev, &ddr_timeout); 627325618Ssbruno if (ret) { 628325618Ssbruno lio_dev_err(octeon_dev, 629325618Ssbruno "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n", 630325618Ssbruno ret); 631325618Ssbruno return (1); 632325618Ssbruno } 633325618Ssbruno 634325618Ssbruno if (lio_wait_for_bootloader(octeon_dev, 1100)) { 635325618Ssbruno lio_dev_err(octeon_dev, "Board not responding\n"); 636325618Ssbruno return (1); 637325618Ssbruno } 638325618Ssbruno 639325618Ssbruno /* Divert uboot to take commands from host instead. */ 640325618Ssbruno ret = lio_console_send_cmd(octeon_dev, bootcmd, 50); 641325618Ssbruno 642325618Ssbruno lio_dev_dbg(octeon_dev, "Initializing consoles\n"); 643325618Ssbruno ret = lio_init_consoles(octeon_dev); 644325618Ssbruno if (ret) { 645325618Ssbruno lio_dev_err(octeon_dev, "Could not access board consoles\n"); 646325618Ssbruno return (1); 647325618Ssbruno } 648325618Ssbruno 649325618Ssbruno /* 650325618Ssbruno * If console debug enabled, specify empty string to 651325618Ssbruno * use default enablement ELSE specify NULL string for 652325618Ssbruno * 'disabled'. 653325618Ssbruno */ 654325618Ssbruno dbg_enb = lio_console_debug_enabled(0) ? "" : NULL; 655325618Ssbruno ret = lio_add_console(octeon_dev, 0, dbg_enb); 656325618Ssbruno 657325618Ssbruno if (ret) { 658325618Ssbruno lio_dev_err(octeon_dev, "Could not access board console\n"); 659325618Ssbruno return (1); 660325618Ssbruno } else if (lio_console_debug_enabled(0)) { 661325618Ssbruno /* 662325618Ssbruno * If console was added AND we're logging console output 663325618Ssbruno * then set our console print function. 664325618Ssbruno */ 665325618Ssbruno octeon_dev->console[0].print = lio_dbg_console_print; 666325618Ssbruno } 667325618Ssbruno 668325618Ssbruno atomic_store_rel_int(&octeon_dev->status, 669325618Ssbruno LIO_DEV_CONSOLE_INIT_DONE); 670325618Ssbruno 671325618Ssbruno lio_dev_dbg(octeon_dev, "Loading firmware\n"); 672325618Ssbruno 673325618Ssbruno ret = lio_load_firmware(octeon_dev); 674325618Ssbruno if (ret) { 675325618Ssbruno lio_dev_err(octeon_dev, "Could not load firmware to board\n"); 676325618Ssbruno return (1); 677325618Ssbruno } 678325618Ssbruno } 679325618Ssbruno 680325618Ssbruno atomic_store_rel_int(&octeon_dev->status, LIO_DEV_HOST_OK); 681325618Ssbruno 682325618Ssbruno return (0); 683325618Ssbruno} 684325618Ssbruno 685325618Ssbruno/* 686325618Ssbruno * \brief PCI FLR for each Octeon device. 687325618Ssbruno * @param oct octeon device 688325618Ssbruno */ 689325618Ssbrunostatic void 690325618Ssbrunolio_pci_flr(struct octeon_device *oct) 691325618Ssbruno{ 692325618Ssbruno uint32_t exppos, status; 693325618Ssbruno 694325618Ssbruno pci_find_cap(oct->device, PCIY_EXPRESS, &exppos); 695325618Ssbruno 696325618Ssbruno pci_save_state(oct->device); 697325618Ssbruno 698325618Ssbruno /* Quiesce the device completely */ 699325618Ssbruno pci_write_config(oct->device, PCIR_COMMAND, PCIM_CMD_INTxDIS, 2); 700325618Ssbruno 701325618Ssbruno /* Wait for Transaction Pending bit clean */ 702325618Ssbruno lio_mdelay(100); 703325618Ssbruno 704325618Ssbruno status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2); 705325618Ssbruno if (status & PCIEM_STA_TRANSACTION_PND) { 706325618Ssbruno lio_dev_info(oct, "Function reset incomplete after 100ms, sleeping for 5 seconds\n"); 707325618Ssbruno lio_mdelay(5); 708325618Ssbruno 709325618Ssbruno status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2); 710325618Ssbruno if (status & PCIEM_STA_TRANSACTION_PND) 711325618Ssbruno lio_dev_info(oct, "Function reset still incomplete after 5s, reset anyway\n"); 712325618Ssbruno } 713325618Ssbruno 714325618Ssbruno pci_write_config(oct->device, exppos + PCIER_DEVICE_CTL, PCIEM_CTL_INITIATE_FLR, 2); 715325618Ssbruno lio_mdelay(100); 716325618Ssbruno 717325618Ssbruno pci_restore_state(oct->device); 718325618Ssbruno} 719325618Ssbruno 720325618Ssbruno/* 721325618Ssbruno * \brief Debug console print function 722325618Ssbruno * @param octeon_dev octeon device 723325618Ssbruno * @param console_num console number 724325618Ssbruno * @param prefix first portion of line to display 725325618Ssbruno * @param suffix second portion of line to display 726325618Ssbruno * 727325618Ssbruno * The OCTEON debug console outputs entire lines (excluding '\n'). 728325618Ssbruno * Normally, the line will be passed in the 'prefix' parameter. 729325618Ssbruno * However, due to buffering, it is possible for a line to be split into two 730325618Ssbruno * parts, in which case they will be passed as the 'prefix' parameter and 731325618Ssbruno * 'suffix' parameter. 732325618Ssbruno */ 733325618Ssbrunostatic int 734325618Ssbrunolio_dbg_console_print(struct octeon_device *oct, uint32_t console_num, 735325618Ssbruno char *prefix, char *suffix) 736325618Ssbruno{ 737325618Ssbruno 738325618Ssbruno if (prefix != NULL && suffix != NULL) 739325618Ssbruno lio_dev_info(oct, "%u: %s%s\n", console_num, prefix, suffix); 740325618Ssbruno else if (prefix != NULL) 741325618Ssbruno lio_dev_info(oct, "%u: %s\n", console_num, prefix); 742325618Ssbruno else if (suffix != NULL) 743325618Ssbruno lio_dev_info(oct, "%u: %s\n", console_num, suffix); 744325618Ssbruno 745325618Ssbruno return (0); 746325618Ssbruno} 747325618Ssbruno 748325618Ssbrunostatic void 749325618Ssbrunolio_watchdog(void *param) 750325618Ssbruno{ 751325618Ssbruno int core_num; 752325618Ssbruno uint16_t mask_of_crashed_or_stuck_cores = 0; 753325618Ssbruno struct octeon_device *oct = param; 754325618Ssbruno bool err_msg_was_printed[12]; 755325618Ssbruno 756325618Ssbruno bzero(err_msg_was_printed, sizeof(err_msg_was_printed)); 757325618Ssbruno 758325618Ssbruno while (1) { 759325618Ssbruno kproc_suspend_check(oct->watchdog_task); 760325618Ssbruno mask_of_crashed_or_stuck_cores = 761325618Ssbruno (uint16_t)lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH2); 762325618Ssbruno 763325618Ssbruno if (mask_of_crashed_or_stuck_cores) { 764325618Ssbruno struct octeon_device *other_oct; 765325618Ssbruno 766325618Ssbruno oct->cores_crashed = true; 767325618Ssbruno other_oct = lio_get_other_octeon_device(oct); 768325618Ssbruno if (other_oct != NULL) 769325618Ssbruno other_oct->cores_crashed = true; 770325618Ssbruno 771325618Ssbruno for (core_num = 0; core_num < LIO_MAX_CORES; 772325618Ssbruno core_num++) { 773325618Ssbruno bool core_crashed_or_got_stuck; 774325618Ssbruno 775325618Ssbruno core_crashed_or_got_stuck = 776325618Ssbruno (mask_of_crashed_or_stuck_cores >> 777325618Ssbruno core_num) & 1; 778325618Ssbruno if (core_crashed_or_got_stuck && 779325618Ssbruno !err_msg_was_printed[core_num]) { 780325618Ssbruno lio_dev_err(oct, 781325618Ssbruno "ERROR: Octeon core %d crashed or got stuck! See oct-fwdump for details.\n", 782325618Ssbruno core_num); 783325618Ssbruno err_msg_was_printed[core_num] = true; 784325618Ssbruno } 785325618Ssbruno } 786325618Ssbruno 787325618Ssbruno } 788325618Ssbruno 789325618Ssbruno /* sleep for two seconds */ 790325618Ssbruno pause("-", lio_ms_to_ticks(2000)); 791325618Ssbruno } 792325618Ssbruno} 793325618Ssbruno 794325618Ssbrunostatic int 795325618Ssbrunolio_chip_specific_setup(struct octeon_device *oct) 796325618Ssbruno{ 797325618Ssbruno char *s; 798325618Ssbruno uint32_t dev_id, rev_id; 799325618Ssbruno int ret = 1; 800325618Ssbruno 801325618Ssbruno dev_id = lio_read_pci_cfg(oct, 0); 802325618Ssbruno rev_id = pci_get_revid(oct->device); 803325618Ssbruno oct->subdevice_id = pci_get_subdevice(oct->device); 804325618Ssbruno 805325618Ssbruno switch (dev_id) { 806325618Ssbruno case LIO_CN23XX_PF_PCIID: 807325618Ssbruno oct->chip_id = LIO_CN23XX_PF_VID; 808325618Ssbruno if (pci_get_function(oct->device) == 0) { 809325618Ssbruno if (num_queues_per_pf0 < 0) { 810325618Ssbruno lio_dev_info(oct, "Invalid num_queues_per_pf0: %d, Setting it to default\n", 811325618Ssbruno num_queues_per_pf0); 812325618Ssbruno num_queues_per_pf0 = 0; 813325618Ssbruno } 814325618Ssbruno 815325618Ssbruno oct->sriov_info.num_pf_rings = num_queues_per_pf0; 816325618Ssbruno } else { 817325618Ssbruno if (num_queues_per_pf1 < 0) { 818325618Ssbruno lio_dev_info(oct, "Invalid num_queues_per_pf1: %d, Setting it to default\n", 819325618Ssbruno num_queues_per_pf1); 820325618Ssbruno num_queues_per_pf1 = 0; 821325618Ssbruno } 822325618Ssbruno 823325618Ssbruno oct->sriov_info.num_pf_rings = num_queues_per_pf1; 824325618Ssbruno } 825325618Ssbruno 826325618Ssbruno ret = lio_cn23xx_pf_setup_device(oct); 827325618Ssbruno s = "CN23XX"; 828325618Ssbruno break; 829325618Ssbruno 830325618Ssbruno default: 831325618Ssbruno s = "?"; 832325618Ssbruno lio_dev_err(oct, "Unknown device found (dev_id: %x)\n", dev_id); 833325618Ssbruno } 834325618Ssbruno 835325618Ssbruno if (!ret) 836325618Ssbruno lio_dev_info(oct, "%s PASS%d.%d %s Version: %s\n", s, 837325618Ssbruno OCTEON_MAJOR_REV(oct), OCTEON_MINOR_REV(oct), 838325618Ssbruno lio_get_conf(oct)->card_name, LIO_VERSION); 839325618Ssbruno 840325618Ssbruno return (ret); 841325618Ssbruno} 842325618Ssbruno 843325618Ssbrunostatic struct octeon_device * 844325618Ssbrunolio_get_other_octeon_device(struct octeon_device *oct) 845325618Ssbruno{ 846325618Ssbruno struct octeon_device *other_oct; 847325618Ssbruno 848325618Ssbruno other_oct = lio_get_device(oct->octeon_id + 1); 849325618Ssbruno 850325618Ssbruno if ((other_oct != NULL) && other_oct->device) { 851325618Ssbruno int oct_busnum, other_oct_busnum; 852325618Ssbruno 853325618Ssbruno oct_busnum = pci_get_bus(oct->device); 854325618Ssbruno other_oct_busnum = pci_get_bus(other_oct->device); 855325618Ssbruno 856325618Ssbruno if (oct_busnum == other_oct_busnum) { 857325618Ssbruno int oct_slot, other_oct_slot; 858325618Ssbruno 859325618Ssbruno oct_slot = pci_get_slot(oct->device); 860325618Ssbruno other_oct_slot = pci_get_slot(other_oct->device); 861325618Ssbruno 862325618Ssbruno if (oct_slot == other_oct_slot) 863325618Ssbruno return (other_oct); 864325618Ssbruno } 865325618Ssbruno } 866325618Ssbruno return (NULL); 867325618Ssbruno} 868325618Ssbruno 869325618Ssbruno/* 870325618Ssbruno * \brief Load firmware to device 871325618Ssbruno * @param oct octeon device 872325618Ssbruno * 873325618Ssbruno * Maps device to firmware filename, requests firmware, and downloads it 874325618Ssbruno */ 875325618Ssbrunostatic int 876325618Ssbrunolio_load_firmware(struct octeon_device *oct) 877325618Ssbruno{ 878325618Ssbruno const struct firmware *fw; 879325618Ssbruno char *tmp_fw_type = NULL; 880325618Ssbruno int ret = 0; 881325618Ssbruno char fw_name[LIO_MAX_FW_FILENAME_LEN]; 882325618Ssbruno 883325618Ssbruno if (fw_type[0] == '\0') 884325618Ssbruno tmp_fw_type = LIO_FW_NAME_TYPE_NIC; 885325618Ssbruno else 886325618Ssbruno tmp_fw_type = fw_type; 887325618Ssbruno 888325618Ssbruno sprintf(fw_name, "%s%s_%s%s", LIO_FW_BASE_NAME, 889325618Ssbruno lio_get_conf(oct)->card_name, tmp_fw_type, LIO_FW_NAME_SUFFIX); 890325618Ssbruno 891325618Ssbruno fw = firmware_get(fw_name); 892325618Ssbruno if (fw == NULL) { 893325618Ssbruno lio_dev_err(oct, "Request firmware failed. Could not find file %s.\n", 894325618Ssbruno fw_name); 895325618Ssbruno return (EINVAL); 896325618Ssbruno } 897325618Ssbruno 898325618Ssbruno ret = lio_download_firmware(oct, fw->data, fw->datasize); 899325618Ssbruno 900325618Ssbruno firmware_put(fw, FIRMWARE_UNLOAD); 901325618Ssbruno 902325618Ssbruno return (ret); 903325618Ssbruno} 904325618Ssbruno 905325618Ssbrunostatic int 906325618Ssbrunolio_nic_starter(struct octeon_device *oct) 907325618Ssbruno{ 908325618Ssbruno int ret = 0; 909325618Ssbruno 910325618Ssbruno atomic_store_rel_int(&oct->status, LIO_DEV_RUNNING); 911325618Ssbruno 912325618Ssbruno if (oct->app_mode && oct->app_mode == LIO_DRV_NIC_APP) { 913325618Ssbruno if (lio_init_nic_module(oct)) { 914325618Ssbruno lio_dev_err(oct, "NIC initialization failed\n"); 915325618Ssbruno ret = -1; 916325618Ssbruno#ifdef CAVIUM_ONiLY_23XX_VF 917325618Ssbruno } else { 918325618Ssbruno if (octeon_enable_sriov(oct) < 0) 919325618Ssbruno ret = -1; 920325618Ssbruno#endif 921325618Ssbruno } 922325618Ssbruno } else { 923325618Ssbruno lio_dev_err(oct, 924325618Ssbruno "Unexpected application running on NIC (%d). Check firmware.\n", 925325618Ssbruno oct->app_mode); 926325618Ssbruno ret = -1; 927325618Ssbruno } 928325618Ssbruno 929325618Ssbruno return (ret); 930325618Ssbruno} 931325618Ssbruno 932325618Ssbrunostatic int 933325618Ssbrunolio_init_nic_module(struct octeon_device *oct) 934325618Ssbruno{ 935325618Ssbruno int num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG(lio_get_conf(oct)); 936325618Ssbruno int retval = 0; 937325618Ssbruno 938325618Ssbruno lio_dev_dbg(oct, "Initializing network interfaces\n"); 939325618Ssbruno 940325618Ssbruno /* 941325618Ssbruno * only default iq and oq were initialized 942325618Ssbruno * initialize the rest as well 943325618Ssbruno */ 944325618Ssbruno 945325618Ssbruno /* run port_config command for each port */ 946325618Ssbruno oct->ifcount = num_nic_ports; 947325618Ssbruno 948325618Ssbruno bzero(&oct->props, sizeof(struct lio_if_props)); 949325618Ssbruno 950325618Ssbruno oct->props.gmxport = -1; 951325618Ssbruno 952325618Ssbruno retval = lio_setup_nic_devices(oct); 953325618Ssbruno if (retval) { 954325618Ssbruno lio_dev_err(oct, "Setup NIC devices failed\n"); 955325618Ssbruno goto lio_init_failure; 956325618Ssbruno } 957325618Ssbruno 958325618Ssbruno lio_dev_dbg(oct, "Network interfaces ready\n"); 959325618Ssbruno 960325618Ssbruno return (retval); 961325618Ssbruno 962325618Ssbrunolio_init_failure: 963325618Ssbruno 964325618Ssbruno oct->ifcount = 0; 965325618Ssbruno 966325618Ssbruno return (retval); 967325618Ssbruno} 968325618Ssbruno 969325618Ssbrunostatic int 970325618Ssbrunolio_ifmedia_update(struct ifnet *ifp) 971325618Ssbruno{ 972325618Ssbruno struct lio *lio = if_getsoftc(ifp); 973325618Ssbruno struct ifmedia *ifm; 974325618Ssbruno 975325618Ssbruno ifm = &lio->ifmedia; 976325618Ssbruno 977325618Ssbruno /* We only support Ethernet media type. */ 978325618Ssbruno if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 979325618Ssbruno return (EINVAL); 980325618Ssbruno 981325618Ssbruno switch (IFM_SUBTYPE(ifm->ifm_media)) { 982325618Ssbruno case IFM_AUTO: 983325618Ssbruno break; 984325618Ssbruno case IFM_10G_CX4: 985325618Ssbruno case IFM_10G_SR: 986325618Ssbruno case IFM_10G_T: 987325618Ssbruno case IFM_10G_TWINAX: 988325618Ssbruno default: 989325618Ssbruno /* We don't support changing the media type. */ 990325618Ssbruno lio_dev_err(lio->oct_dev, "Invalid media type (%d)\n", 991325618Ssbruno IFM_SUBTYPE(ifm->ifm_media)); 992325618Ssbruno return (EINVAL); 993325618Ssbruno } 994325618Ssbruno 995325618Ssbruno return (0); 996325618Ssbruno} 997325618Ssbruno 998325618Ssbrunostatic int 999325618Ssbrunolio_get_media_subtype(struct octeon_device *oct) 1000325618Ssbruno{ 1001325618Ssbruno 1002325618Ssbruno switch(oct->subdevice_id) { 1003325618Ssbruno case LIO_CN2350_10G_SUBDEVICE: 1004325618Ssbruno case LIO_CN2350_10G_SUBDEVICE1: 1005325618Ssbruno case LIO_CN2360_10G_SUBDEVICE: 1006325618Ssbruno return (IFM_10G_SR); 1007325618Ssbruno 1008325618Ssbruno case LIO_CN2350_25G_SUBDEVICE: 1009325618Ssbruno case LIO_CN2360_25G_SUBDEVICE: 1010325618Ssbruno return (IFM_25G_SR); 1011325618Ssbruno } 1012325618Ssbruno 1013325618Ssbruno return (IFM_10G_SR); 1014325618Ssbruno} 1015325618Ssbruno 1016325618Ssbrunostatic uint64_t 1017325618Ssbrunolio_get_baudrate(struct octeon_device *oct) 1018325618Ssbruno{ 1019325618Ssbruno 1020325618Ssbruno switch(oct->subdevice_id) { 1021325618Ssbruno case LIO_CN2350_10G_SUBDEVICE: 1022325618Ssbruno case LIO_CN2350_10G_SUBDEVICE1: 1023325618Ssbruno case LIO_CN2360_10G_SUBDEVICE: 1024325618Ssbruno return (IF_Gbps(10)); 1025325618Ssbruno 1026325618Ssbruno case LIO_CN2350_25G_SUBDEVICE: 1027325618Ssbruno case LIO_CN2360_25G_SUBDEVICE: 1028325618Ssbruno return (IF_Gbps(25)); 1029325618Ssbruno } 1030325618Ssbruno 1031325618Ssbruno return (IF_Gbps(10)); 1032325618Ssbruno} 1033325618Ssbruno 1034325618Ssbrunostatic void 1035325618Ssbrunolio_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1036325618Ssbruno{ 1037325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1038325618Ssbruno 1039325618Ssbruno /* Report link down if the driver isn't running. */ 1040325618Ssbruno if (!lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) { 1041325618Ssbruno ifmr->ifm_active |= IFM_NONE; 1042325618Ssbruno return; 1043325618Ssbruno } 1044325618Ssbruno 1045325618Ssbruno /* Setup the default interface info. */ 1046325618Ssbruno ifmr->ifm_status = IFM_AVALID; 1047325618Ssbruno ifmr->ifm_active = IFM_ETHER; 1048325618Ssbruno 1049325618Ssbruno if (lio->linfo.link.s.link_up) { 1050325618Ssbruno ifmr->ifm_status |= IFM_ACTIVE; 1051325618Ssbruno } else { 1052325618Ssbruno ifmr->ifm_active |= IFM_NONE; 1053325618Ssbruno return; 1054325618Ssbruno } 1055325618Ssbruno 1056325618Ssbruno ifmr->ifm_active |= lio_get_media_subtype(lio->oct_dev); 1057325618Ssbruno 1058325618Ssbruno if (lio->linfo.link.s.duplex) 1059325618Ssbruno ifmr->ifm_active |= IFM_FDX; 1060325618Ssbruno else 1061325618Ssbruno ifmr->ifm_active |= IFM_HDX; 1062325618Ssbruno} 1063325618Ssbruno 1064325618Ssbrunostatic uint64_t 1065325618Ssbrunolio_get_counter(if_t ifp, ift_counter cnt) 1066325618Ssbruno{ 1067325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1068325618Ssbruno struct octeon_device *oct = lio->oct_dev; 1069325618Ssbruno uint64_t counter = 0; 1070325618Ssbruno int i, q_no; 1071325618Ssbruno 1072325618Ssbruno switch (cnt) { 1073325618Ssbruno case IFCOUNTER_IPACKETS: 1074325618Ssbruno for (i = 0; i < oct->num_oqs; i++) { 1075325618Ssbruno q_no = lio->linfo.rxpciq[i].s.q_no; 1076325618Ssbruno counter += oct->droq[q_no]->stats.rx_pkts_received; 1077325618Ssbruno } 1078325618Ssbruno break; 1079325618Ssbruno case IFCOUNTER_OPACKETS: 1080325618Ssbruno for (i = 0; i < oct->num_iqs; i++) { 1081325618Ssbruno q_no = lio->linfo.txpciq[i].s.q_no; 1082325618Ssbruno counter += oct->instr_queue[q_no]->stats.tx_done; 1083325618Ssbruno } 1084325618Ssbruno break; 1085325618Ssbruno case IFCOUNTER_IBYTES: 1086325618Ssbruno for (i = 0; i < oct->num_oqs; i++) { 1087325618Ssbruno q_no = lio->linfo.rxpciq[i].s.q_no; 1088325618Ssbruno counter += oct->droq[q_no]->stats.rx_bytes_received; 1089325618Ssbruno } 1090325618Ssbruno break; 1091325618Ssbruno case IFCOUNTER_OBYTES: 1092325618Ssbruno for (i = 0; i < oct->num_iqs; i++) { 1093325618Ssbruno q_no = lio->linfo.txpciq[i].s.q_no; 1094325618Ssbruno counter += oct->instr_queue[q_no]->stats.tx_tot_bytes; 1095325618Ssbruno } 1096325618Ssbruno break; 1097325618Ssbruno case IFCOUNTER_IQDROPS: 1098325618Ssbruno for (i = 0; i < oct->num_oqs; i++) { 1099325618Ssbruno q_no = lio->linfo.rxpciq[i].s.q_no; 1100325618Ssbruno counter += oct->droq[q_no]->stats.rx_dropped; 1101325618Ssbruno } 1102325618Ssbruno break; 1103325618Ssbruno case IFCOUNTER_OQDROPS: 1104325618Ssbruno for (i = 0; i < oct->num_iqs; i++) { 1105325618Ssbruno q_no = lio->linfo.txpciq[i].s.q_no; 1106325618Ssbruno counter += oct->instr_queue[q_no]->stats.tx_dropped; 1107325618Ssbruno } 1108325618Ssbruno break; 1109325618Ssbruno case IFCOUNTER_IMCASTS: 1110325618Ssbruno counter = oct->link_stats.fromwire.total_mcst; 1111325618Ssbruno break; 1112325618Ssbruno case IFCOUNTER_OMCASTS: 1113325618Ssbruno counter = oct->link_stats.fromhost.mcast_pkts_sent; 1114325618Ssbruno break; 1115325618Ssbruno case IFCOUNTER_COLLISIONS: 1116325618Ssbruno counter = oct->link_stats.fromhost.total_collisions; 1117325618Ssbruno break; 1118325618Ssbruno case IFCOUNTER_IERRORS: 1119325618Ssbruno counter = oct->link_stats.fromwire.fcs_err + 1120325618Ssbruno oct->link_stats.fromwire.l2_err + 1121325618Ssbruno oct->link_stats.fromwire.frame_err; 1122325618Ssbruno break; 1123325618Ssbruno default: 1124325618Ssbruno return (if_get_counter_default(ifp, cnt)); 1125325618Ssbruno } 1126325618Ssbruno 1127325618Ssbruno return (counter); 1128325618Ssbruno} 1129325618Ssbruno 1130325618Ssbrunostatic int 1131325618Ssbrunolio_init_ifnet(struct lio *lio) 1132325618Ssbruno{ 1133325618Ssbruno struct octeon_device *oct = lio->oct_dev; 1134325618Ssbruno if_t ifp = lio->ifp; 1135325618Ssbruno 1136325618Ssbruno /* ifconfig entrypoint for media type/status reporting */ 1137325618Ssbruno ifmedia_init(&lio->ifmedia, IFM_IMASK, lio_ifmedia_update, 1138325618Ssbruno lio_ifmedia_status); 1139325618Ssbruno 1140325618Ssbruno /* set the default interface values */ 1141325618Ssbruno ifmedia_add(&lio->ifmedia, 1142325618Ssbruno (IFM_ETHER | IFM_FDX | lio_get_media_subtype(oct)), 1143325618Ssbruno 0, NULL); 1144325618Ssbruno ifmedia_add(&lio->ifmedia, (IFM_ETHER | IFM_AUTO), 0, NULL); 1145325618Ssbruno ifmedia_set(&lio->ifmedia, (IFM_ETHER | IFM_AUTO)); 1146325618Ssbruno 1147325618Ssbruno lio->ifmedia.ifm_media = lio->ifmedia.ifm_cur->ifm_media; 1148325618Ssbruno lio_dev_dbg(oct, "IFMEDIA flags : %x\n", lio->ifmedia.ifm_media); 1149325618Ssbruno 1150325618Ssbruno if_initname(ifp, device_get_name(oct->device), 1151325618Ssbruno device_get_unit(oct->device)); 1152325618Ssbruno if_setflags(ifp, (IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST)); 1153325618Ssbruno if_setioctlfn(ifp, lio_ioctl); 1154325618Ssbruno if_setgetcounterfn(ifp, lio_get_counter); 1155325618Ssbruno if_settransmitfn(ifp, lio_mq_start); 1156325618Ssbruno if_setqflushfn(ifp, lio_qflush); 1157325618Ssbruno if_setinitfn(ifp, lio_open); 1158325618Ssbruno if_setmtu(ifp, lio->linfo.link.s.mtu); 1159325618Ssbruno lio->mtu = lio->linfo.link.s.mtu; 1160325618Ssbruno if_sethwassist(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO | 1161325618Ssbruno CSUM_TCP_IPV6 | CSUM_UDP_IPV6)); 1162325618Ssbruno 1163325618Ssbruno if_setcapabilitiesbit(ifp, (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | 1164325618Ssbruno IFCAP_TSO | IFCAP_LRO | 1165325618Ssbruno IFCAP_JUMBO_MTU | IFCAP_HWSTATS | 1166325618Ssbruno IFCAP_LINKSTATE | IFCAP_VLAN_HWFILTER | 1167325618Ssbruno IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTAGGING | 1168325618Ssbruno IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU), 0); 1169325618Ssbruno 1170325618Ssbruno if_setcapenable(ifp, if_getcapabilities(ifp)); 1171325618Ssbruno if_setbaudrate(ifp, lio_get_baudrate(oct)); 1172325618Ssbruno 1173325618Ssbruno return (0); 1174325618Ssbruno} 1175325618Ssbruno 1176325618Ssbrunostatic void 1177325618Ssbrunolio_tcp_lro_free(struct octeon_device *octeon_dev, struct ifnet *ifp) 1178325618Ssbruno{ 1179325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1180325618Ssbruno struct lio_droq *droq; 1181325618Ssbruno int q_no; 1182325618Ssbruno int i; 1183325618Ssbruno 1184325618Ssbruno for (i = 0; i < octeon_dev->num_oqs; i++) { 1185325618Ssbruno q_no = lio->linfo.rxpciq[i].s.q_no; 1186325618Ssbruno droq = octeon_dev->droq[q_no]; 1187325618Ssbruno if (droq->lro.ifp) { 1188325618Ssbruno tcp_lro_free(&droq->lro); 1189325618Ssbruno droq->lro.ifp = NULL; 1190325618Ssbruno } 1191325618Ssbruno } 1192325618Ssbruno} 1193325618Ssbruno 1194325618Ssbrunostatic int 1195325618Ssbrunolio_tcp_lro_init(struct octeon_device *octeon_dev, struct ifnet *ifp) 1196325618Ssbruno{ 1197325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1198325618Ssbruno struct lio_droq *droq; 1199325618Ssbruno struct lro_ctrl *lro; 1200325618Ssbruno int i, q_no, ret = 0; 1201325618Ssbruno 1202325618Ssbruno for (i = 0; i < octeon_dev->num_oqs; i++) { 1203325618Ssbruno q_no = lio->linfo.rxpciq[i].s.q_no; 1204325618Ssbruno droq = octeon_dev->droq[q_no]; 1205325618Ssbruno lro = &droq->lro; 1206325618Ssbruno ret = tcp_lro_init(lro); 1207325618Ssbruno if (ret) { 1208325618Ssbruno lio_dev_err(octeon_dev, "LRO Initialization failed ret %d\n", 1209325618Ssbruno ret); 1210325618Ssbruno goto lro_init_failed; 1211325618Ssbruno } 1212325618Ssbruno 1213325618Ssbruno lro->ifp = ifp; 1214325618Ssbruno } 1215325618Ssbruno 1216325618Ssbruno return (ret); 1217325618Ssbruno 1218325618Ssbrunolro_init_failed: 1219325618Ssbruno lio_tcp_lro_free(octeon_dev, ifp); 1220325618Ssbruno 1221325618Ssbruno return (ret); 1222325618Ssbruno} 1223325618Ssbruno 1224325618Ssbrunostatic int 1225325618Ssbrunolio_setup_nic_devices(struct octeon_device *octeon_dev) 1226325618Ssbruno{ 1227325618Ssbruno union octeon_if_cfg if_cfg; 1228325618Ssbruno struct lio *lio = NULL; 1229325618Ssbruno struct ifnet *ifp = NULL; 1230325618Ssbruno struct lio_version *vdata; 1231325618Ssbruno struct lio_soft_command *sc; 1232325618Ssbruno struct lio_if_cfg_context *ctx; 1233325618Ssbruno struct lio_if_cfg_resp *resp; 1234325618Ssbruno struct lio_if_props *props; 1235325618Ssbruno int num_iqueues, num_oqueues, retval; 1236325618Ssbruno unsigned int base_queue; 1237325618Ssbruno unsigned int gmx_port_id; 1238325618Ssbruno uint32_t ctx_size, data_size; 1239325618Ssbruno uint32_t ifidx_or_pfnum, resp_size; 1240325618Ssbruno uint8_t mac[ETHER_HDR_LEN], i, j; 1241325618Ssbruno 1242325618Ssbruno /* This is to handle link status changes */ 1243325618Ssbruno lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC, 1244325618Ssbruno LIO_OPCODE_NIC_INFO, 1245325618Ssbruno lio_link_info, octeon_dev); 1246325618Ssbruno 1247325618Ssbruno for (i = 0; i < octeon_dev->ifcount; i++) { 1248325618Ssbruno resp_size = sizeof(struct lio_if_cfg_resp); 1249325618Ssbruno ctx_size = sizeof(struct lio_if_cfg_context); 1250325618Ssbruno data_size = sizeof(struct lio_version); 1251325618Ssbruno sc = lio_alloc_soft_command(octeon_dev, data_size, resp_size, 1252325618Ssbruno ctx_size); 1253325618Ssbruno if (sc == NULL) 1254325618Ssbruno return (ENOMEM); 1255325618Ssbruno 1256325618Ssbruno resp = (struct lio_if_cfg_resp *)sc->virtrptr; 1257325618Ssbruno ctx = (struct lio_if_cfg_context *)sc->ctxptr; 1258325618Ssbruno vdata = (struct lio_version *)sc->virtdptr; 1259325618Ssbruno 1260325618Ssbruno *((uint64_t *)vdata) = 0; 1261325618Ssbruno vdata->major = htobe16(LIO_BASE_MAJOR_VERSION); 1262325618Ssbruno vdata->minor = htobe16(LIO_BASE_MINOR_VERSION); 1263325618Ssbruno vdata->micro = htobe16(LIO_BASE_MICRO_VERSION); 1264325618Ssbruno 1265325618Ssbruno num_iqueues = octeon_dev->sriov_info.num_pf_rings; 1266325618Ssbruno num_oqueues = octeon_dev->sriov_info.num_pf_rings; 1267325618Ssbruno base_queue = octeon_dev->sriov_info.pf_srn; 1268325618Ssbruno 1269325618Ssbruno gmx_port_id = octeon_dev->pf_num; 1270325618Ssbruno ifidx_or_pfnum = octeon_dev->pf_num; 1271325618Ssbruno 1272325618Ssbruno lio_dev_dbg(octeon_dev, "requesting config for interface %d, iqs %d, oqs %d\n", 1273325618Ssbruno ifidx_or_pfnum, num_iqueues, num_oqueues); 1274325618Ssbruno ctx->cond = 0; 1275325618Ssbruno ctx->octeon_id = lio_get_device_id(octeon_dev); 1276325618Ssbruno 1277325618Ssbruno if_cfg.if_cfg64 = 0; 1278325618Ssbruno if_cfg.s.num_iqueues = num_iqueues; 1279325618Ssbruno if_cfg.s.num_oqueues = num_oqueues; 1280325618Ssbruno if_cfg.s.base_queue = base_queue; 1281325618Ssbruno if_cfg.s.gmx_port_id = gmx_port_id; 1282325618Ssbruno 1283325618Ssbruno sc->iq_no = 0; 1284325618Ssbruno 1285325618Ssbruno lio_prepare_soft_command(octeon_dev, sc, LIO_OPCODE_NIC, 1286325618Ssbruno LIO_OPCODE_NIC_IF_CFG, 0, 1287325618Ssbruno if_cfg.if_cfg64, 0); 1288325618Ssbruno 1289325618Ssbruno sc->callback = lio_if_cfg_callback; 1290325618Ssbruno sc->callback_arg = sc; 1291325618Ssbruno sc->wait_time = 3000; 1292325618Ssbruno 1293325618Ssbruno retval = lio_send_soft_command(octeon_dev, sc); 1294325618Ssbruno if (retval == LIO_IQ_SEND_FAILED) { 1295325618Ssbruno lio_dev_err(octeon_dev, "iq/oq config failed status: %x\n", 1296325618Ssbruno retval); 1297325618Ssbruno /* Soft instr is freed by driver in case of failure. */ 1298325618Ssbruno goto setup_nic_dev_fail; 1299325618Ssbruno } 1300325618Ssbruno 1301325618Ssbruno /* 1302325618Ssbruno * Sleep on a wait queue till the cond flag indicates that the 1303325618Ssbruno * response arrived or timed-out. 1304325618Ssbruno */ 1305325618Ssbruno lio_sleep_cond(octeon_dev, &ctx->cond); 1306325618Ssbruno 1307325618Ssbruno retval = resp->status; 1308325618Ssbruno if (retval) { 1309325618Ssbruno lio_dev_err(octeon_dev, "iq/oq config failed\n"); 1310325618Ssbruno goto setup_nic_dev_fail; 1311325618Ssbruno } 1312325618Ssbruno 1313325618Ssbruno lio_swap_8B_data((uint64_t *)(&resp->cfg_info), 1314325618Ssbruno (sizeof(struct octeon_if_cfg_info)) >> 3); 1315325618Ssbruno 1316325618Ssbruno num_iqueues = bitcount64(resp->cfg_info.iqmask); 1317325618Ssbruno num_oqueues = bitcount64(resp->cfg_info.oqmask); 1318325618Ssbruno 1319325618Ssbruno if (!(num_iqueues) || !(num_oqueues)) { 1320325618Ssbruno lio_dev_err(octeon_dev, 1321325618Ssbruno "Got bad iqueues (%016llX) or oqueues (%016llX) from firmware.\n", 1322325618Ssbruno LIO_CAST64(resp->cfg_info.iqmask), 1323325618Ssbruno LIO_CAST64(resp->cfg_info.oqmask)); 1324325618Ssbruno goto setup_nic_dev_fail; 1325325618Ssbruno } 1326325618Ssbruno 1327325618Ssbruno lio_dev_dbg(octeon_dev, 1328325618Ssbruno "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n", 1329325618Ssbruno i, LIO_CAST64(resp->cfg_info.iqmask), 1330325618Ssbruno LIO_CAST64(resp->cfg_info.oqmask), 1331325618Ssbruno num_iqueues, num_oqueues); 1332325618Ssbruno 1333325618Ssbruno ifp = if_alloc(IFT_ETHER); 1334325618Ssbruno 1335325618Ssbruno if (ifp == NULL) { 1336325618Ssbruno lio_dev_err(octeon_dev, "Device allocation failed\n"); 1337325618Ssbruno goto setup_nic_dev_fail; 1338325618Ssbruno } 1339325618Ssbruno 1340325618Ssbruno lio = malloc(sizeof(struct lio), M_DEVBUF, M_NOWAIT | M_ZERO); 1341325618Ssbruno 1342325618Ssbruno if (lio == NULL) { 1343325618Ssbruno lio_dev_err(octeon_dev, "Lio allocation failed\n"); 1344325618Ssbruno goto setup_nic_dev_fail; 1345325618Ssbruno } 1346325618Ssbruno 1347325618Ssbruno if_setsoftc(ifp, lio); 1348325618Ssbruno 1349325618Ssbruno ifp->if_hw_tsomax = LIO_MAX_FRAME_SIZE; 1350325618Ssbruno ifp->if_hw_tsomaxsegcount = LIO_MAX_SG; 1351325618Ssbruno ifp->if_hw_tsomaxsegsize = PAGE_SIZE; 1352325618Ssbruno 1353325618Ssbruno lio->ifidx = ifidx_or_pfnum; 1354325618Ssbruno 1355325618Ssbruno props = &octeon_dev->props; 1356325618Ssbruno props->gmxport = resp->cfg_info.linfo.gmxport; 1357325618Ssbruno props->ifp = ifp; 1358325618Ssbruno 1359325618Ssbruno lio->linfo.num_rxpciq = num_oqueues; 1360325618Ssbruno lio->linfo.num_txpciq = num_iqueues; 1361325618Ssbruno for (j = 0; j < num_oqueues; j++) { 1362325618Ssbruno lio->linfo.rxpciq[j].rxpciq64 = 1363325618Ssbruno resp->cfg_info.linfo.rxpciq[j].rxpciq64; 1364325618Ssbruno } 1365325618Ssbruno 1366325618Ssbruno for (j = 0; j < num_iqueues; j++) { 1367325618Ssbruno lio->linfo.txpciq[j].txpciq64 = 1368325618Ssbruno resp->cfg_info.linfo.txpciq[j].txpciq64; 1369325618Ssbruno } 1370325618Ssbruno 1371325618Ssbruno lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; 1372325618Ssbruno lio->linfo.gmxport = resp->cfg_info.linfo.gmxport; 1373325618Ssbruno lio->linfo.link.link_status64 = 1374325618Ssbruno resp->cfg_info.linfo.link.link_status64; 1375325618Ssbruno 1376325618Ssbruno /* 1377325618Ssbruno * Point to the properties for octeon device to which this 1378325618Ssbruno * interface belongs. 1379325618Ssbruno */ 1380325618Ssbruno lio->oct_dev = octeon_dev; 1381325618Ssbruno lio->ifp = ifp; 1382325618Ssbruno 1383325618Ssbruno lio_dev_dbg(octeon_dev, "if%d gmx: %d hw_addr: 0x%llx\n", i, 1384325618Ssbruno lio->linfo.gmxport, LIO_CAST64(lio->linfo.hw_addr)); 1385325618Ssbruno lio_init_ifnet(lio); 1386325618Ssbruno /* 64-bit swap required on LE machines */ 1387325618Ssbruno lio_swap_8B_data(&lio->linfo.hw_addr, 1); 1388325618Ssbruno for (j = 0; j < 6; j++) 1389325618Ssbruno mac[j] = *((uint8_t *)( 1390325618Ssbruno ((uint8_t *)&lio->linfo.hw_addr) + 2 + j)); 1391325618Ssbruno 1392325618Ssbruno ether_ifattach(ifp, mac); 1393325618Ssbruno 1394325618Ssbruno /* 1395325618Ssbruno * By default all interfaces on a single Octeon uses the same 1396325618Ssbruno * tx and rx queues 1397325618Ssbruno */ 1398325618Ssbruno lio->txq = lio->linfo.txpciq[0].s.q_no; 1399325618Ssbruno lio->rxq = lio->linfo.rxpciq[0].s.q_no; 1400325618Ssbruno if (lio_setup_io_queues(octeon_dev, i, lio->linfo.num_txpciq, 1401325618Ssbruno lio->linfo.num_rxpciq)) { 1402325618Ssbruno lio_dev_err(octeon_dev, "I/O queues creation failed\n"); 1403325618Ssbruno goto setup_nic_dev_fail; 1404325618Ssbruno } 1405325618Ssbruno 1406325618Ssbruno lio_ifstate_set(lio, LIO_IFSTATE_DROQ_OPS); 1407325618Ssbruno 1408325618Ssbruno lio->tx_qsize = lio_get_tx_qsize(octeon_dev, lio->txq); 1409325618Ssbruno lio->rx_qsize = lio_get_rx_qsize(octeon_dev, lio->rxq); 1410325618Ssbruno 1411325618Ssbruno if (lio_setup_glists(octeon_dev, lio, num_iqueues)) { 1412325618Ssbruno lio_dev_err(octeon_dev, "Gather list allocation failed\n"); 1413325618Ssbruno goto setup_nic_dev_fail; 1414325618Ssbruno } 1415325618Ssbruno 1416325618Ssbruno if ((lio_hwlro == 0) && lio_tcp_lro_init(octeon_dev, ifp)) 1417325618Ssbruno goto setup_nic_dev_fail; 1418325618Ssbruno 1419325618Ssbruno if (lio_hwlro && 1420325618Ssbruno (if_getcapenable(ifp) & IFCAP_LRO) && 1421325618Ssbruno (if_getcapenable(ifp) & IFCAP_RXCSUM) && 1422325618Ssbruno (if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6)) 1423325618Ssbruno lio_set_feature(ifp, LIO_CMD_LRO_ENABLE, 1424325618Ssbruno LIO_LROIPV4 | LIO_LROIPV6); 1425325618Ssbruno 1426325618Ssbruno if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)) 1427325618Ssbruno lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 1); 1428325618Ssbruno else 1429325618Ssbruno lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 0); 1430325618Ssbruno 1431325618Ssbruno if (lio_setup_rx_oom_poll_fn(ifp)) 1432325618Ssbruno goto setup_nic_dev_fail; 1433325618Ssbruno 1434325618Ssbruno lio_dev_dbg(octeon_dev, "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n", 1435325618Ssbruno i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); 1436325618Ssbruno lio->link_changes++; 1437325618Ssbruno 1438325618Ssbruno lio_ifstate_set(lio, LIO_IFSTATE_REGISTERED); 1439325618Ssbruno 1440325618Ssbruno /* 1441325618Ssbruno * Sending command to firmware to enable Rx checksum offload 1442325618Ssbruno * by default at the time of setup of Liquidio driver for 1443325618Ssbruno * this device 1444325618Ssbruno */ 1445325618Ssbruno lio_set_rxcsum_command(ifp, LIO_CMD_TNL_RX_CSUM_CTL, 1446325618Ssbruno LIO_CMD_RXCSUM_ENABLE); 1447325618Ssbruno lio_set_feature(ifp, LIO_CMD_TNL_TX_CSUM_CTL, 1448325618Ssbruno LIO_CMD_TXCSUM_ENABLE); 1449325618Ssbruno 1450325618Ssbruno#ifdef RSS 1451325618Ssbruno if (lio_rss) { 1452325618Ssbruno if (lio_send_rss_param(lio)) 1453325618Ssbruno goto setup_nic_dev_fail; 1454325618Ssbruno } else 1455325618Ssbruno#endif /* RSS */ 1456325618Ssbruno 1457325618Ssbruno lio_set_feature(ifp, LIO_CMD_SET_FNV, 1458325618Ssbruno LIO_CMD_FNV_ENABLE); 1459325618Ssbruno 1460325618Ssbruno lio_dev_dbg(octeon_dev, "NIC ifidx:%d Setup successful\n", i); 1461325618Ssbruno 1462325618Ssbruno lio_free_soft_command(octeon_dev, sc); 1463325618Ssbruno lio->vlan_attach = 1464325618Ssbruno EVENTHANDLER_REGISTER(vlan_config, 1465325618Ssbruno lio_vlan_rx_add_vid, lio, 1466325618Ssbruno EVENTHANDLER_PRI_FIRST); 1467325618Ssbruno lio->vlan_detach = 1468325618Ssbruno EVENTHANDLER_REGISTER(vlan_unconfig, 1469325618Ssbruno lio_vlan_rx_kill_vid, lio, 1470325618Ssbruno EVENTHANDLER_PRI_FIRST); 1471325618Ssbruno 1472325618Ssbruno /* Update stats periodically */ 1473325618Ssbruno callout_init(&lio->stats_timer, 0); 1474325618Ssbruno lio->stats_interval = LIO_DEFAULT_STATS_INTERVAL; 1475325618Ssbruno 1476325618Ssbruno lio_add_hw_stats(lio); 1477325618Ssbruno } 1478325618Ssbruno 1479325618Ssbruno return (0); 1480325618Ssbruno 1481325618Ssbrunosetup_nic_dev_fail: 1482325618Ssbruno 1483325618Ssbruno lio_free_soft_command(octeon_dev, sc); 1484325618Ssbruno 1485325618Ssbruno while (i--) { 1486325618Ssbruno lio_dev_err(octeon_dev, "NIC ifidx:%d Setup failed\n", i); 1487325618Ssbruno lio_destroy_nic_device(octeon_dev, i); 1488325618Ssbruno } 1489325618Ssbruno 1490325618Ssbruno return (ENODEV); 1491325618Ssbruno} 1492325618Ssbruno 1493325618Ssbrunostatic int 1494325618Ssbrunolio_link_info(struct lio_recv_info *recv_info, void *ptr) 1495325618Ssbruno{ 1496325618Ssbruno struct octeon_device *oct = (struct octeon_device *)ptr; 1497325618Ssbruno struct lio_recv_pkt *recv_pkt = recv_info->recv_pkt; 1498325618Ssbruno union octeon_link_status *ls; 1499325618Ssbruno int gmxport = 0, i; 1500325618Ssbruno 1501325618Ssbruno lio_dev_dbg(oct, "%s Called\n", __func__); 1502325618Ssbruno if (recv_pkt->buffer_size[0] != (sizeof(*ls) + LIO_DROQ_INFO_SIZE)) { 1503325618Ssbruno lio_dev_err(oct, "Malformed NIC_INFO, len=%d, ifidx=%d\n", 1504325618Ssbruno recv_pkt->buffer_size[0], 1505325618Ssbruno recv_pkt->rh.r_nic_info.gmxport); 1506325618Ssbruno goto nic_info_err; 1507325618Ssbruno } 1508325618Ssbruno gmxport = recv_pkt->rh.r_nic_info.gmxport; 1509325618Ssbruno ls = (union octeon_link_status *)(recv_pkt->buffer_ptr[0]->m_data + 1510325618Ssbruno LIO_DROQ_INFO_SIZE); 1511325618Ssbruno lio_swap_8B_data((uint64_t *)ls, 1512325618Ssbruno (sizeof(union octeon_link_status)) >> 3); 1513325618Ssbruno 1514325618Ssbruno if (oct->props.gmxport == gmxport) 1515325618Ssbruno lio_update_link_status(oct->props.ifp, ls); 1516325618Ssbruno 1517325618Ssbrunonic_info_err: 1518325618Ssbruno for (i = 0; i < recv_pkt->buffer_count; i++) 1519325618Ssbruno lio_recv_buffer_free(recv_pkt->buffer_ptr[i]); 1520325618Ssbruno 1521325618Ssbruno lio_free_recv_info(recv_info); 1522325618Ssbruno return (0); 1523325618Ssbruno} 1524325618Ssbruno 1525325618Ssbrunovoid 1526325618Ssbrunolio_free_mbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo) 1527325618Ssbruno{ 1528325618Ssbruno 1529325618Ssbruno bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE); 1530325618Ssbruno bus_dmamap_unload(iq->txtag, finfo->map); 1531325618Ssbruno m_freem(finfo->mb); 1532325618Ssbruno} 1533325618Ssbruno 1534325618Ssbrunovoid 1535325618Ssbrunolio_free_sgmbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo) 1536325618Ssbruno{ 1537325618Ssbruno struct lio_gather *g; 1538325618Ssbruno struct octeon_device *oct; 1539325618Ssbruno struct lio *lio; 1540325618Ssbruno int iq_no; 1541325618Ssbruno 1542325618Ssbruno g = finfo->g; 1543325618Ssbruno iq_no = iq->txpciq.s.q_no; 1544325618Ssbruno oct = iq->oct_dev; 1545325618Ssbruno lio = if_getsoftc(oct->props.ifp); 1546325618Ssbruno 1547325618Ssbruno mtx_lock(&lio->glist_lock[iq_no]); 1548325618Ssbruno STAILQ_INSERT_TAIL(&lio->ghead[iq_no], &g->node, entries); 1549325618Ssbruno mtx_unlock(&lio->glist_lock[iq_no]); 1550325618Ssbruno 1551325618Ssbruno bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE); 1552325618Ssbruno bus_dmamap_unload(iq->txtag, finfo->map); 1553325618Ssbruno m_freem(finfo->mb); 1554325618Ssbruno} 1555325618Ssbruno 1556325618Ssbrunostatic void 1557325618Ssbrunolio_if_cfg_callback(struct octeon_device *oct, uint32_t status, void *buf) 1558325618Ssbruno{ 1559325618Ssbruno struct lio_soft_command *sc = (struct lio_soft_command *)buf; 1560325618Ssbruno struct lio_if_cfg_resp *resp; 1561325618Ssbruno struct lio_if_cfg_context *ctx; 1562325618Ssbruno 1563325618Ssbruno resp = (struct lio_if_cfg_resp *)sc->virtrptr; 1564325618Ssbruno ctx = (struct lio_if_cfg_context *)sc->ctxptr; 1565325618Ssbruno 1566325618Ssbruno oct = lio_get_device(ctx->octeon_id); 1567325618Ssbruno if (resp->status) 1568325618Ssbruno lio_dev_err(oct, "nic if cfg instruction failed. Status: %llx (0x%08x)\n", 1569325618Ssbruno LIO_CAST64(resp->status), status); 1570325618Ssbruno ctx->cond = 1; 1571325618Ssbruno 1572325618Ssbruno snprintf(oct->fw_info.lio_firmware_version, 32, "%s", 1573325618Ssbruno resp->cfg_info.lio_firmware_version); 1574325618Ssbruno 1575325618Ssbruno /* 1576325618Ssbruno * This barrier is required to be sure that the response has been 1577325618Ssbruno * written fully before waking up the handler 1578325618Ssbruno */ 1579325618Ssbruno wmb(); 1580325618Ssbruno} 1581325618Ssbruno 1582325618Ssbrunostatic int 1583325618Ssbrunolio_is_mac_changed(uint8_t *new, uint8_t *old) 1584325618Ssbruno{ 1585325618Ssbruno 1586325618Ssbruno return ((new[0] != old[0]) || (new[1] != old[1]) || 1587325618Ssbruno (new[2] != old[2]) || (new[3] != old[3]) || 1588325618Ssbruno (new[4] != old[4]) || (new[5] != old[5])); 1589325618Ssbruno} 1590325618Ssbruno 1591325618Ssbrunovoid 1592325618Ssbrunolio_open(void *arg) 1593325618Ssbruno{ 1594325618Ssbruno struct lio *lio = arg; 1595325618Ssbruno struct ifnet *ifp = lio->ifp; 1596325618Ssbruno struct octeon_device *oct = lio->oct_dev; 1597325618Ssbruno uint8_t *mac_new, mac_old[ETHER_HDR_LEN]; 1598325618Ssbruno int ret = 0; 1599325618Ssbruno 1600325618Ssbruno lio_ifstate_set(lio, LIO_IFSTATE_RUNNING); 1601325618Ssbruno 1602325618Ssbruno /* Ready for link status updates */ 1603325618Ssbruno lio->intf_open = 1; 1604325618Ssbruno 1605325618Ssbruno lio_dev_info(oct, "Interface Open, ready for traffic\n"); 1606325618Ssbruno 1607325618Ssbruno /* tell Octeon to start forwarding packets to host */ 1608325618Ssbruno lio_send_rx_ctrl_cmd(lio, 1); 1609325618Ssbruno 1610325618Ssbruno mac_new = IF_LLADDR(ifp); 1611325618Ssbruno memcpy(mac_old, ((uint8_t *)&lio->linfo.hw_addr) + 2, ETHER_HDR_LEN); 1612325618Ssbruno 1613325618Ssbruno if (lio_is_mac_changed(mac_new, mac_old)) { 1614325618Ssbruno ret = lio_set_mac(ifp, mac_new); 1615325618Ssbruno if (ret) 1616325618Ssbruno lio_dev_err(oct, "MAC change failed, error: %d\n", ret); 1617325618Ssbruno } 1618325618Ssbruno 1619325618Ssbruno /* Now inform the stack we're ready */ 1620325618Ssbruno if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 1621325618Ssbruno 1622325618Ssbruno lio_dev_info(oct, "Interface is opened\n"); 1623325618Ssbruno} 1624325618Ssbruno 1625325618Ssbrunostatic int 1626325618Ssbrunolio_set_rxcsum_command(struct ifnet *ifp, int command, uint8_t rx_cmd) 1627325618Ssbruno{ 1628325618Ssbruno struct lio_ctrl_pkt nctrl; 1629325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1630325618Ssbruno struct octeon_device *oct = lio->oct_dev; 1631325618Ssbruno int ret = 0; 1632325618Ssbruno 1633325618Ssbruno nctrl.ncmd.cmd64 = 0; 1634325618Ssbruno nctrl.ncmd.s.cmd = command; 1635325618Ssbruno nctrl.ncmd.s.param1 = rx_cmd; 1636325618Ssbruno nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1637325618Ssbruno nctrl.wait_time = 100; 1638325618Ssbruno nctrl.lio = lio; 1639325618Ssbruno nctrl.cb_fn = lio_ctrl_cmd_completion; 1640325618Ssbruno 1641325618Ssbruno ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl); 1642325618Ssbruno if (ret < 0) { 1643325618Ssbruno lio_dev_err(oct, "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n", 1644325618Ssbruno ret); 1645325618Ssbruno } 1646325618Ssbruno 1647325618Ssbruno return (ret); 1648325618Ssbruno} 1649325618Ssbruno 1650325618Ssbrunostatic int 1651325618Ssbrunolio_stop_nic_module(struct octeon_device *oct) 1652325618Ssbruno{ 1653325618Ssbruno int i, j; 1654325618Ssbruno struct lio *lio; 1655325618Ssbruno 1656325618Ssbruno lio_dev_dbg(oct, "Stopping network interfaces\n"); 1657325618Ssbruno if (!oct->ifcount) { 1658325618Ssbruno lio_dev_err(oct, "Init for Octeon was not completed\n"); 1659325618Ssbruno return (1); 1660325618Ssbruno } 1661325618Ssbruno 1662325618Ssbruno mtx_lock(&oct->cmd_resp_wqlock); 1663325618Ssbruno oct->cmd_resp_state = LIO_DRV_OFFLINE; 1664325618Ssbruno mtx_unlock(&oct->cmd_resp_wqlock); 1665325618Ssbruno 1666325618Ssbruno for (i = 0; i < oct->ifcount; i++) { 1667325618Ssbruno lio = if_getsoftc(oct->props.ifp); 1668325618Ssbruno for (j = 0; j < oct->num_oqs; j++) 1669325618Ssbruno lio_unregister_droq_ops(oct, 1670325618Ssbruno lio->linfo.rxpciq[j].s.q_no); 1671325618Ssbruno } 1672325618Ssbruno 1673325618Ssbruno callout_drain(&lio->stats_timer); 1674325618Ssbruno 1675325618Ssbruno for (i = 0; i < oct->ifcount; i++) 1676325618Ssbruno lio_destroy_nic_device(oct, i); 1677325618Ssbruno 1678325618Ssbruno lio_dev_dbg(oct, "Network interface stopped\n"); 1679325618Ssbruno 1680325618Ssbruno return (0); 1681325618Ssbruno} 1682325618Ssbruno 1683325618Ssbrunostatic void 1684325618Ssbrunolio_delete_glists(struct octeon_device *oct, struct lio *lio) 1685325618Ssbruno{ 1686325618Ssbruno struct lio_gather *g; 1687325618Ssbruno int i; 1688325618Ssbruno 1689325618Ssbruno if (lio->glist_lock != NULL) { 1690325618Ssbruno free((void *)lio->glist_lock, M_DEVBUF); 1691325618Ssbruno lio->glist_lock = NULL; 1692325618Ssbruno } 1693325618Ssbruno 1694325618Ssbruno if (lio->ghead == NULL) 1695325618Ssbruno return; 1696325618Ssbruno 1697325618Ssbruno for (i = 0; i < lio->linfo.num_txpciq; i++) { 1698325618Ssbruno do { 1699325618Ssbruno g = (struct lio_gather *) 1700325618Ssbruno lio_delete_first_node(&lio->ghead[i]); 1701325618Ssbruno free(g, M_DEVBUF); 1702325618Ssbruno } while (g); 1703325618Ssbruno 1704325618Ssbruno if ((lio->glists_virt_base != NULL) && 1705325618Ssbruno (lio->glists_virt_base[i] != NULL)) { 1706325618Ssbruno lio_dma_free(lio->glist_entry_size * lio->tx_qsize, 1707325618Ssbruno lio->glists_virt_base[i]); 1708325618Ssbruno } 1709325618Ssbruno } 1710325618Ssbruno 1711325618Ssbruno free(lio->glists_virt_base, M_DEVBUF); 1712325618Ssbruno lio->glists_virt_base = NULL; 1713325618Ssbruno 1714325618Ssbruno free(lio->glists_dma_base, M_DEVBUF); 1715325618Ssbruno lio->glists_dma_base = NULL; 1716325618Ssbruno 1717325618Ssbruno free(lio->ghead, M_DEVBUF); 1718325618Ssbruno lio->ghead = NULL; 1719325618Ssbruno} 1720325618Ssbruno 1721325618Ssbrunostatic int 1722325618Ssbrunolio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs) 1723325618Ssbruno{ 1724325618Ssbruno struct lio_gather *g; 1725325618Ssbruno int i, j; 1726325618Ssbruno 1727325618Ssbruno lio->glist_lock = malloc(num_iqs * sizeof(*lio->glist_lock), M_DEVBUF, 1728325618Ssbruno M_NOWAIT | M_ZERO); 1729325618Ssbruno if (lio->glist_lock == NULL) 1730325618Ssbruno return (1); 1731325618Ssbruno 1732325618Ssbruno lio->ghead = malloc(num_iqs * sizeof(*lio->ghead), M_DEVBUF, 1733325618Ssbruno M_NOWAIT | M_ZERO); 1734325618Ssbruno if (lio->ghead == NULL) { 1735325618Ssbruno free((void *)lio->glist_lock, M_DEVBUF); 1736325618Ssbruno lio->glist_lock = NULL; 1737325618Ssbruno return (1); 1738325618Ssbruno } 1739325618Ssbruno 1740325618Ssbruno lio->glist_entry_size = ROUNDUP8((ROUNDUP4(LIO_MAX_SG) >> 2) * 1741325618Ssbruno LIO_SG_ENTRY_SIZE); 1742325618Ssbruno /* 1743325618Ssbruno * allocate memory to store virtual and dma base address of 1744325618Ssbruno * per glist consistent memory 1745325618Ssbruno */ 1746325618Ssbruno lio->glists_virt_base = malloc(num_iqs * sizeof(void *), M_DEVBUF, 1747325618Ssbruno M_NOWAIT | M_ZERO); 1748325618Ssbruno lio->glists_dma_base = malloc(num_iqs * sizeof(vm_paddr_t), M_DEVBUF, 1749325618Ssbruno M_NOWAIT | M_ZERO); 1750325618Ssbruno if ((lio->glists_virt_base == NULL) || (lio->glists_dma_base == NULL)) { 1751325618Ssbruno lio_delete_glists(oct, lio); 1752325618Ssbruno return (1); 1753325618Ssbruno } 1754325618Ssbruno 1755325618Ssbruno for (i = 0; i < num_iqs; i++) { 1756325618Ssbruno mtx_init(&lio->glist_lock[i], "glist_lock", NULL, MTX_DEF); 1757325618Ssbruno 1758325618Ssbruno STAILQ_INIT(&lio->ghead[i]); 1759325618Ssbruno 1760325618Ssbruno lio->glists_virt_base[i] = 1761325618Ssbruno lio_dma_alloc(lio->glist_entry_size * lio->tx_qsize, 1762325618Ssbruno (vm_paddr_t *)&lio->glists_dma_base[i]); 1763325618Ssbruno if (lio->glists_virt_base[i] == NULL) { 1764325618Ssbruno lio_delete_glists(oct, lio); 1765325618Ssbruno return (1); 1766325618Ssbruno } 1767325618Ssbruno 1768325618Ssbruno for (j = 0; j < lio->tx_qsize; j++) { 1769325618Ssbruno g = malloc(sizeof(*g), M_DEVBUF, M_NOWAIT | M_ZERO); 1770325618Ssbruno if (g == NULL) 1771325618Ssbruno break; 1772325618Ssbruno 1773335293Sdim g->sg = (struct lio_sg_entry *)(uintptr_t) 1774335293Sdim ((uint64_t)(uintptr_t)lio->glists_virt_base[i] + 1775325618Ssbruno (j * lio->glist_entry_size)); 1776325618Ssbruno g->sg_dma_ptr = (uint64_t)lio->glists_dma_base[i] + 1777325618Ssbruno (j * lio->glist_entry_size); 1778325618Ssbruno STAILQ_INSERT_TAIL(&lio->ghead[i], &g->node, entries); 1779325618Ssbruno } 1780325618Ssbruno 1781325618Ssbruno if (j != lio->tx_qsize) { 1782325618Ssbruno lio_delete_glists(oct, lio); 1783325618Ssbruno return (1); 1784325618Ssbruno } 1785325618Ssbruno } 1786325618Ssbruno 1787325618Ssbruno return (0); 1788325618Ssbruno} 1789325618Ssbruno 1790325618Ssbrunovoid 1791325618Ssbrunolio_stop(struct ifnet *ifp) 1792325618Ssbruno{ 1793325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1794325618Ssbruno struct octeon_device *oct = lio->oct_dev; 1795325618Ssbruno 1796325618Ssbruno lio_ifstate_reset(lio, LIO_IFSTATE_RUNNING); 1797325618Ssbruno if_link_state_change(ifp, LINK_STATE_DOWN); 1798325618Ssbruno 1799325618Ssbruno lio->intf_open = 0; 1800325618Ssbruno lio->linfo.link.s.link_up = 0; 1801325618Ssbruno lio->link_changes++; 1802325618Ssbruno 1803325618Ssbruno lio_send_rx_ctrl_cmd(lio, 0); 1804325618Ssbruno 1805325618Ssbruno /* Tell the stack that the interface is no longer active */ 1806325618Ssbruno if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 1807325618Ssbruno 1808325618Ssbruno lio_dev_info(oct, "Interface is stopped\n"); 1809325618Ssbruno} 1810325618Ssbruno 1811325618Ssbrunostatic void 1812325618Ssbrunolio_check_rx_oom_status(struct lio *lio) 1813325618Ssbruno{ 1814325618Ssbruno struct lio_droq *droq; 1815325618Ssbruno struct octeon_device *oct = lio->oct_dev; 1816325618Ssbruno int desc_refilled; 1817325618Ssbruno int q, q_no = 0; 1818325618Ssbruno 1819325618Ssbruno for (q = 0; q < oct->num_oqs; q++) { 1820325618Ssbruno q_no = lio->linfo.rxpciq[q].s.q_no; 1821325618Ssbruno droq = oct->droq[q_no]; 1822325618Ssbruno if (droq == NULL) 1823325618Ssbruno continue; 1824325618Ssbruno if (lio_read_csr32(oct, droq->pkts_credit_reg) <= 0x40) { 1825325618Ssbruno mtx_lock(&droq->lock); 1826325618Ssbruno desc_refilled = lio_droq_refill(oct, droq); 1827325618Ssbruno /* 1828325618Ssbruno * Flush the droq descriptor data to memory to be sure 1829325618Ssbruno * that when we update the credits the data in memory 1830325618Ssbruno * is accurate. 1831325618Ssbruno */ 1832325618Ssbruno wmb(); 1833325618Ssbruno lio_write_csr32(oct, droq->pkts_credit_reg, 1834325618Ssbruno desc_refilled); 1835325618Ssbruno /* make sure mmio write completes */ 1836325618Ssbruno __compiler_membar(); 1837325618Ssbruno mtx_unlock(&droq->lock); 1838325618Ssbruno } 1839325618Ssbruno } 1840325618Ssbruno} 1841325618Ssbruno 1842325618Ssbrunostatic void 1843325618Ssbrunolio_poll_check_rx_oom_status(void *arg, int pending __unused) 1844325618Ssbruno{ 1845325618Ssbruno struct lio_tq *rx_status_tq = arg; 1846325618Ssbruno struct lio *lio = rx_status_tq->ctxptr; 1847325618Ssbruno 1848325618Ssbruno if (lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) 1849325618Ssbruno lio_check_rx_oom_status(lio); 1850325618Ssbruno 1851325618Ssbruno taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work, 1852325618Ssbruno lio_ms_to_ticks(50)); 1853325618Ssbruno} 1854325618Ssbruno 1855325618Ssbrunostatic int 1856325618Ssbrunolio_setup_rx_oom_poll_fn(struct ifnet *ifp) 1857325618Ssbruno{ 1858325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1859325618Ssbruno struct octeon_device *oct = lio->oct_dev; 1860325618Ssbruno struct lio_tq *rx_status_tq; 1861325618Ssbruno 1862325618Ssbruno rx_status_tq = &lio->rx_status_tq; 1863325618Ssbruno 1864325618Ssbruno rx_status_tq->tq = taskqueue_create("lio_rx_oom_status", M_WAITOK, 1865325618Ssbruno taskqueue_thread_enqueue, 1866325618Ssbruno &rx_status_tq->tq); 1867325618Ssbruno if (rx_status_tq->tq == NULL) { 1868325618Ssbruno lio_dev_err(oct, "unable to create lio rx oom status tq\n"); 1869325618Ssbruno return (-1); 1870325618Ssbruno } 1871325618Ssbruno 1872325618Ssbruno TIMEOUT_TASK_INIT(rx_status_tq->tq, &rx_status_tq->work, 0, 1873325618Ssbruno lio_poll_check_rx_oom_status, (void *)rx_status_tq); 1874325618Ssbruno 1875325618Ssbruno rx_status_tq->ctxptr = lio; 1876325618Ssbruno 1877325618Ssbruno taskqueue_start_threads(&rx_status_tq->tq, 1, PI_NET, 1878325618Ssbruno "lio%d_rx_oom_status", 1879325618Ssbruno oct->octeon_id); 1880325618Ssbruno 1881325618Ssbruno taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work, 1882325618Ssbruno lio_ms_to_ticks(50)); 1883325618Ssbruno 1884325618Ssbruno return (0); 1885325618Ssbruno} 1886325618Ssbruno 1887325618Ssbrunostatic void 1888325618Ssbrunolio_cleanup_rx_oom_poll_fn(struct ifnet *ifp) 1889325618Ssbruno{ 1890325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1891325618Ssbruno 1892325618Ssbruno if (lio->rx_status_tq.tq != NULL) { 1893325618Ssbruno while (taskqueue_cancel_timeout(lio->rx_status_tq.tq, 1894325618Ssbruno &lio->rx_status_tq.work, NULL)) 1895325618Ssbruno taskqueue_drain_timeout(lio->rx_status_tq.tq, 1896325618Ssbruno &lio->rx_status_tq.work); 1897325618Ssbruno 1898325618Ssbruno taskqueue_free(lio->rx_status_tq.tq); 1899325618Ssbruno 1900325618Ssbruno lio->rx_status_tq.tq = NULL; 1901325618Ssbruno } 1902325618Ssbruno} 1903325618Ssbruno 1904325618Ssbrunostatic void 1905325618Ssbrunolio_destroy_nic_device(struct octeon_device *oct, int ifidx) 1906325618Ssbruno{ 1907325618Ssbruno struct ifnet *ifp = oct->props.ifp; 1908325618Ssbruno struct lio *lio; 1909325618Ssbruno 1910325618Ssbruno if (ifp == NULL) { 1911325618Ssbruno lio_dev_err(oct, "%s No ifp ptr for index %d\n", 1912325618Ssbruno __func__, ifidx); 1913325618Ssbruno return; 1914325618Ssbruno } 1915325618Ssbruno 1916325618Ssbruno lio = if_getsoftc(ifp); 1917325618Ssbruno 1918325618Ssbruno lio_ifstate_set(lio, LIO_IFSTATE_DETACH); 1919325618Ssbruno 1920325618Ssbruno lio_dev_dbg(oct, "NIC device cleanup\n"); 1921325618Ssbruno 1922325618Ssbruno if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_RUNNING) 1923325618Ssbruno lio_stop(ifp); 1924325618Ssbruno 1925325618Ssbruno if (lio_wait_for_pending_requests(oct)) 1926325618Ssbruno lio_dev_err(oct, "There were pending requests\n"); 1927325618Ssbruno 1928325618Ssbruno if (lio_wait_for_instr_fetch(oct)) 1929325618Ssbruno lio_dev_err(oct, "IQ had pending instructions\n"); 1930325618Ssbruno 1931325618Ssbruno if (lio_wait_for_oq_pkts(oct)) 1932325618Ssbruno lio_dev_err(oct, "OQ had pending packets\n"); 1933325618Ssbruno 1934325618Ssbruno if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_REGISTERED) 1935325618Ssbruno ether_ifdetach(ifp); 1936325618Ssbruno 1937325618Ssbruno lio_tcp_lro_free(oct, ifp); 1938325618Ssbruno 1939325618Ssbruno lio_cleanup_rx_oom_poll_fn(ifp); 1940325618Ssbruno 1941325618Ssbruno lio_delete_glists(oct, lio); 1942325618Ssbruno 1943325618Ssbruno EVENTHANDLER_DEREGISTER(vlan_config, lio->vlan_attach); 1944325618Ssbruno EVENTHANDLER_DEREGISTER(vlan_unconfig, lio->vlan_detach); 1945325618Ssbruno 1946325618Ssbruno free(lio, M_DEVBUF); 1947325618Ssbruno 1948325618Ssbruno if_free(ifp); 1949325618Ssbruno 1950325618Ssbruno oct->props.gmxport = -1; 1951325618Ssbruno 1952325618Ssbruno oct->props.ifp = NULL; 1953325618Ssbruno} 1954325618Ssbruno 1955325618Ssbrunostatic void 1956325618Ssbrunoprint_link_info(struct ifnet *ifp) 1957325618Ssbruno{ 1958325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1959325618Ssbruno 1960325618Ssbruno if (!lio_ifstate_check(lio, LIO_IFSTATE_RESETTING) && 1961325618Ssbruno lio_ifstate_check(lio, LIO_IFSTATE_REGISTERED)) { 1962325618Ssbruno struct octeon_link_info *linfo = &lio->linfo; 1963325618Ssbruno 1964325618Ssbruno if (linfo->link.s.link_up) { 1965325618Ssbruno lio_dev_info(lio->oct_dev, "%d Mbps %s Duplex UP\n", 1966325618Ssbruno linfo->link.s.speed, 1967325618Ssbruno (linfo->link.s.duplex) ? "Full" : "Half"); 1968325618Ssbruno } else { 1969325618Ssbruno lio_dev_info(lio->oct_dev, "Link Down\n"); 1970325618Ssbruno } 1971325618Ssbruno } 1972325618Ssbruno} 1973325618Ssbruno 1974325618Ssbrunostatic inline void 1975325618Ssbrunolio_update_link_status(struct ifnet *ifp, union octeon_link_status *ls) 1976325618Ssbruno{ 1977325618Ssbruno struct lio *lio = if_getsoftc(ifp); 1978325618Ssbruno int changed = (lio->linfo.link.link_status64 != ls->link_status64); 1979325618Ssbruno 1980325618Ssbruno lio->linfo.link.link_status64 = ls->link_status64; 1981325618Ssbruno 1982325618Ssbruno if ((lio->intf_open) && (changed)) { 1983325618Ssbruno print_link_info(ifp); 1984325618Ssbruno lio->link_changes++; 1985325618Ssbruno if (lio->linfo.link.s.link_up) 1986325618Ssbruno if_link_state_change(ifp, LINK_STATE_UP); 1987325618Ssbruno else 1988325618Ssbruno if_link_state_change(ifp, LINK_STATE_DOWN); 1989325618Ssbruno } 1990325618Ssbruno} 1991325618Ssbruno 1992325618Ssbruno/* 1993325618Ssbruno * \brief Callback for rx ctrl 1994325618Ssbruno * @param status status of request 1995325618Ssbruno * @param buf pointer to resp structure 1996325618Ssbruno */ 1997325618Ssbrunostatic void 1998325618Ssbrunolio_rx_ctl_callback(struct octeon_device *oct, uint32_t status, void *buf) 1999325618Ssbruno{ 2000325618Ssbruno struct lio_soft_command *sc = (struct lio_soft_command *)buf; 2001325618Ssbruno struct lio_rx_ctl_context *ctx; 2002325618Ssbruno 2003325618Ssbruno ctx = (struct lio_rx_ctl_context *)sc->ctxptr; 2004325618Ssbruno 2005325618Ssbruno oct = lio_get_device(ctx->octeon_id); 2006325618Ssbruno if (status) 2007325618Ssbruno lio_dev_err(oct, "rx ctl instruction failed. Status: %llx\n", 2008325618Ssbruno LIO_CAST64(status)); 2009325618Ssbruno ctx->cond = 1; 2010325618Ssbruno 2011325618Ssbruno /* 2012325618Ssbruno * This barrier is required to be sure that the response has been 2013325618Ssbruno * written fully before waking up the handler 2014325618Ssbruno */ 2015325618Ssbruno wmb(); 2016325618Ssbruno} 2017325618Ssbruno 2018325618Ssbrunostatic void 2019325618Ssbrunolio_send_rx_ctrl_cmd(struct lio *lio, int start_stop) 2020325618Ssbruno{ 2021325618Ssbruno struct lio_soft_command *sc; 2022325618Ssbruno struct lio_rx_ctl_context *ctx; 2023325618Ssbruno union octeon_cmd *ncmd; 2024325618Ssbruno struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 2025325618Ssbruno int ctx_size = sizeof(struct lio_rx_ctl_context); 2026325618Ssbruno int retval; 2027325618Ssbruno 2028325618Ssbruno if (oct->props.rx_on == start_stop) 2029325618Ssbruno return; 2030325618Ssbruno 2031325618Ssbruno sc = lio_alloc_soft_command(oct, OCTEON_CMD_SIZE, 16, ctx_size); 2032325618Ssbruno if (sc == NULL) 2033325618Ssbruno return; 2034325618Ssbruno 2035325618Ssbruno ncmd = (union octeon_cmd *)sc->virtdptr; 2036325618Ssbruno ctx = (struct lio_rx_ctl_context *)sc->ctxptr; 2037325618Ssbruno 2038325618Ssbruno ctx->cond = 0; 2039325618Ssbruno ctx->octeon_id = lio_get_device_id(oct); 2040325618Ssbruno ncmd->cmd64 = 0; 2041325618Ssbruno ncmd->s.cmd = LIO_CMD_RX_CTL; 2042325618Ssbruno ncmd->s.param1 = start_stop; 2043325618Ssbruno 2044325618Ssbruno lio_swap_8B_data((uint64_t *)ncmd, (OCTEON_CMD_SIZE >> 3)); 2045325618Ssbruno 2046325618Ssbruno sc->iq_no = lio->linfo.txpciq[0].s.q_no; 2047325618Ssbruno 2048325618Ssbruno lio_prepare_soft_command(oct, sc, LIO_OPCODE_NIC, LIO_OPCODE_NIC_CMD, 0, 2049325618Ssbruno 0, 0); 2050325618Ssbruno 2051325618Ssbruno sc->callback = lio_rx_ctl_callback; 2052325618Ssbruno sc->callback_arg = sc; 2053325618Ssbruno sc->wait_time = 5000; 2054325618Ssbruno 2055325618Ssbruno retval = lio_send_soft_command(oct, sc); 2056325618Ssbruno if (retval == LIO_IQ_SEND_FAILED) { 2057325618Ssbruno lio_dev_err(oct, "Failed to send RX Control message\n"); 2058325618Ssbruno } else { 2059325618Ssbruno /* 2060325618Ssbruno * Sleep on a wait queue till the cond flag indicates that the 2061325618Ssbruno * response arrived or timed-out. 2062325618Ssbruno */ 2063325618Ssbruno lio_sleep_cond(oct, &ctx->cond); 2064325618Ssbruno oct->props.rx_on = start_stop; 2065325618Ssbruno } 2066325618Ssbruno 2067325618Ssbruno lio_free_soft_command(oct, sc); 2068325618Ssbruno} 2069325618Ssbruno 2070325618Ssbrunostatic void 2071325618Ssbrunolio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid) 2072325618Ssbruno{ 2073325618Ssbruno struct lio_ctrl_pkt nctrl; 2074325618Ssbruno struct lio *lio = if_getsoftc(ifp); 2075325618Ssbruno struct octeon_device *oct = lio->oct_dev; 2076325618Ssbruno int ret = 0; 2077325618Ssbruno 2078325618Ssbruno if (if_getsoftc(ifp) != arg) /* Not our event */ 2079325618Ssbruno return; 2080325618Ssbruno 2081325618Ssbruno if ((vid == 0) || (vid > 4095)) /* Invalid */ 2082325618Ssbruno return; 2083325618Ssbruno 2084325618Ssbruno bzero(&nctrl, sizeof(struct lio_ctrl_pkt)); 2085325618Ssbruno 2086325618Ssbruno nctrl.ncmd.cmd64 = 0; 2087325618Ssbruno nctrl.ncmd.s.cmd = LIO_CMD_ADD_VLAN_FILTER; 2088325618Ssbruno nctrl.ncmd.s.param1 = vid; 2089325618Ssbruno nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 2090325618Ssbruno nctrl.wait_time = 100; 2091325618Ssbruno nctrl.lio = lio; 2092325618Ssbruno nctrl.cb_fn = lio_ctrl_cmd_completion; 2093325618Ssbruno 2094325618Ssbruno ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl); 2095325618Ssbruno if (ret < 0) { 2096325618Ssbruno lio_dev_err(oct, "Add VLAN filter failed in core (ret: 0x%x)\n", 2097325618Ssbruno ret); 2098325618Ssbruno } 2099325618Ssbruno} 2100325618Ssbruno 2101325618Ssbrunostatic void 2102325618Ssbrunolio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, uint16_t vid) 2103325618Ssbruno{ 2104325618Ssbruno struct lio_ctrl_pkt nctrl; 2105325618Ssbruno struct lio *lio = if_getsoftc(ifp); 2106325618Ssbruno struct octeon_device *oct = lio->oct_dev; 2107325618Ssbruno int ret = 0; 2108325618Ssbruno 2109325618Ssbruno if (if_getsoftc(ifp) != arg) /* Not our event */ 2110325618Ssbruno return; 2111325618Ssbruno 2112325618Ssbruno if ((vid == 0) || (vid > 4095)) /* Invalid */ 2113325618Ssbruno return; 2114325618Ssbruno 2115325618Ssbruno bzero(&nctrl, sizeof(struct lio_ctrl_pkt)); 2116325618Ssbruno 2117325618Ssbruno nctrl.ncmd.cmd64 = 0; 2118325618Ssbruno nctrl.ncmd.s.cmd = LIO_CMD_DEL_VLAN_FILTER; 2119325618Ssbruno nctrl.ncmd.s.param1 = vid; 2120325618Ssbruno nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 2121325618Ssbruno nctrl.wait_time = 100; 2122325618Ssbruno nctrl.lio = lio; 2123325618Ssbruno nctrl.cb_fn = lio_ctrl_cmd_completion; 2124325618Ssbruno 2125325618Ssbruno ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl); 2126325618Ssbruno if (ret < 0) { 2127325618Ssbruno lio_dev_err(oct, 2128325618Ssbruno "Kill VLAN filter failed in core (ret: 0x%x)\n", 2129325618Ssbruno ret); 2130325618Ssbruno } 2131325618Ssbruno} 2132325618Ssbruno 2133325618Ssbrunostatic int 2134325618Ssbrunolio_wait_for_oq_pkts(struct octeon_device *oct) 2135325618Ssbruno{ 2136325618Ssbruno int i, pending_pkts, pkt_cnt = 0, retry = 100; 2137325618Ssbruno 2138325618Ssbruno do { 2139325618Ssbruno pending_pkts = 0; 2140325618Ssbruno 2141325618Ssbruno for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) { 2142325618Ssbruno if (!(oct->io_qmask.oq & BIT_ULL(i))) 2143325618Ssbruno continue; 2144325618Ssbruno 2145325618Ssbruno pkt_cnt = lio_droq_check_hw_for_pkts(oct->droq[i]); 2146325618Ssbruno if (pkt_cnt > 0) { 2147325618Ssbruno pending_pkts += pkt_cnt; 2148325618Ssbruno taskqueue_enqueue(oct->droq[i]->droq_taskqueue, 2149325618Ssbruno &oct->droq[i]->droq_task); 2150325618Ssbruno } 2151325618Ssbruno } 2152325618Ssbruno 2153325618Ssbruno pkt_cnt = 0; 2154325618Ssbruno lio_sleep_timeout(1); 2155325618Ssbruno } while (retry-- && pending_pkts); 2156325618Ssbruno 2157325618Ssbruno return (pkt_cnt); 2158325618Ssbruno} 2159325618Ssbruno 2160325618Ssbrunostatic void 2161325618Ssbrunolio_destroy_resources(struct octeon_device *oct) 2162325618Ssbruno{ 2163325618Ssbruno int i, refcount; 2164325618Ssbruno 2165325618Ssbruno switch (atomic_load_acq_int(&oct->status)) { 2166325618Ssbruno case LIO_DEV_RUNNING: 2167325618Ssbruno case LIO_DEV_CORE_OK: 2168325618Ssbruno /* No more instructions will be forwarded. */ 2169325618Ssbruno atomic_store_rel_int(&oct->status, LIO_DEV_IN_RESET); 2170325618Ssbruno 2171325618Ssbruno oct->app_mode = LIO_DRV_INVALID_APP; 2172325618Ssbruno lio_dev_dbg(oct, "Device state is now %s\n", 2173325618Ssbruno lio_get_state_string(&oct->status)); 2174325618Ssbruno 2175325618Ssbruno lio_sleep_timeout(100); 2176325618Ssbruno 2177325618Ssbruno /* fallthrough */ 2178325618Ssbruno case LIO_DEV_HOST_OK: 2179325618Ssbruno 2180325618Ssbruno /* fallthrough */ 2181325618Ssbruno case LIO_DEV_CONSOLE_INIT_DONE: 2182325618Ssbruno /* Remove any consoles */ 2183325618Ssbruno lio_remove_consoles(oct); 2184325618Ssbruno 2185325618Ssbruno /* fallthrough */ 2186325618Ssbruno case LIO_DEV_IO_QUEUES_DONE: 2187325618Ssbruno if (lio_wait_for_pending_requests(oct)) 2188325618Ssbruno lio_dev_err(oct, "There were pending requests\n"); 2189325618Ssbruno 2190325618Ssbruno if (lio_wait_for_instr_fetch(oct)) 2191325618Ssbruno lio_dev_err(oct, "IQ had pending instructions\n"); 2192325618Ssbruno 2193325618Ssbruno /* 2194325618Ssbruno * Disable the input and output queues now. No more packets will 2195325618Ssbruno * arrive from Octeon, but we should wait for all packet 2196325618Ssbruno * processing to finish. 2197325618Ssbruno */ 2198325618Ssbruno oct->fn_list.disable_io_queues(oct); 2199325618Ssbruno 2200325618Ssbruno if (lio_wait_for_oq_pkts(oct)) 2201325618Ssbruno lio_dev_err(oct, "OQ had pending packets\n"); 2202325618Ssbruno 2203325618Ssbruno /* fallthrough */ 2204325618Ssbruno case LIO_DEV_INTR_SET_DONE: 2205325618Ssbruno /* Disable interrupts */ 2206325618Ssbruno oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 2207325618Ssbruno 2208325618Ssbruno if (oct->msix_on) { 2209325618Ssbruno for (i = 0; i < oct->num_msix_irqs - 1; i++) { 2210325618Ssbruno if (oct->ioq_vector[i].tag != NULL) { 2211325618Ssbruno bus_teardown_intr(oct->device, 2212325618Ssbruno oct->ioq_vector[i].msix_res, 2213325618Ssbruno oct->ioq_vector[i].tag); 2214325618Ssbruno oct->ioq_vector[i].tag = NULL; 2215325618Ssbruno } 2216325618Ssbruno if (oct->ioq_vector[i].msix_res != NULL) { 2217325618Ssbruno bus_release_resource(oct->device, 2218325618Ssbruno SYS_RES_IRQ, 2219325618Ssbruno oct->ioq_vector[i].vector, 2220325618Ssbruno oct->ioq_vector[i].msix_res); 2221325618Ssbruno oct->ioq_vector[i].msix_res = NULL; 2222325618Ssbruno } 2223325618Ssbruno } 2224325618Ssbruno /* non-iov vector's argument is oct struct */ 2225325618Ssbruno if (oct->tag != NULL) { 2226325618Ssbruno bus_teardown_intr(oct->device, oct->msix_res, 2227325618Ssbruno oct->tag); 2228325618Ssbruno oct->tag = NULL; 2229325618Ssbruno } 2230325618Ssbruno 2231325618Ssbruno if (oct->msix_res != NULL) { 2232325618Ssbruno bus_release_resource(oct->device, SYS_RES_IRQ, 2233325618Ssbruno oct->aux_vector, 2234325618Ssbruno oct->msix_res); 2235325618Ssbruno oct->msix_res = NULL; 2236325618Ssbruno } 2237325618Ssbruno 2238325618Ssbruno pci_release_msi(oct->device); 2239325618Ssbruno } 2240325618Ssbruno /* fallthrough */ 2241325618Ssbruno case LIO_DEV_IN_RESET: 2242325618Ssbruno case LIO_DEV_DROQ_INIT_DONE: 2243325618Ssbruno /* Wait for any pending operations */ 2244325618Ssbruno lio_mdelay(100); 2245325618Ssbruno for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) { 2246325618Ssbruno if (!(oct->io_qmask.oq & BIT_ULL(i))) 2247325618Ssbruno continue; 2248325618Ssbruno lio_delete_droq(oct, i); 2249325618Ssbruno } 2250325618Ssbruno 2251325618Ssbruno /* fallthrough */ 2252325618Ssbruno case LIO_DEV_RESP_LIST_INIT_DONE: 2253325618Ssbruno for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) { 2254325618Ssbruno if (oct->droq[i] != NULL) { 2255325618Ssbruno free(oct->droq[i], M_DEVBUF); 2256325618Ssbruno oct->droq[i] = NULL; 2257325618Ssbruno } 2258325618Ssbruno } 2259325618Ssbruno lio_delete_response_list(oct); 2260325618Ssbruno 2261325618Ssbruno /* fallthrough */ 2262325618Ssbruno case LIO_DEV_INSTR_QUEUE_INIT_DONE: 2263325618Ssbruno for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) { 2264325618Ssbruno if (!(oct->io_qmask.iq & BIT_ULL(i))) 2265325618Ssbruno continue; 2266325618Ssbruno 2267325618Ssbruno lio_delete_instr_queue(oct, i); 2268325618Ssbruno } 2269325618Ssbruno 2270325618Ssbruno /* fallthrough */ 2271325618Ssbruno case LIO_DEV_MSIX_ALLOC_VECTOR_DONE: 2272325618Ssbruno for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) { 2273325618Ssbruno if (oct->instr_queue[i] != NULL) { 2274325618Ssbruno free(oct->instr_queue[i], M_DEVBUF); 2275325618Ssbruno oct->instr_queue[i] = NULL; 2276325618Ssbruno } 2277325618Ssbruno } 2278325618Ssbruno lio_free_ioq_vector(oct); 2279325618Ssbruno 2280325618Ssbruno /* fallthrough */ 2281325618Ssbruno case LIO_DEV_SC_BUFF_POOL_INIT_DONE: 2282325618Ssbruno lio_free_sc_buffer_pool(oct); 2283325618Ssbruno 2284325618Ssbruno /* fallthrough */ 2285325618Ssbruno case LIO_DEV_DISPATCH_INIT_DONE: 2286325618Ssbruno lio_delete_dispatch_list(oct); 2287325618Ssbruno 2288325618Ssbruno /* fallthrough */ 2289325618Ssbruno case LIO_DEV_PCI_MAP_DONE: 2290325618Ssbruno refcount = lio_deregister_device(oct); 2291325618Ssbruno 2292325618Ssbruno if (fw_type_is_none()) 2293325618Ssbruno lio_pci_flr(oct); 2294325618Ssbruno 2295325618Ssbruno if (!refcount) 2296325618Ssbruno oct->fn_list.soft_reset(oct); 2297325618Ssbruno 2298325618Ssbruno lio_unmap_pci_barx(oct, 0); 2299325618Ssbruno lio_unmap_pci_barx(oct, 1); 2300325618Ssbruno 2301325618Ssbruno /* fallthrough */ 2302325618Ssbruno case LIO_DEV_PCI_ENABLE_DONE: 2303325618Ssbruno /* Disable the device, releasing the PCI INT */ 2304325618Ssbruno pci_disable_busmaster(oct->device); 2305325618Ssbruno 2306325618Ssbruno /* fallthrough */ 2307325618Ssbruno case LIO_DEV_BEGIN_STATE: 2308325618Ssbruno break; 2309325618Ssbruno } /* end switch (oct->status) */ 2310325618Ssbruno} 2311