1325618Ssbruno/* 2325618Ssbruno * BSD LICENSE 3325618Ssbruno * 4325618Ssbruno * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5325618Ssbruno * All rights reserved. 6325618Ssbruno * 7325618Ssbruno * Redistribution and use in source and binary forms, with or without 8325618Ssbruno * modification, are permitted provided that the following conditions 9325618Ssbruno * are met: 10325618Ssbruno * 11325618Ssbruno * * Redistributions of source code must retain the above copyright 12325618Ssbruno * notice, this list of conditions and the following disclaimer. 13325618Ssbruno * * Redistributions in binary form must reproduce the above copyright 14325618Ssbruno * notice, this list of conditions and the following disclaimer in 15325618Ssbruno * the documentation and/or other materials provided with the 16325618Ssbruno * distribution. 17325618Ssbruno * * Neither the name of Cavium, Inc. nor the names of its 18325618Ssbruno * contributors may be used to endorse or promote products derived 19325618Ssbruno * from this software without specific prior written permission. 20325618Ssbruno * 21325618Ssbruno * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22325618Ssbruno * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23325618Ssbruno * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24325618Ssbruno * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25325618Ssbruno * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26325618Ssbruno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27325618Ssbruno * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28325618Ssbruno * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29325618Ssbruno * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30325618Ssbruno * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31325618Ssbruno * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32325618Ssbruno */ 33325618Ssbruno/*$FreeBSD: stable/11/sys/dev/liquidio/lio_core.c 325618 2017-11-09 19:52:56Z sbruno $*/ 34325618Ssbruno 35325618Ssbruno#include "lio_bsd.h" 36325618Ssbruno#include "lio_common.h" 37325618Ssbruno#include "lio_droq.h" 38325618Ssbruno#include "lio_iq.h" 39325618Ssbruno#include "lio_response_manager.h" 40325618Ssbruno#include "lio_device.h" 41325618Ssbruno#include "lio_ctrl.h" 42325618Ssbruno#include "lio_main.h" 43325618Ssbruno#include "lio_rxtx.h" 44325618Ssbruno#include "lio_network.h" 45325618Ssbruno 46325618Ssbrunoint 47325618Ssbrunolio_set_feature(struct ifnet *ifp, int cmd, uint16_t param1) 48325618Ssbruno{ 49325618Ssbruno struct lio_ctrl_pkt nctrl; 50325618Ssbruno struct lio *lio = if_getsoftc(ifp); 51325618Ssbruno struct octeon_device *oct = lio->oct_dev; 52325618Ssbruno int ret = 0; 53325618Ssbruno 54325618Ssbruno bzero(&nctrl, sizeof(struct lio_ctrl_pkt)); 55325618Ssbruno 56325618Ssbruno nctrl.ncmd.cmd64 = 0; 57325618Ssbruno nctrl.ncmd.s.cmd = cmd; 58325618Ssbruno nctrl.ncmd.s.param1 = param1; 59325618Ssbruno nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 60325618Ssbruno nctrl.wait_time = 100; 61325618Ssbruno nctrl.lio = lio; 62325618Ssbruno nctrl.cb_fn = lio_ctrl_cmd_completion; 63325618Ssbruno 64325618Ssbruno ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl); 65325618Ssbruno if (ret < 0) { 66325618Ssbruno lio_dev_err(oct, "Feature change failed in core (ret: 0x%x)\n", 67325618Ssbruno ret); 68325618Ssbruno } 69325618Ssbruno 70325618Ssbruno return (ret); 71325618Ssbruno} 72325618Ssbruno 73325618Ssbrunovoid 74325618Ssbrunolio_ctrl_cmd_completion(void *nctrl_ptr) 75325618Ssbruno{ 76325618Ssbruno struct lio_ctrl_pkt *nctrl = (struct lio_ctrl_pkt *)nctrl_ptr; 77325618Ssbruno struct lio *lio; 78325618Ssbruno struct octeon_device *oct; 79325618Ssbruno uint8_t *mac; 80325618Ssbruno 81325618Ssbruno lio = nctrl->lio; 82325618Ssbruno 83325618Ssbruno if (lio->oct_dev == NULL) 84325618Ssbruno return; 85325618Ssbruno 86325618Ssbruno oct = lio->oct_dev; 87325618Ssbruno 88325618Ssbruno switch (nctrl->ncmd.s.cmd) { 89325618Ssbruno case LIO_CMD_CHANGE_DEVFLAGS: 90325618Ssbruno case LIO_CMD_SET_MULTI_LIST: 91325618Ssbruno break; 92325618Ssbruno 93325618Ssbruno case LIO_CMD_CHANGE_MACADDR: 94325618Ssbruno mac = ((uint8_t *)&nctrl->udd[0]) + 2; 95325618Ssbruno if (nctrl->ncmd.s.param1) { 96325618Ssbruno /* vfidx is 0 based, but vf_num (param1) is 1 based */ 97325618Ssbruno int vfidx = nctrl->ncmd.s.param1 - 1; 98325618Ssbruno bool mac_is_admin_assigned = nctrl->ncmd.s.param2; 99325618Ssbruno 100325618Ssbruno if (mac_is_admin_assigned) 101325618Ssbruno lio_dev_info(oct, "MAC Address %pM is configured for VF %d\n", 102325618Ssbruno mac, vfidx); 103325618Ssbruno } else { 104325618Ssbruno lio_dev_info(oct, "MAC Address changed to %02x:%02x:%02x:%02x:%02x:%02x\n", 105325618Ssbruno mac[0], mac[1], mac[2], mac[3], mac[4], 106325618Ssbruno mac[5]); 107325618Ssbruno } 108325618Ssbruno break; 109325618Ssbruno 110325618Ssbruno case LIO_CMD_GPIO_ACCESS: 111325618Ssbruno lio_dev_info(oct, "LED Flashing visual identification\n"); 112325618Ssbruno break; 113325618Ssbruno 114325618Ssbruno case LIO_CMD_ID_ACTIVE: 115325618Ssbruno lio_dev_info(oct, "LED Flashing visual identification\n"); 116325618Ssbruno break; 117325618Ssbruno 118325618Ssbruno case LIO_CMD_LRO_ENABLE: 119325618Ssbruno lio_dev_info(oct, "HW LRO Enabled\n"); 120325618Ssbruno break; 121325618Ssbruno 122325618Ssbruno case LIO_CMD_LRO_DISABLE: 123325618Ssbruno lio_dev_info(oct, "HW LRO Disabled\n"); 124325618Ssbruno break; 125325618Ssbruno 126325618Ssbruno case LIO_CMD_VERBOSE_ENABLE: 127325618Ssbruno lio_dev_info(oct, "Firmware debug enabled\n"); 128325618Ssbruno break; 129325618Ssbruno 130325618Ssbruno case LIO_CMD_VERBOSE_DISABLE: 131325618Ssbruno lio_dev_info(oct, "Firmware debug disabled\n"); 132325618Ssbruno break; 133325618Ssbruno 134325618Ssbruno case LIO_CMD_VLAN_FILTER_CTL: 135325618Ssbruno if (nctrl->ncmd.s.param1) 136325618Ssbruno lio_dev_info(oct, "VLAN filter enabled\n"); 137325618Ssbruno else 138325618Ssbruno lio_dev_info(oct, "VLAN filter disabled\n"); 139325618Ssbruno break; 140325618Ssbruno 141325618Ssbruno case LIO_CMD_ADD_VLAN_FILTER: 142325618Ssbruno lio_dev_info(oct, "VLAN filter %d added\n", 143325618Ssbruno nctrl->ncmd.s.param1); 144325618Ssbruno break; 145325618Ssbruno 146325618Ssbruno case LIO_CMD_DEL_VLAN_FILTER: 147325618Ssbruno lio_dev_info(oct, "VLAN filter %d removed\n", 148325618Ssbruno nctrl->ncmd.s.param1); 149325618Ssbruno break; 150325618Ssbruno 151325618Ssbruno case LIO_CMD_SET_SETTINGS: 152325618Ssbruno lio_dev_info(oct, "Settings changed\n"); 153325618Ssbruno break; 154325618Ssbruno 155325618Ssbruno /* 156325618Ssbruno * Case to handle "LIO_CMD_TNL_RX_CSUM_CTL" 157325618Ssbruno * Command passed by NIC driver 158325618Ssbruno */ 159325618Ssbruno case LIO_CMD_TNL_RX_CSUM_CTL: 160325618Ssbruno if (nctrl->ncmd.s.param1 == LIO_CMD_RXCSUM_ENABLE) { 161325618Ssbruno lio_dev_info(oct, "RX Checksum Offload Enabled\n"); 162325618Ssbruno } else if (nctrl->ncmd.s.param1 == LIO_CMD_RXCSUM_DISABLE) { 163325618Ssbruno lio_dev_info(oct, "RX Checksum Offload Disabled\n"); 164325618Ssbruno } 165325618Ssbruno break; 166325618Ssbruno 167325618Ssbruno /* 168325618Ssbruno * Case to handle "LIO_CMD_TNL_TX_CSUM_CTL" 169325618Ssbruno * Command passed by NIC driver 170325618Ssbruno */ 171325618Ssbruno case LIO_CMD_TNL_TX_CSUM_CTL: 172325618Ssbruno if (nctrl->ncmd.s.param1 == LIO_CMD_TXCSUM_ENABLE) { 173325618Ssbruno lio_dev_info(oct, "TX Checksum Offload Enabled\n"); 174325618Ssbruno } else if (nctrl->ncmd.s.param1 == LIO_CMD_TXCSUM_DISABLE) { 175325618Ssbruno lio_dev_info(oct, "TX Checksum Offload Disabled\n"); 176325618Ssbruno } 177325618Ssbruno break; 178325618Ssbruno 179325618Ssbruno /* 180325618Ssbruno * Case to handle "LIO_CMD_VXLAN_PORT_CONFIG" 181325618Ssbruno * Command passed by NIC driver 182325618Ssbruno */ 183325618Ssbruno case LIO_CMD_VXLAN_PORT_CONFIG: 184325618Ssbruno if (nctrl->ncmd.s.more == LIO_CMD_VXLAN_PORT_ADD) { 185325618Ssbruno lio_dev_info(oct, "VxLAN Destination UDP PORT:%d ADDED\n", 186325618Ssbruno nctrl->ncmd.s.param1); 187325618Ssbruno } else if (nctrl->ncmd.s.more == LIO_CMD_VXLAN_PORT_DEL) { 188325618Ssbruno lio_dev_info(oct, "VxLAN Destination UDP PORT:%d DELETED\n", 189325618Ssbruno nctrl->ncmd.s.param1); 190325618Ssbruno } 191325618Ssbruno break; 192325618Ssbruno 193325618Ssbruno case LIO_CMD_SET_FLOW_CTL: 194325618Ssbruno lio_dev_info(oct, "Set RX/TX flow control parameters\n"); 195325618Ssbruno break; 196325618Ssbruno 197325618Ssbruno case LIO_CMD_SET_FNV: 198325618Ssbruno if (nctrl->ncmd.s.param1 == LIO_CMD_FNV_ENABLE) 199325618Ssbruno lio_dev_info(oct, "FNV Enabled\n"); 200325618Ssbruno else if (nctrl->ncmd.s.param1 == LIO_CMD_FNV_DISABLE) 201325618Ssbruno lio_dev_info(oct, "FNV Disabled\n"); 202325618Ssbruno break; 203325618Ssbruno 204325618Ssbruno case LIO_CMD_PKT_STEERING_CTL: 205325618Ssbruno if (nctrl->ncmd.s.param1 == LIO_CMD_PKT_STEERING_ENABLE) { 206325618Ssbruno lio_dev_info(oct, "Packet Steering Enabled\n"); 207325618Ssbruno } else if (nctrl->ncmd.s.param1 == 208325618Ssbruno LIO_CMD_PKT_STEERING_DISABLE) { 209325618Ssbruno lio_dev_info(oct, "Packet Steering Disabled\n"); 210325618Ssbruno } 211325618Ssbruno 212325618Ssbruno break; 213325618Ssbruno 214325618Ssbruno case LIO_CMD_QUEUE_COUNT_CTL: 215325618Ssbruno lio_dev_info(oct, "Queue count updated to %d\n", 216325618Ssbruno nctrl->ncmd.s.param1); 217325618Ssbruno break; 218325618Ssbruno 219325618Ssbruno default: 220325618Ssbruno lio_dev_err(oct, "%s Unknown cmd %d\n", __func__, 221325618Ssbruno nctrl->ncmd.s.cmd); 222325618Ssbruno } 223325618Ssbruno} 224325618Ssbruno 225325618Ssbruno 226325618Ssbruno/* 227325618Ssbruno * \brief Setup output queue 228325618Ssbruno * @param oct octeon device 229325618Ssbruno * @param q_no which queue 230325618Ssbruno * @param num_descs how many descriptors 231325618Ssbruno * @param desc_size size of each descriptor 232325618Ssbruno * @param app_ctx application context 233325618Ssbruno */ 234325618Ssbrunostatic int 235325618Ssbrunolio_setup_droq(struct octeon_device *oct, int q_no, int num_descs, 236325618Ssbruno int desc_size, void *app_ctx) 237325618Ssbruno{ 238325618Ssbruno int ret_val = 0; 239325618Ssbruno 240325618Ssbruno lio_dev_dbg(oct, "Creating Droq: %d\n", q_no); 241325618Ssbruno /* droq creation and local register settings. */ 242325618Ssbruno ret_val = lio_create_droq(oct, q_no, num_descs, desc_size, app_ctx); 243325618Ssbruno if (ret_val < 0) 244325618Ssbruno return (ret_val); 245325618Ssbruno 246325618Ssbruno if (ret_val == 1) { 247325618Ssbruno lio_dev_dbg(oct, "Using default droq %d\n", q_no); 248325618Ssbruno return (0); 249325618Ssbruno } 250325618Ssbruno 251325618Ssbruno /* 252325618Ssbruno * Send Credit for Octeon Output queues. Credits are always 253325618Ssbruno * sent after the output queue is enabled. 254325618Ssbruno */ 255325618Ssbruno lio_write_csr32(oct, oct->droq[q_no]->pkts_credit_reg, 256325618Ssbruno oct->droq[q_no]->max_count); 257325618Ssbruno 258325618Ssbruno return (ret_val); 259325618Ssbruno} 260325618Ssbruno 261325618Ssbrunostatic void 262325618Ssbrunolio_push_packet(void *m_buff, uint32_t len, union octeon_rh *rh, void *rxq, 263325618Ssbruno void *arg) 264325618Ssbruno{ 265325618Ssbruno struct mbuf *mbuf = m_buff; 266325618Ssbruno struct ifnet *ifp = arg; 267325618Ssbruno struct lio_droq *droq = rxq; 268325618Ssbruno 269325618Ssbruno if (ifp != NULL) { 270325618Ssbruno struct lio *lio = if_getsoftc(ifp); 271325618Ssbruno 272325618Ssbruno /* Do not proceed if the interface is not in RUNNING state. */ 273325618Ssbruno if (!lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) { 274325618Ssbruno lio_recv_buffer_free(mbuf); 275325618Ssbruno droq->stats.rx_dropped++; 276325618Ssbruno return; 277325618Ssbruno } 278325618Ssbruno 279325618Ssbruno if (rh->r_dh.has_hash) { 280325618Ssbruno uint32_t hashtype, hashval; 281325618Ssbruno 282325618Ssbruno if (rh->r_dh.has_hwtstamp) { 283325618Ssbruno hashval = htobe32(*(uint32_t *) 284325618Ssbruno (((uint8_t *)mbuf->m_data) + 285325618Ssbruno ((rh->r_dh.len - 2) * 286325618Ssbruno BYTES_PER_DHLEN_UNIT))); 287325618Ssbruno hashtype = 288325618Ssbruno htobe32(*(((uint32_t *) 289325618Ssbruno (((uint8_t *)mbuf->m_data) + 290325618Ssbruno ((rh->r_dh.len - 2) * 291325618Ssbruno BYTES_PER_DHLEN_UNIT))) + 1)); 292325618Ssbruno } else { 293325618Ssbruno hashval = htobe32(*(uint32_t *) 294325618Ssbruno (((uint8_t *)mbuf->m_data) + 295325618Ssbruno ((rh->r_dh.len - 1) * 296325618Ssbruno BYTES_PER_DHLEN_UNIT))); 297325618Ssbruno hashtype = 298325618Ssbruno htobe32(*(((uint32_t *) 299325618Ssbruno (((uint8_t *)mbuf->m_data) + 300325618Ssbruno ((rh->r_dh.len - 1) * 301325618Ssbruno BYTES_PER_DHLEN_UNIT))) + 1)); 302325618Ssbruno } 303325618Ssbruno 304325618Ssbruno mbuf->m_pkthdr.flowid = hashval; 305325618Ssbruno 306325618Ssbruno switch (hashtype) { 307325618Ssbruno case LIO_RSS_HASH_IPV4: 308325618Ssbruno M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4); 309325618Ssbruno break; 310325618Ssbruno case LIO_RSS_HASH_TCP_IPV4: 311325618Ssbruno M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4); 312325618Ssbruno break; 313325618Ssbruno case LIO_RSS_HASH_IPV6: 314325618Ssbruno M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6); 315325618Ssbruno break; 316325618Ssbruno case LIO_RSS_HASH_TCP_IPV6: 317325618Ssbruno M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6); 318325618Ssbruno break; 319325618Ssbruno case LIO_RSS_HASH_IPV6_EX: 320325618Ssbruno M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6_EX); 321325618Ssbruno break; 322325618Ssbruno case LIO_RSS_HASH_TCP_IPV6_EX: 323325618Ssbruno M_HASHTYPE_SET(mbuf, 324325618Ssbruno M_HASHTYPE_RSS_TCP_IPV6_EX); 325325618Ssbruno break; 326325618Ssbruno default: 327325618Ssbruno M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 328325618Ssbruno } 329325618Ssbruno 330325618Ssbruno } else { 331325618Ssbruno /* 332325618Ssbruno * This case won't hit as FW will always set has_hash 333325618Ssbruno * in rh. 334325618Ssbruno */ 335325618Ssbruno M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE); 336325618Ssbruno mbuf->m_pkthdr.flowid = droq->q_no; 337325618Ssbruno } 338325618Ssbruno 339325618Ssbruno m_adj(mbuf, rh->r_dh.len * 8); 340325618Ssbruno len -= rh->r_dh.len * 8; 341325618Ssbruno mbuf->m_flags |= M_PKTHDR; 342325618Ssbruno 343325618Ssbruno if ((if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) && 344325618Ssbruno (rh->r_dh.priority || rh->r_dh.vlan)) { 345325618Ssbruno uint16_t priority = rh->r_dh.priority; 346325618Ssbruno uint16_t vid = rh->r_dh.vlan; 347325618Ssbruno uint16_t vtag; 348325618Ssbruno 349325618Ssbruno vtag = priority << 13 | vid; 350325618Ssbruno mbuf->m_pkthdr.ether_vtag = vtag; 351325618Ssbruno mbuf->m_flags |= M_VLANTAG; 352325618Ssbruno } 353325618Ssbruno 354325618Ssbruno if (rh->r_dh.csum_verified & LIO_IPSUM_VERIFIED) 355325618Ssbruno mbuf->m_pkthdr.csum_flags |= (CSUM_L3_CALC | 356325618Ssbruno CSUM_L3_VALID); 357325618Ssbruno 358325618Ssbruno if (rh->r_dh.csum_verified & LIO_L4SUM_VERIFIED) { 359325618Ssbruno mbuf->m_pkthdr.csum_flags |= (CSUM_L4_CALC | 360325618Ssbruno CSUM_L4_VALID); 361325618Ssbruno mbuf->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | 362325618Ssbruno CSUM_PSEUDO_HDR); 363325618Ssbruno mbuf->m_pkthdr.csum_data = htons(0xffff); 364325618Ssbruno } 365325618Ssbruno 366325618Ssbruno mbuf->m_pkthdr.rcvif = ifp; 367325618Ssbruno mbuf->m_pkthdr.len = len; 368325618Ssbruno 369325618Ssbruno if ((lio_hwlro == 0) && 370325618Ssbruno (if_getcapenable(ifp) & IFCAP_LRO) && 371325618Ssbruno (mbuf->m_pkthdr.csum_flags & 372325618Ssbruno (CSUM_L3_VALID | CSUM_L4_VALID | CSUM_DATA_VALID | 373325618Ssbruno CSUM_PSEUDO_HDR)) == (CSUM_L3_VALID | CSUM_L4_VALID | 374325618Ssbruno CSUM_DATA_VALID | 375325618Ssbruno CSUM_PSEUDO_HDR)) { 376325618Ssbruno if (droq->lro.lro_cnt) { 377325618Ssbruno if (tcp_lro_rx(&droq->lro, mbuf, 0) == 0) { 378325618Ssbruno droq->stats.rx_bytes_received += len; 379325618Ssbruno droq->stats.rx_pkts_received++; 380325618Ssbruno return; 381325618Ssbruno } 382325618Ssbruno } 383325618Ssbruno } 384325618Ssbruno 385325618Ssbruno if_input(ifp, mbuf); 386325618Ssbruno 387325618Ssbruno droq->stats.rx_bytes_received += len; 388325618Ssbruno droq->stats.rx_pkts_received++; 389325618Ssbruno 390325618Ssbruno } else { 391325618Ssbruno lio_recv_buffer_free(mbuf); 392325618Ssbruno droq->stats.rx_dropped++; 393325618Ssbruno } 394325618Ssbruno} 395325618Ssbruno 396325618Ssbruno/* 397325618Ssbruno * \brief Setup input and output queues 398325618Ssbruno * @param octeon_dev octeon device 399325618Ssbruno * @param ifidx Interface Index 400325618Ssbruno * 401325618Ssbruno * Note: Queues are with respect to the octeon device. Thus 402325618Ssbruno * an input queue is for egress packets, and output queues 403325618Ssbruno * are for ingress packets. 404325618Ssbruno */ 405325618Ssbrunoint 406325618Ssbrunolio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx, 407325618Ssbruno uint32_t num_iqs, uint32_t num_oqs) 408325618Ssbruno{ 409325618Ssbruno struct lio_droq_ops droq_ops; 410325618Ssbruno struct ifnet *ifp; 411325618Ssbruno struct lio_droq *droq; 412325618Ssbruno struct lio *lio; 413325618Ssbruno static int cpu_id, cpu_id_modulus; 414325618Ssbruno int num_tx_descs, q, q_no, retval = 0; 415325618Ssbruno 416325618Ssbruno ifp = octeon_dev->props.ifp; 417325618Ssbruno 418325618Ssbruno lio = if_getsoftc(ifp); 419325618Ssbruno 420325618Ssbruno bzero(&droq_ops, sizeof(struct lio_droq_ops)); 421325618Ssbruno 422325618Ssbruno droq_ops.fptr = lio_push_packet; 423325618Ssbruno droq_ops.farg = (void *)ifp; 424325618Ssbruno 425325618Ssbruno cpu_id = 0; 426325618Ssbruno cpu_id_modulus = mp_ncpus; 427325618Ssbruno /* set up DROQs. */ 428325618Ssbruno for (q = 0; q < num_oqs; q++) { 429325618Ssbruno q_no = lio->linfo.rxpciq[q].s.q_no; 430325618Ssbruno lio_dev_dbg(octeon_dev, "lio_setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n", 431325618Ssbruno q, q_no); 432325618Ssbruno retval = lio_setup_droq(octeon_dev, q_no, 433325618Ssbruno LIO_GET_NUM_RX_DESCS_NIC_IF_CFG( 434325618Ssbruno lio_get_conf(octeon_dev), 435325618Ssbruno lio->ifidx), 436325618Ssbruno LIO_GET_NUM_RX_BUF_SIZE_NIC_IF_CFG( 437325618Ssbruno lio_get_conf(octeon_dev), 438325618Ssbruno lio->ifidx), NULL); 439325618Ssbruno if (retval) { 440325618Ssbruno lio_dev_err(octeon_dev, "%s : Runtime DROQ(RxQ) creation failed.\n", 441325618Ssbruno __func__); 442325618Ssbruno return (1); 443325618Ssbruno } 444325618Ssbruno 445325618Ssbruno droq = octeon_dev->droq[q_no]; 446325618Ssbruno 447325618Ssbruno /* designate a CPU for this droq */ 448325618Ssbruno droq->cpu_id = cpu_id; 449325618Ssbruno cpu_id++; 450325618Ssbruno if (cpu_id >= cpu_id_modulus) 451325618Ssbruno cpu_id = 0; 452325618Ssbruno 453325618Ssbruno lio_register_droq_ops(octeon_dev, q_no, &droq_ops); 454325618Ssbruno } 455325618Ssbruno 456325618Ssbruno /* set up IQs. */ 457325618Ssbruno for (q = 0; q < num_iqs; q++) { 458325618Ssbruno num_tx_descs = LIO_GET_NUM_TX_DESCS_NIC_IF_CFG( 459325618Ssbruno lio_get_conf(octeon_dev), 460325618Ssbruno lio->ifidx); 461325618Ssbruno retval = lio_setup_iq(octeon_dev, ifidx, q, 462325618Ssbruno lio->linfo.txpciq[q], num_tx_descs); 463325618Ssbruno if (retval) { 464325618Ssbruno lio_dev_err(octeon_dev, " %s : Runtime IQ(TxQ) creation failed.\n", 465325618Ssbruno __func__); 466325618Ssbruno return (1); 467325618Ssbruno } 468325618Ssbruno } 469325618Ssbruno 470325618Ssbruno return (0); 471325618Ssbruno} 472325618Ssbruno 473325618Ssbruno/* 474325618Ssbruno * \brief Droq packet processor sceduler 475325618Ssbruno * @param oct octeon device 476325618Ssbruno */ 477325618Ssbrunostatic void 478325618Ssbrunolio_schedule_droq_pkt_handlers(struct octeon_device *oct) 479325618Ssbruno{ 480325618Ssbruno struct lio_droq *droq; 481325618Ssbruno uint64_t oq_no; 482325618Ssbruno 483325618Ssbruno if (oct->int_status & LIO_DEV_INTR_PKT_DATA) { 484325618Ssbruno for (oq_no = 0; oq_no < LIO_MAX_OUTPUT_QUEUES(oct); oq_no++) { 485325618Ssbruno if (!(oct->io_qmask.oq & BIT_ULL(oq_no))) 486325618Ssbruno continue; 487325618Ssbruno 488325618Ssbruno droq = oct->droq[oq_no]; 489325618Ssbruno 490325618Ssbruno taskqueue_enqueue(droq->droq_taskqueue, 491325618Ssbruno &droq->droq_task); 492325618Ssbruno } 493325618Ssbruno } 494325618Ssbruno} 495325618Ssbruno 496325618Ssbrunostatic void 497325618Ssbrunolio_msix_intr_handler(void *vector) 498325618Ssbruno{ 499325618Ssbruno struct lio_ioq_vector *ioq_vector = (struct lio_ioq_vector *)vector; 500325618Ssbruno struct octeon_device *oct = ioq_vector->oct_dev; 501325618Ssbruno struct lio_droq *droq = oct->droq[ioq_vector->droq_index]; 502325618Ssbruno uint64_t ret; 503325618Ssbruno 504325618Ssbruno ret = oct->fn_list.msix_interrupt_handler(ioq_vector); 505325618Ssbruno 506325618Ssbruno if ((ret & LIO_MSIX_PO_INT) || (ret & LIO_MSIX_PI_INT)) { 507325618Ssbruno struct lio_instr_queue *iq = oct->instr_queue[droq->q_no]; 508325618Ssbruno int reschedule, tx_done = 1; 509325618Ssbruno 510325618Ssbruno reschedule = lio_droq_process_packets(oct, droq, oct->rx_budget); 511325618Ssbruno 512325618Ssbruno if (atomic_load_acq_int(&iq->instr_pending)) 513325618Ssbruno tx_done = lio_flush_iq(oct, iq, oct->tx_budget); 514325618Ssbruno 515325618Ssbruno if ((oct->props.ifp != NULL) && (iq->br != NULL)) { 516325618Ssbruno if (mtx_trylock(&iq->enq_lock)) { 517325618Ssbruno if (!drbr_empty(oct->props.ifp, iq->br)) 518325618Ssbruno lio_mq_start_locked(oct->props.ifp, 519325618Ssbruno iq); 520325618Ssbruno mtx_unlock(&iq->enq_lock); 521325618Ssbruno } 522325618Ssbruno } 523325618Ssbruno 524325618Ssbruno if (reschedule || !tx_done) 525325618Ssbruno taskqueue_enqueue(droq->droq_taskqueue, &droq->droq_task); 526325618Ssbruno else 527325618Ssbruno lio_enable_irq(droq, iq); 528325618Ssbruno } 529325618Ssbruno} 530325618Ssbruno 531325618Ssbrunostatic void 532325618Ssbrunolio_intr_handler(void *dev) 533325618Ssbruno{ 534325618Ssbruno struct octeon_device *oct = (struct octeon_device *)dev; 535325618Ssbruno 536325618Ssbruno /* Disable our interrupts for the duration of ISR */ 537325618Ssbruno oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 538325618Ssbruno 539325618Ssbruno oct->fn_list.process_interrupt_regs(oct); 540325618Ssbruno 541325618Ssbruno lio_schedule_droq_pkt_handlers(oct); 542325618Ssbruno 543325618Ssbruno /* Re-enable our interrupts */ 544325618Ssbruno if (!(atomic_load_acq_int(&oct->status) == LIO_DEV_IN_RESET)) 545325618Ssbruno oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR); 546325618Ssbruno} 547325618Ssbruno 548325618Ssbrunoint 549325618Ssbrunolio_setup_interrupt(struct octeon_device *oct, uint32_t num_ioqs) 550325618Ssbruno{ 551325618Ssbruno device_t device; 552325618Ssbruno struct lio_ioq_vector *ioq_vector; 553325618Ssbruno int cpu_id, err, i; 554325618Ssbruno int num_alloc_ioq_vectors; 555325618Ssbruno int num_ioq_vectors; 556325618Ssbruno int res_id; 557325618Ssbruno 558325618Ssbruno if (!oct->msix_on) 559325618Ssbruno return (1); 560325618Ssbruno 561325618Ssbruno ioq_vector = oct->ioq_vector; 562325618Ssbruno 563325618Ssbruno#ifdef RSS 564325618Ssbruno if (oct->sriov_info.num_pf_rings != rss_getnumbuckets()) { 565325618Ssbruno lio_dev_info(oct, "IOQ vectors (%d) are not equal number of RSS buckets (%d)\n", 566325618Ssbruno oct->sriov_info.num_pf_rings, rss_getnumbuckets()); 567325618Ssbruno } 568325618Ssbruno#endif 569325618Ssbruno 570325618Ssbruno device = oct->device; 571325618Ssbruno 572325618Ssbruno oct->num_msix_irqs = num_ioqs; 573325618Ssbruno /* one non ioq interrupt for handling sli_mac_pf_int_sum */ 574325618Ssbruno oct->num_msix_irqs += 1; 575325618Ssbruno num_alloc_ioq_vectors = oct->num_msix_irqs; 576325618Ssbruno 577325618Ssbruno if (pci_alloc_msix(device, &num_alloc_ioq_vectors) || 578325618Ssbruno (num_alloc_ioq_vectors != oct->num_msix_irqs)) 579325618Ssbruno goto err; 580325618Ssbruno 581325618Ssbruno num_ioq_vectors = oct->num_msix_irqs; 582325618Ssbruno 583325618Ssbruno /* For PF, there is one non-ioq interrupt handler */ 584325618Ssbruno for (i = 0; i < num_ioq_vectors - 1; i++, ioq_vector++) { 585325618Ssbruno res_id = i + 1; 586325618Ssbruno 587325618Ssbruno ioq_vector->msix_res = 588325618Ssbruno bus_alloc_resource_any(device, SYS_RES_IRQ, &res_id, 589325618Ssbruno RF_SHAREABLE | RF_ACTIVE); 590325618Ssbruno if (ioq_vector->msix_res == NULL) { 591325618Ssbruno lio_dev_err(oct, 592325618Ssbruno "Unable to allocate bus res msix[%d]\n", i); 593325618Ssbruno goto err_1; 594325618Ssbruno } 595325618Ssbruno 596325618Ssbruno err = bus_setup_intr(device, ioq_vector->msix_res, 597325618Ssbruno INTR_TYPE_NET | INTR_MPSAFE, NULL, 598325618Ssbruno lio_msix_intr_handler, ioq_vector, 599325618Ssbruno &ioq_vector->tag); 600325618Ssbruno if (err) { 601325618Ssbruno bus_release_resource(device, SYS_RES_IRQ, res_id, 602325618Ssbruno ioq_vector->msix_res); 603325618Ssbruno ioq_vector->msix_res = NULL; 604325618Ssbruno lio_dev_err(oct, "Failed to register intr handler"); 605325618Ssbruno goto err_1; 606325618Ssbruno } 607325618Ssbruno 608325618Ssbruno bus_describe_intr(device, ioq_vector->msix_res, ioq_vector->tag, 609325618Ssbruno "rxtx%u", i); 610325618Ssbruno ioq_vector->vector = res_id; 611325618Ssbruno 612325618Ssbruno#ifdef RSS 613325618Ssbruno cpu_id = rss_getcpu(i % rss_getnumbuckets()); 614325618Ssbruno#else 615325618Ssbruno cpu_id = i % mp_ncpus; 616325618Ssbruno#endif 617325618Ssbruno CPU_SETOF(cpu_id, &ioq_vector->affinity_mask); 618325618Ssbruno 619325618Ssbruno /* Setting the IRQ affinity. */ 620325618Ssbruno err = bus_bind_intr(device, ioq_vector->msix_res, cpu_id); 621325618Ssbruno if (err) 622325618Ssbruno lio_dev_err(oct, "bus bind interrupt fail"); 623325618Ssbruno#ifdef RSS 624325618Ssbruno lio_dev_dbg(oct, "Bound RSS bucket %d to CPU %d\n", i, cpu_id); 625325618Ssbruno#else 626325618Ssbruno lio_dev_dbg(oct, "Bound Queue %d to CPU %d\n", i, cpu_id); 627325618Ssbruno#endif 628325618Ssbruno } 629325618Ssbruno 630325618Ssbruno lio_dev_dbg(oct, "MSI-X enabled\n"); 631325618Ssbruno 632325618Ssbruno res_id = num_ioq_vectors; 633325618Ssbruno oct->msix_res = bus_alloc_resource_any(device, SYS_RES_IRQ, &res_id, 634325618Ssbruno RF_SHAREABLE | RF_ACTIVE); 635325618Ssbruno if (oct->msix_res == NULL) { 636325618Ssbruno lio_dev_err(oct, "Unable to allocate bus res msix for non-ioq interrupt\n"); 637325618Ssbruno goto err_1; 638325618Ssbruno } 639325618Ssbruno 640325618Ssbruno err = bus_setup_intr(device, oct->msix_res, INTR_TYPE_NET | INTR_MPSAFE, 641325618Ssbruno NULL, lio_intr_handler, oct, &oct->tag); 642325618Ssbruno if (err) { 643325618Ssbruno bus_release_resource(device, SYS_RES_IRQ, res_id, 644325618Ssbruno oct->msix_res); 645325618Ssbruno oct->msix_res = NULL; 646325618Ssbruno lio_dev_err(oct, "Failed to register intr handler"); 647325618Ssbruno goto err_1; 648325618Ssbruno } 649325618Ssbruno 650325618Ssbruno bus_describe_intr(device, oct->msix_res, oct->tag, "aux"); 651325618Ssbruno oct->aux_vector = res_id; 652325618Ssbruno 653325618Ssbruno return (0); 654325618Ssbrunoerr_1: 655325618Ssbruno if (oct->tag != NULL) { 656325618Ssbruno bus_teardown_intr(device, oct->msix_res, oct->tag); 657325618Ssbruno oct->tag = NULL; 658325618Ssbruno } 659325618Ssbruno 660325618Ssbruno while (i) { 661325618Ssbruno i--; 662325618Ssbruno ioq_vector--; 663325618Ssbruno 664325618Ssbruno if (ioq_vector->tag != NULL) { 665325618Ssbruno bus_teardown_intr(device, ioq_vector->msix_res, 666325618Ssbruno ioq_vector->tag); 667325618Ssbruno ioq_vector->tag = NULL; 668325618Ssbruno } 669325618Ssbruno 670325618Ssbruno if (ioq_vector->msix_res != NULL) { 671325618Ssbruno bus_release_resource(device, SYS_RES_IRQ, 672325618Ssbruno ioq_vector->vector, 673325618Ssbruno ioq_vector->msix_res); 674325618Ssbruno ioq_vector->msix_res = NULL; 675325618Ssbruno } 676325618Ssbruno } 677325618Ssbruno 678325618Ssbruno if (oct->msix_res != NULL) { 679325618Ssbruno bus_release_resource(device, SYS_RES_IRQ, oct->aux_vector, 680325618Ssbruno oct->msix_res); 681325618Ssbruno oct->msix_res = NULL; 682325618Ssbruno } 683325618Ssbrunoerr: 684325618Ssbruno pci_release_msi(device); 685325618Ssbruno lio_dev_err(oct, "MSI-X disabled\n"); 686325618Ssbruno return (1); 687325618Ssbruno} 688