1333153Snp/*- 2333153Snp * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3333153Snp * 4333153Snp * Copyright (c) 2018 Chelsio Communications, Inc. 5333153Snp * All rights reserved. 6333153Snp * 7333153Snp * Redistribution and use in source and binary forms, with or without 8333153Snp * modification, are permitted provided that the following conditions 9333153Snp * are met: 10333153Snp * 1. Redistributions of source code must retain the above copyright 11333153Snp * notice, this list of conditions and the following disclaimer. 12333153Snp * 2. Redistributions in binary form must reproduce the above copyright 13333153Snp * notice, this list of conditions and the following disclaimer in the 14333153Snp * documentation and/or other materials provided with the distribution. 15333153Snp * 16333153Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17333153Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18333153Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19333153Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20333153Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21333153Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22333153Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23333153Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24333153Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25333153Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26333153Snp * SUCH DAMAGE. 27333153Snp */ 28333153Snp#include <sys/cdefs.h> 29333153Snp__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/t4_filter.c 346951 2019-04-30 08:01:59Z np $"); 30333153Snp 31333153Snp#include "opt_inet.h" 32333153Snp#include "opt_inet6.h" 33333153Snp 34333153Snp#include <sys/param.h> 35333153Snp#include <sys/eventhandler.h> 36346877Snp#include <sys/fnv_hash.h> 37333153Snp#include <sys/systm.h> 38333153Snp#include <sys/kernel.h> 39333153Snp#include <sys/module.h> 40333153Snp#include <sys/bus.h> 41333153Snp#include <sys/lock.h> 42333153Snp#include <sys/mutex.h> 43333153Snp#include <sys/rwlock.h> 44333153Snp#include <sys/socket.h> 45333153Snp#include <sys/sbuf.h> 46333153Snp#include <netinet/in.h> 47333153Snp 48333153Snp#include "common/common.h" 49333153Snp#include "common/t4_msg.h" 50333153Snp#include "common/t4_regs.h" 51346855Snp#include "common/t4_regs_values.h" 52346855Snp#include "common/t4_tcb.h" 53333153Snp#include "t4_l2t.h" 54346855Snp#include "t4_smt.h" 55333153Snp 56333153Snpstruct filter_entry { 57346877Snp LIST_ENTRY(filter_entry) link_4t; 58346877Snp LIST_ENTRY(filter_entry) link_tid; 59346877Snp 60346855Snp uint32_t valid:1; /* filter allocated and valid */ 61346855Snp uint32_t locked:1; /* filter is administratively locked or busy */ 62346855Snp uint32_t pending:1; /* filter action is pending firmware reply */ 63346855Snp int tid; /* tid of the filter TCB */ 64346855Snp struct l2t_entry *l2te; /* L2 table entry for DMAC rewrite */ 65346855Snp struct smt_entry *smt; /* SMT entry for SMAC rewrite */ 66333153Snp 67346855Snp struct t4_filter_specification fs; 68333153Snp}; 69333153Snp 70346855Snpstatic void free_filter_resources(struct filter_entry *); 71346874Snpstatic int get_tcamfilter(struct adapter *, struct t4_filter *); 72346855Snpstatic int get_hashfilter(struct adapter *, struct t4_filter *); 73346855Snpstatic int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t, 74346855Snp struct l2t_entry *, struct smt_entry *); 75346855Snpstatic int del_hashfilter(struct adapter *, struct t4_filter *); 76346855Snpstatic int configure_hashfilter_tcb(struct adapter *, struct filter_entry *); 77346855Snp 78346874Snpstatic inline bool 79346874Snpseparate_hpfilter_region(struct adapter *sc) 80346874Snp{ 81346874Snp 82346874Snp return (chip_id(sc) >= CHELSIO_T6); 83346874Snp} 84346874Snp 85346877Snpstatic inline uint32_t 86346877Snphf_hashfn_4t(struct t4_filter_specification *fs) 87346877Snp{ 88346877Snp struct t4_filter_tuple *ft = &fs->val; 89346877Snp uint32_t hash; 90346877Snp 91346877Snp if (fs->type) { 92346877Snp /* IPv6 */ 93346877Snp hash = fnv_32_buf(&ft->sip[0], 16, FNV1_32_INIT); 94346877Snp hash = fnv_32_buf(&ft->dip[0], 16, hash); 95346877Snp } else { 96346877Snp hash = fnv_32_buf(&ft->sip[0], 4, FNV1_32_INIT); 97346877Snp hash = fnv_32_buf(&ft->dip[0], 4, hash); 98346877Snp } 99346877Snp hash = fnv_32_buf(&ft->sport, sizeof(ft->sport), hash); 100346877Snp hash = fnv_32_buf(&ft->dport, sizeof(ft->dport), hash); 101346877Snp 102346877Snp return (hash); 103346877Snp} 104346877Snp 105346877Snpstatic inline uint32_t 106346877Snphf_hashfn_tid(int tid) 107346877Snp{ 108346877Snp 109346877Snp return (fnv_32_buf(&tid, sizeof(tid), FNV1_32_INIT)); 110346877Snp} 111346877Snp 112346855Snpstatic int 113346877Snpalloc_hftid_hash(struct tid_info *t, int flags) 114333153Snp{ 115346877Snp int n; 116333153Snp 117346855Snp MPASS(t->ntids > 0); 118346877Snp MPASS(t->hftid_hash_4t == NULL); 119346877Snp MPASS(t->hftid_hash_tid == NULL); 120333153Snp 121346877Snp n = max(t->ntids / 1024, 16); 122346877Snp t->hftid_hash_4t = hashinit_flags(n, M_CXGBE, &t->hftid_4t_mask, flags); 123346877Snp if (t->hftid_hash_4t == NULL) 124346855Snp return (ENOMEM); 125346877Snp t->hftid_hash_tid = hashinit_flags(n, M_CXGBE, &t->hftid_tid_mask, 126346877Snp flags); 127346877Snp if (t->hftid_hash_tid == NULL) { 128346877Snp hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask); 129346877Snp t->hftid_hash_4t = NULL; 130346877Snp return (ENOMEM); 131346877Snp } 132346877Snp 133346855Snp mtx_init(&t->hftid_lock, "T4 hashfilters", 0, MTX_DEF); 134346855Snp cv_init(&t->hftid_cv, "t4hfcv"); 135333153Snp 136346855Snp return (0); 137346855Snp} 138333153Snp 139346855Snpvoid 140346877Snpfree_hftid_hash(struct tid_info *t) 141346855Snp{ 142346877Snp struct filter_entry *f, *ftmp; 143346877Snp LIST_HEAD(, filter_entry) *head; 144346855Snp int i; 145346877Snp#ifdef INVARIANTS 146346877Snp int n = 0; 147346877Snp#endif 148333153Snp 149346877Snp if (t->tids_in_use > 0) { 150346877Snp /* Remove everything from the tid hash. */ 151346877Snp head = t->hftid_hash_tid; 152346877Snp for (i = 0; i <= t->hftid_tid_mask; i++) { 153346877Snp LIST_FOREACH_SAFE(f, &head[i], link_tid, ftmp) { 154346877Snp LIST_REMOVE(f, link_tid); 155346877Snp } 156346855Snp } 157346877Snp 158346877Snp /* Remove and then free each filter in the 4t hash. */ 159346877Snp head = t->hftid_hash_4t; 160346877Snp for (i = 0; i <= t->hftid_4t_mask; i++) { 161346877Snp LIST_FOREACH_SAFE(f, &head[i], link_4t, ftmp) { 162346877Snp#ifdef INVARIANTS 163346877Snp n += f->fs.type ? 2 : 1; 164346877Snp#endif 165346877Snp LIST_REMOVE(f, link_4t); 166346877Snp free(f, M_CXGBE); 167346877Snp } 168346877Snp } 169346877Snp MPASS(t->tids_in_use == n); 170346877Snp t->tids_in_use = 0; 171346855Snp } 172333153Snp 173346877Snp if (t->hftid_hash_4t) { 174346877Snp hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask); 175346877Snp t->hftid_hash_4t = NULL; 176346877Snp } 177346877Snp if (t->hftid_hash_tid) { 178346877Snp hashdestroy(t->hftid_hash_tid, M_CXGBE, t->hftid_tid_mask); 179346877Snp t->hftid_hash_tid = NULL; 180346877Snp } 181346855Snp if (mtx_initialized(&t->hftid_lock)) { 182346855Snp mtx_destroy(&t->hftid_lock); 183346855Snp cv_destroy(&t->hftid_cv); 184346855Snp } 185346855Snp} 186333153Snp 187346855Snpstatic void 188346877Snpinsert_hf(struct adapter *sc, struct filter_entry *f, uint32_t hash) 189346855Snp{ 190346855Snp struct tid_info *t = &sc->tids; 191346877Snp LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t; 192333153Snp 193346877Snp MPASS(head != NULL); 194346877Snp if (hash == 0) 195346877Snp hash = hf_hashfn_4t(&f->fs); 196346877Snp LIST_INSERT_HEAD(&head[hash & t->hftid_4t_mask], f, link_4t); 197346877Snp atomic_add_int(&t->tids_in_use, f->fs.type ? 2 : 1); 198346855Snp} 199333153Snp 200346877Snpstatic void 201346877Snpinsert_hftid(struct adapter *sc, struct filter_entry *f) 202346877Snp{ 203346877Snp struct tid_info *t = &sc->tids; 204346877Snp LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid; 205346877Snp uint32_t hash; 206346877Snp 207346877Snp MPASS(f->tid >= t->tid_base); 208346877Snp MPASS(f->tid - t->tid_base < t->ntids); 209346877Snp mtx_assert(&t->hftid_lock, MA_OWNED); 210346877Snp 211346877Snp hash = hf_hashfn_tid(f->tid); 212346877Snp LIST_INSERT_HEAD(&head[hash & t->hftid_tid_mask], f, link_tid); 213346877Snp} 214346877Snp 215346877Snpstatic bool 216346877Snpfilter_eq(struct t4_filter_specification *fs1, 217346877Snp struct t4_filter_specification *fs2) 218346877Snp{ 219346877Snp int n; 220346877Snp 221346877Snp MPASS(fs1->hash && fs2->hash); 222346877Snp 223346877Snp if (fs1->type != fs2->type) 224346877Snp return (false); 225346877Snp 226346877Snp n = fs1->type ? 16 : 4; 227346877Snp if (bcmp(&fs1->val.sip[0], &fs2->val.sip[0], n) || 228346877Snp bcmp(&fs1->val.dip[0], &fs2->val.dip[0], n) || 229346877Snp fs1->val.sport != fs2->val.sport || 230346877Snp fs1->val.dport != fs2->val.dport) 231346877Snp return (false); 232346877Snp 233346877Snp /* 234346877Snp * We know the masks are the same because all hashfilter masks have to 235346877Snp * conform to the global tp->hash_filter_mask and the driver has 236346877Snp * verified that already. 237346877Snp */ 238346877Snp 239346877Snp if ((fs1->mask.pfvf_vld || fs1->mask.ovlan_vld) && 240346877Snp fs1->val.vnic != fs2->val.vnic) 241346877Snp return (false); 242346877Snp if (fs1->mask.vlan_vld && fs1->val.vlan != fs2->val.vlan) 243346877Snp return (false); 244346877Snp if (fs1->mask.macidx && fs1->val.macidx != fs2->val.macidx) 245346877Snp return (false); 246346877Snp if (fs1->mask.frag && fs1->val.frag != fs2->val.frag) 247346877Snp return (false); 248346877Snp if (fs1->mask.matchtype && fs1->val.matchtype != fs2->val.matchtype) 249346877Snp return (false); 250346877Snp if (fs1->mask.iport && fs1->val.iport != fs2->val.iport) 251346877Snp return (false); 252346877Snp if (fs1->mask.fcoe && fs1->val.fcoe != fs2->val.fcoe) 253346877Snp return (false); 254346877Snp if (fs1->mask.proto && fs1->val.proto != fs2->val.proto) 255346877Snp return (false); 256346877Snp if (fs1->mask.tos && fs1->val.tos != fs2->val.tos) 257346877Snp return (false); 258346877Snp if (fs1->mask.ethtype && fs1->val.ethtype != fs2->val.ethtype) 259346877Snp return (false); 260346877Snp 261346877Snp return (true); 262346877Snp} 263346877Snp 264346877Snpstatic struct filter_entry * 265346877Snplookup_hf(struct adapter *sc, struct t4_filter_specification *fs, uint32_t hash) 266346877Snp{ 267346877Snp struct tid_info *t = &sc->tids; 268346877Snp LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t; 269346877Snp struct filter_entry *f; 270346877Snp 271346877Snp mtx_assert(&t->hftid_lock, MA_OWNED); 272346877Snp MPASS(head != NULL); 273346877Snp 274346877Snp if (hash == 0) 275346877Snp hash = hf_hashfn_4t(fs); 276346877Snp 277346877Snp LIST_FOREACH(f, &head[hash & t->hftid_4t_mask], link_4t) { 278346877Snp if (filter_eq(&f->fs, fs)) 279346877Snp return (f); 280346877Snp } 281346877Snp 282346877Snp return (NULL); 283346877Snp} 284346877Snp 285346877Snpstatic struct filter_entry * 286346855Snplookup_hftid(struct adapter *sc, int tid) 287346855Snp{ 288346855Snp struct tid_info *t = &sc->tids; 289346877Snp LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid; 290346877Snp struct filter_entry *f; 291346877Snp uint32_t hash; 292333153Snp 293346877Snp mtx_assert(&t->hftid_lock, MA_OWNED); 294346877Snp MPASS(head != NULL); 295346877Snp 296346877Snp hash = hf_hashfn_tid(tid); 297346877Snp LIST_FOREACH(f, &head[hash & t->hftid_tid_mask], link_tid) { 298346877Snp if (f->tid == tid) 299346877Snp return (f); 300346877Snp } 301346877Snp 302346877Snp return (NULL); 303346855Snp} 304333153Snp 305346855Snpstatic void 306346877Snpremove_hf(struct adapter *sc, struct filter_entry *f) 307346855Snp{ 308346855Snp struct tid_info *t = &sc->tids; 309333153Snp 310346877Snp mtx_assert(&t->hftid_lock, MA_OWNED); 311346877Snp 312346877Snp LIST_REMOVE(f, link_4t); 313346877Snp atomic_subtract_int(&t->tids_in_use, f->fs.type ? 2 : 1); 314333153Snp} 315333153Snp 316346877Snpstatic void 317346877Snpremove_hftid(struct adapter *sc, struct filter_entry *f) 318346877Snp{ 319346877Snp#ifdef INVARIANTS 320346877Snp struct tid_info *t = &sc->tids; 321346877Snp 322346877Snp mtx_assert(&t->hftid_lock, MA_OWNED); 323346877Snp#endif 324346877Snp 325346877Snp LIST_REMOVE(f, link_tid); 326346877Snp} 327346877Snp 328333153Snpstatic uint32_t 329333153Snpmode_to_fconf(uint32_t mode) 330333153Snp{ 331333153Snp uint32_t fconf = 0; 332333153Snp 333333153Snp if (mode & T4_FILTER_IP_FRAGMENT) 334333153Snp fconf |= F_FRAGMENTATION; 335333153Snp 336333153Snp if (mode & T4_FILTER_MPS_HIT_TYPE) 337333153Snp fconf |= F_MPSHITTYPE; 338333153Snp 339333153Snp if (mode & T4_FILTER_MAC_IDX) 340333153Snp fconf |= F_MACMATCH; 341333153Snp 342333153Snp if (mode & T4_FILTER_ETH_TYPE) 343333153Snp fconf |= F_ETHERTYPE; 344333153Snp 345333153Snp if (mode & T4_FILTER_IP_PROTO) 346333153Snp fconf |= F_PROTOCOL; 347333153Snp 348333153Snp if (mode & T4_FILTER_IP_TOS) 349333153Snp fconf |= F_TOS; 350333153Snp 351333153Snp if (mode & T4_FILTER_VLAN) 352333153Snp fconf |= F_VLAN; 353333153Snp 354333153Snp if (mode & T4_FILTER_VNIC) 355333153Snp fconf |= F_VNIC_ID; 356333153Snp 357333153Snp if (mode & T4_FILTER_PORT) 358333153Snp fconf |= F_PORT; 359333153Snp 360333153Snp if (mode & T4_FILTER_FCoE) 361333153Snp fconf |= F_FCOE; 362333153Snp 363333153Snp return (fconf); 364333153Snp} 365333153Snp 366333153Snpstatic uint32_t 367333153Snpmode_to_iconf(uint32_t mode) 368333153Snp{ 369333153Snp 370333153Snp if (mode & T4_FILTER_IC_VNIC) 371333153Snp return (F_VNIC); 372333153Snp return (0); 373333153Snp} 374333153Snp 375346855Snpstatic int 376346855Snpcheck_fspec_against_fconf_iconf(struct adapter *sc, 377333153Snp struct t4_filter_specification *fs) 378333153Snp{ 379333153Snp struct tp_params *tpp = &sc->params.tp; 380333153Snp uint32_t fconf = 0; 381333153Snp 382333153Snp if (fs->val.frag || fs->mask.frag) 383333153Snp fconf |= F_FRAGMENTATION; 384333153Snp 385333153Snp if (fs->val.matchtype || fs->mask.matchtype) 386333153Snp fconf |= F_MPSHITTYPE; 387333153Snp 388333153Snp if (fs->val.macidx || fs->mask.macidx) 389333153Snp fconf |= F_MACMATCH; 390333153Snp 391333153Snp if (fs->val.ethtype || fs->mask.ethtype) 392333153Snp fconf |= F_ETHERTYPE; 393333153Snp 394333153Snp if (fs->val.proto || fs->mask.proto) 395333153Snp fconf |= F_PROTOCOL; 396333153Snp 397333153Snp if (fs->val.tos || fs->mask.tos) 398333153Snp fconf |= F_TOS; 399333153Snp 400333153Snp if (fs->val.vlan_vld || fs->mask.vlan_vld) 401333153Snp fconf |= F_VLAN; 402333153Snp 403333153Snp if (fs->val.ovlan_vld || fs->mask.ovlan_vld) { 404333153Snp fconf |= F_VNIC_ID; 405333153Snp if (tpp->ingress_config & F_VNIC) 406333153Snp return (EINVAL); 407333153Snp } 408333153Snp 409333153Snp if (fs->val.pfvf_vld || fs->mask.pfvf_vld) { 410333153Snp fconf |= F_VNIC_ID; 411333153Snp if ((tpp->ingress_config & F_VNIC) == 0) 412333153Snp return (EINVAL); 413333153Snp } 414333153Snp 415333153Snp if (fs->val.iport || fs->mask.iport) 416333153Snp fconf |= F_PORT; 417333153Snp 418333153Snp if (fs->val.fcoe || fs->mask.fcoe) 419333153Snp fconf |= F_FCOE; 420333153Snp 421333153Snp if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map) 422333153Snp return (E2BIG); 423333153Snp 424333153Snp return (0); 425333153Snp} 426333153Snp 427333153Snpint 428333153Snpget_filter_mode(struct adapter *sc, uint32_t *mode) 429333153Snp{ 430346855Snp struct tp_params *tp = &sc->params.tp; 431346855Snp uint64_t mask; 432333153Snp 433346855Snp /* Non-zero incoming value in mode means "hashfilter mode". */ 434346855Snp mask = *mode ? tp->hash_filter_mask : UINT64_MAX; 435333153Snp 436346855Snp /* Always */ 437346855Snp *mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 438346855Snp T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 439346855Snp 440346855Snp#define CHECK_FIELD(fconf_bit, field_shift, field_mask, mode_bit) do { \ 441346855Snp if (tp->vlan_pri_map & (fconf_bit)) { \ 442346855Snp MPASS(tp->field_shift >= 0); \ 443346855Snp if ((mask >> tp->field_shift & field_mask) == field_mask) \ 444346855Snp *mode |= (mode_bit); \ 445346855Snp } \ 446346855Snp} while (0) 447346855Snp 448346855Snp CHECK_FIELD(F_FRAGMENTATION, frag_shift, M_FT_FRAGMENTATION, T4_FILTER_IP_FRAGMENT); 449346855Snp CHECK_FIELD(F_MPSHITTYPE, matchtype_shift, M_FT_MPSHITTYPE, T4_FILTER_MPS_HIT_TYPE); 450346855Snp CHECK_FIELD(F_MACMATCH, macmatch_shift, M_FT_MACMATCH, T4_FILTER_MAC_IDX); 451346855Snp CHECK_FIELD(F_ETHERTYPE, ethertype_shift, M_FT_ETHERTYPE, T4_FILTER_ETH_TYPE); 452346855Snp CHECK_FIELD(F_PROTOCOL, protocol_shift, M_FT_PROTOCOL, T4_FILTER_IP_PROTO); 453346855Snp CHECK_FIELD(F_TOS, tos_shift, M_FT_TOS, T4_FILTER_IP_TOS); 454346855Snp CHECK_FIELD(F_VLAN, vlan_shift, M_FT_VLAN, T4_FILTER_VLAN); 455346855Snp CHECK_FIELD(F_VNIC_ID, vnic_shift, M_FT_VNIC_ID , T4_FILTER_VNIC); 456346855Snp if (tp->ingress_config & F_VNIC) 457346855Snp *mode |= T4_FILTER_IC_VNIC; 458346855Snp CHECK_FIELD(F_PORT, port_shift, M_FT_PORT , T4_FILTER_PORT); 459346855Snp CHECK_FIELD(F_FCOE, fcoe_shift, M_FT_FCOE , T4_FILTER_FCoE); 460346855Snp#undef CHECK_FIELD 461346855Snp 462333153Snp return (0); 463333153Snp} 464333153Snp 465333153Snpint 466333153Snpset_filter_mode(struct adapter *sc, uint32_t mode) 467333153Snp{ 468333153Snp struct tp_params *tpp = &sc->params.tp; 469333153Snp uint32_t fconf, iconf; 470333153Snp int rc; 471333153Snp 472333153Snp iconf = mode_to_iconf(mode); 473333153Snp if ((iconf ^ tpp->ingress_config) & F_VNIC) { 474333153Snp /* 475333153Snp * For now we just complain if A_TP_INGRESS_CONFIG is not 476333153Snp * already set to the correct value for the requested filter 477333153Snp * mode. It's not clear if it's safe to write to this register 478333153Snp * on the fly. (And we trust the cached value of the register). 479346855Snp * 480346855Snp * check_fspec_against_fconf_iconf and other code that looks at 481346855Snp * tp->vlan_pri_map and tp->ingress_config needs to be reviewed 482346855Snp * thorougly before allowing dynamic filter mode changes. 483333153Snp */ 484333153Snp return (EBUSY); 485333153Snp } 486333153Snp 487333153Snp fconf = mode_to_fconf(mode); 488333153Snp 489333153Snp rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 490333153Snp "t4setfm"); 491333153Snp if (rc) 492333153Snp return (rc); 493333153Snp 494346874Snp if (sc->tids.ftids_in_use > 0 || sc->tids.hpftids_in_use > 0) { 495333153Snp rc = EBUSY; 496333153Snp goto done; 497333153Snp } 498333153Snp 499333153Snp#ifdef TCP_OFFLOAD 500333153Snp if (uld_active(sc, ULD_TOM)) { 501333153Snp rc = EBUSY; 502333153Snp goto done; 503333153Snp } 504333153Snp#endif 505333153Snp 506333153Snp rc = -t4_set_filter_mode(sc, fconf, true); 507333153Snpdone: 508333153Snp end_synchronized_op(sc, LOCK_HELD); 509333153Snp return (rc); 510333153Snp} 511333153Snp 512333153Snpstatic inline uint64_t 513346855Snpget_filter_hits(struct adapter *sc, uint32_t tid) 514333153Snp{ 515333153Snp uint32_t tcb_addr; 516333153Snp 517346855Snp tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE; 518333153Snp 519333153Snp if (is_t4(sc)) { 520333153Snp uint64_t hits; 521333153Snp 522333153Snp read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8); 523333153Snp return (be64toh(hits)); 524333153Snp } else { 525333153Snp uint32_t hits; 526333153Snp 527333153Snp read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4); 528333153Snp return (be32toh(hits)); 529333153Snp } 530333153Snp} 531333153Snp 532333153Snpint 533333153Snpget_filter(struct adapter *sc, struct t4_filter *t) 534333153Snp{ 535346855Snp if (t->fs.hash) 536346855Snp return (get_hashfilter(sc, t)); 537346874Snp else 538346874Snp return (get_tcamfilter(sc, t)); 539333153Snp} 540333153Snp 541333153Snpstatic int 542346855Snpset_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te, 543346855Snp struct smt_entry *smt) 544333153Snp{ 545346855Snp struct filter_entry *f; 546346855Snp struct fw_filter2_wr *fwr; 547346855Snp u_int vnic_vld, vnic_vld_mask; 548333153Snp struct wrq_cookie cookie; 549346855Snp int i, rc, busy, locked; 550346874Snp u_int tid; 551346855Snp const int ntids = t->fs.type ? 4 : 1; 552333153Snp 553346855Snp MPASS(!t->fs.hash); 554346855Snp /* Already validated against fconf, iconf */ 555346855Snp MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0); 556346855Snp MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0); 557333153Snp 558346874Snp if (separate_hpfilter_region(sc) && t->fs.prio) { 559346874Snp MPASS(t->idx < sc->tids.nhpftids); 560346874Snp f = &sc->tids.hpftid_tab[t->idx]; 561346874Snp tid = sc->tids.hpftid_base + t->idx; 562346874Snp } else { 563346874Snp MPASS(t->idx < sc->tids.nftids); 564346874Snp f = &sc->tids.ftid_tab[t->idx]; 565346874Snp tid = sc->tids.ftid_base + t->idx; 566346874Snp } 567346855Snp rc = busy = locked = 0; 568346855Snp mtx_lock(&sc->tids.ftid_lock); 569346855Snp for (i = 0; i < ntids; i++) { 570346855Snp busy += f[i].pending + f[i].valid; 571346855Snp locked += f[i].locked; 572346855Snp } 573346855Snp if (locked > 0) 574346855Snp rc = EPERM; 575346855Snp else if (busy > 0) 576346855Snp rc = EBUSY; 577346855Snp else { 578346855Snp int len16; 579346855Snp 580346855Snp if (sc->params.filter2_wr_support) 581346855Snp len16 = howmany(sizeof(struct fw_filter2_wr), 16); 582346855Snp else 583346855Snp len16 = howmany(sizeof(struct fw_filter_wr), 16); 584346876Snp fwr = start_wrq_wr(&sc->sge.ctrlq[0], len16, &cookie); 585346855Snp if (__predict_false(fwr == NULL)) 586346855Snp rc = ENOMEM; 587346855Snp else { 588346855Snp f->pending = 1; 589346874Snp if (separate_hpfilter_region(sc) && t->fs.prio) 590346874Snp sc->tids.hpftids_in_use++; 591346874Snp else 592346874Snp sc->tids.ftids_in_use++; 593333153Snp } 594333153Snp } 595346855Snp mtx_unlock(&sc->tids.ftid_lock); 596346915Snp if (rc != 0) 597346855Snp return (rc); 598333153Snp 599346855Snp /* 600346855Snp * Can't fail now. A set-filter WR will definitely be sent. 601346855Snp */ 602346855Snp 603346874Snp f->tid = tid; 604346855Snp f->fs = t->fs; 605346855Snp f->l2te = l2te; 606346855Snp f->smt = smt; 607346855Snp 608346855Snp if (t->fs.val.pfvf_vld || t->fs.val.ovlan_vld) 609333153Snp vnic_vld = 1; 610333153Snp else 611333153Snp vnic_vld = 0; 612346855Snp if (t->fs.mask.pfvf_vld || t->fs.mask.ovlan_vld) 613333153Snp vnic_vld_mask = 1; 614333153Snp else 615333153Snp vnic_vld_mask = 0; 616333153Snp 617333153Snp bzero(fwr, sizeof(*fwr)); 618346855Snp if (sc->params.filter2_wr_support) 619346855Snp fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER2_WR)); 620346855Snp else 621346855Snp fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 622333153Snp fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 623333153Snp fwr->tid_to_iq = 624346855Snp htobe32(V_FW_FILTER_WR_TID(f->tid) | 625333153Snp V_FW_FILTER_WR_RQTYPE(f->fs.type) | 626333153Snp V_FW_FILTER_WR_NOREPLY(0) | 627333153Snp V_FW_FILTER_WR_IQ(f->fs.iq)); 628333153Snp fwr->del_filter_to_l2tix = 629333153Snp htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 630333153Snp V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 631333153Snp V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 632333153Snp V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 633333153Snp V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 634333153Snp V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 635333153Snp V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 636333153Snp V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 637333153Snp V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 638333153Snp f->fs.newvlan == VLAN_REWRITE) | 639333153Snp V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 640333153Snp f->fs.newvlan == VLAN_REWRITE) | 641333153Snp V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 642333153Snp V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 643333153Snp V_FW_FILTER_WR_PRIO(f->fs.prio) | 644346855Snp V_FW_FILTER_WR_L2TIX(f->l2te ? f->l2te->idx : 0)); 645333153Snp fwr->ethtype = htobe16(f->fs.val.ethtype); 646333153Snp fwr->ethtypem = htobe16(f->fs.mask.ethtype); 647333153Snp fwr->frag_to_ovlan_vldm = 648333153Snp (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 649333153Snp V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 650333153Snp V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 651333153Snp V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) | 652333153Snp V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 653333153Snp V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask)); 654333153Snp fwr->smac_sel = 0; 655333153Snp fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 656333153Snp V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 657333153Snp fwr->maci_to_matchtypem = 658333153Snp htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 659333153Snp V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 660333153Snp V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 661333153Snp V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 662333153Snp V_FW_FILTER_WR_PORT(f->fs.val.iport) | 663333153Snp V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 664333153Snp V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 665333153Snp V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 666333153Snp fwr->ptcl = f->fs.val.proto; 667333153Snp fwr->ptclm = f->fs.mask.proto; 668333153Snp fwr->ttyp = f->fs.val.tos; 669333153Snp fwr->ttypm = f->fs.mask.tos; 670333153Snp fwr->ivlan = htobe16(f->fs.val.vlan); 671333153Snp fwr->ivlanm = htobe16(f->fs.mask.vlan); 672333153Snp fwr->ovlan = htobe16(f->fs.val.vnic); 673333153Snp fwr->ovlanm = htobe16(f->fs.mask.vnic); 674333153Snp bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 675333153Snp bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 676333153Snp bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 677333153Snp bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 678333153Snp fwr->lp = htobe16(f->fs.val.dport); 679333153Snp fwr->lpm = htobe16(f->fs.mask.dport); 680333153Snp fwr->fp = htobe16(f->fs.val.sport); 681333153Snp fwr->fpm = htobe16(f->fs.mask.sport); 682346855Snp /* sma = 0 tells the fw to use SMAC_SEL for source MAC address */ 683346855Snp bzero(fwr->sma, sizeof (fwr->sma)); 684346855Snp if (sc->params.filter2_wr_support) { 685346855Snp fwr->filter_type_swapmac = 686346855Snp V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac); 687346855Snp fwr->natmode_to_ulp_type = 688346855Snp V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ? 689346855Snp ULP_MODE_TCPDDP : ULP_MODE_NONE) | 690346855Snp V_FW_FILTER2_WR_NATFLAGCHECK(f->fs.nat_flag_chk) | 691346855Snp V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode); 692346855Snp memcpy(fwr->newlip, f->fs.nat_dip, sizeof(fwr->newlip)); 693346855Snp memcpy(fwr->newfip, f->fs.nat_sip, sizeof(fwr->newfip)); 694346855Snp fwr->newlport = htobe16(f->fs.nat_dport); 695346855Snp fwr->newfport = htobe16(f->fs.nat_sport); 696346855Snp fwr->natseqcheck = htobe32(f->fs.nat_seq_chk); 697346855Snp } 698346876Snp commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie); 699333153Snp 700346855Snp /* Wait for response. */ 701346855Snp mtx_lock(&sc->tids.ftid_lock); 702346855Snp for (;;) { 703346855Snp if (f->pending == 0) { 704346855Snp rc = f->valid ? 0 : EIO; 705346855Snp break; 706346855Snp } 707346855Snp if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) { 708346855Snp rc = EINPROGRESS; 709346855Snp break; 710346855Snp } 711346855Snp } 712346855Snp mtx_unlock(&sc->tids.ftid_lock); 713346855Snp return (rc); 714346855Snp} 715333153Snp 716346855Snpstatic int 717346855Snphashfilter_ntuple(struct adapter *sc, const struct t4_filter_specification *fs, 718346855Snp uint64_t *ftuple) 719346855Snp{ 720346855Snp struct tp_params *tp = &sc->params.tp; 721346855Snp uint64_t fmask; 722346855Snp 723346855Snp *ftuple = fmask = 0; 724346855Snp 725346855Snp /* 726346855Snp * Initialize each of the fields which we care about which are present 727346855Snp * in the Compressed Filter Tuple. 728346855Snp */ 729346855Snp if (tp->vlan_shift >= 0 && fs->mask.vlan) { 730346855Snp *ftuple |= (F_FT_VLAN_VLD | fs->val.vlan) << tp->vlan_shift; 731346855Snp fmask |= M_FT_VLAN << tp->vlan_shift; 732346855Snp } 733346855Snp 734346855Snp if (tp->port_shift >= 0 && fs->mask.iport) { 735346855Snp *ftuple |= (uint64_t)fs->val.iport << tp->port_shift; 736346855Snp fmask |= M_FT_PORT << tp->port_shift; 737346855Snp } 738346855Snp 739346855Snp if (tp->protocol_shift >= 0 && fs->mask.proto) { 740346855Snp *ftuple |= (uint64_t)fs->val.proto << tp->protocol_shift; 741346855Snp fmask |= M_FT_PROTOCOL << tp->protocol_shift; 742346855Snp } 743346855Snp 744346855Snp if (tp->tos_shift >= 0 && fs->mask.tos) { 745346855Snp *ftuple |= (uint64_t)(fs->val.tos) << tp->tos_shift; 746346855Snp fmask |= M_FT_TOS << tp->tos_shift; 747346855Snp } 748346855Snp 749346855Snp if (tp->vnic_shift >= 0 && fs->mask.vnic) { 750346855Snp /* F_VNIC in ingress config was already validated. */ 751346855Snp if (tp->ingress_config & F_VNIC) 752346855Snp MPASS(fs->mask.pfvf_vld); 753346855Snp else 754346855Snp MPASS(fs->mask.ovlan_vld); 755346855Snp 756346855Snp *ftuple |= ((1ULL << 16) | fs->val.vnic) << tp->vnic_shift; 757346855Snp fmask |= M_FT_VNIC_ID << tp->vnic_shift; 758346855Snp } 759346855Snp 760346855Snp if (tp->macmatch_shift >= 0 && fs->mask.macidx) { 761346855Snp *ftuple |= (uint64_t)(fs->val.macidx) << tp->macmatch_shift; 762346855Snp fmask |= M_FT_MACMATCH << tp->macmatch_shift; 763346855Snp } 764346855Snp 765346855Snp if (tp->ethertype_shift >= 0 && fs->mask.ethtype) { 766346855Snp *ftuple |= (uint64_t)(fs->val.ethtype) << tp->ethertype_shift; 767346855Snp fmask |= M_FT_ETHERTYPE << tp->ethertype_shift; 768346855Snp } 769346855Snp 770346855Snp if (tp->matchtype_shift >= 0 && fs->mask.matchtype) { 771346855Snp *ftuple |= (uint64_t)(fs->val.matchtype) << tp->matchtype_shift; 772346855Snp fmask |= M_FT_MPSHITTYPE << tp->matchtype_shift; 773346855Snp } 774346855Snp 775346855Snp if (tp->frag_shift >= 0 && fs->mask.frag) { 776346855Snp *ftuple |= (uint64_t)(fs->val.frag) << tp->frag_shift; 777346855Snp fmask |= M_FT_FRAGMENTATION << tp->frag_shift; 778346855Snp } 779346855Snp 780346855Snp if (tp->fcoe_shift >= 0 && fs->mask.fcoe) { 781346855Snp *ftuple |= (uint64_t)(fs->val.fcoe) << tp->fcoe_shift; 782346855Snp fmask |= M_FT_FCOE << tp->fcoe_shift; 783346855Snp } 784346855Snp 785346855Snp /* A hashfilter must conform to the filterMask. */ 786346855Snp if (fmask != tp->hash_filter_mask) 787346855Snp return (EINVAL); 788346855Snp 789333153Snp return (0); 790333153Snp} 791333153Snp 792346877Snpstatic bool 793346877Snpis_4tuple_specified(struct t4_filter_specification *fs) 794346877Snp{ 795346877Snp int i; 796346877Snp const int n = fs->type ? 16 : 4; 797346877Snp 798346877Snp if (fs->mask.sport != 0xffff || fs->mask.dport != 0xffff) 799346877Snp return (false); 800346877Snp 801346877Snp for (i = 0; i < n; i++) { 802346877Snp if (fs->mask.sip[i] != 0xff) 803346877Snp return (false); 804346877Snp if (fs->mask.dip[i] != 0xff) 805346877Snp return (false); 806346877Snp } 807346877Snp 808346877Snp return (true); 809346877Snp} 810346877Snp 811333153Snpint 812333153Snpset_filter(struct adapter *sc, struct t4_filter *t) 813333153Snp{ 814346855Snp struct tid_info *ti = &sc->tids; 815346915Snp struct l2t_entry *l2te = NULL; 816346915Snp struct smt_entry *smt = NULL; 817346855Snp uint64_t ftuple; 818346855Snp int rc; 819333153Snp 820346855Snp /* 821346855Snp * Basic filter checks first. 822346855Snp */ 823346855Snp 824346855Snp if (t->fs.hash) { 825346855Snp if (!is_hashfilter(sc) || ti->ntids == 0) 826346855Snp return (ENOTSUP); 827346855Snp /* Hardware, not user, selects a tid for hashfilters. */ 828346855Snp if (t->idx != (uint32_t)-1) 829346855Snp return (EINVAL); 830346855Snp /* T5 can't count hashfilter hits. */ 831346855Snp if (is_t5(sc) && t->fs.hitcnts) 832346855Snp return (EINVAL); 833346877Snp if (!is_4tuple_specified(&t->fs)) 834346877Snp return (EINVAL); 835346855Snp rc = hashfilter_ntuple(sc, &t->fs, &ftuple); 836346855Snp if (rc != 0) 837346855Snp return (rc); 838346855Snp } else { 839346874Snp if (separate_hpfilter_region(sc) && t->fs.prio) { 840346874Snp if (ti->nhpftids == 0) 841346874Snp return (ENOTSUP); 842346874Snp if (t->idx >= ti->nhpftids) 843346874Snp return (EINVAL); 844346874Snp } else { 845346874Snp if (ti->nftids == 0) 846346874Snp return (ENOTSUP); 847346874Snp if (t->idx >= ti->nftids) 848346874Snp return (EINVAL); 849346874Snp } 850346855Snp /* IPv6 filter idx must be 4 aligned */ 851346855Snp if (t->fs.type == 1 && 852346855Snp ((t->idx & 0x3) || t->idx + 4 >= ti->nftids)) 853346855Snp return (EINVAL); 854346855Snp } 855346855Snp 856346855Snp /* T4 doesn't support VLAN tag removal or rewrite, swapmac, and NAT. */ 857346855Snp if (is_t4(sc) && t->fs.action == FILTER_SWITCH && 858346855Snp (t->fs.newvlan == VLAN_REMOVE || t->fs.newvlan == VLAN_REWRITE || 859346855Snp t->fs.swapmac || t->fs.nat_mode)) 860346855Snp return (ENOTSUP); 861346855Snp 862346855Snp if (t->fs.action == FILTER_SWITCH && t->fs.eport >= sc->params.nports) 863346855Snp return (EINVAL); 864346855Snp if (t->fs.val.iport >= sc->params.nports) 865346855Snp return (EINVAL); 866346855Snp 867346855Snp /* Can't specify an iq if not steering to it */ 868346855Snp if (!t->fs.dirsteer && t->fs.iq) 869346855Snp return (EINVAL); 870346855Snp 871346855Snp /* Validate against the global filter mode and ingress config */ 872346855Snp rc = check_fspec_against_fconf_iconf(sc, &t->fs); 873346855Snp if (rc != 0) 874346855Snp return (rc); 875346855Snp 876346855Snp /* 877346855Snp * Basic checks passed. Make sure the queues and tid tables are setup. 878346855Snp */ 879346855Snp 880333153Snp rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 881333153Snp if (rc) 882333153Snp return (rc); 883346855Snp if (!(sc->flags & FULL_INIT_DONE) && 884346855Snp ((rc = adapter_full_init(sc)) != 0)) { 885346855Snp end_synchronized_op(sc, 0); 886346855Snp return (rc); 887346855Snp } 888346855Snp if (t->fs.hash) { 889346877Snp if (__predict_false(ti->hftid_hash_4t == NULL)) { 890346877Snp rc = alloc_hftid_hash(&sc->tids, HASH_NOWAIT); 891346855Snp if (rc != 0) 892346855Snp goto done; 893346855Snp } 894346855Snp if (__predict_false(sc->tids.atid_tab == NULL)) { 895346855Snp rc = alloc_atid_tab(&sc->tids, M_NOWAIT); 896346855Snp if (rc != 0) 897346855Snp goto done; 898346855Snp } 899346874Snp } else if (separate_hpfilter_region(sc) && t->fs.prio && 900346874Snp __predict_false(ti->hpftid_tab == NULL)) { 901346874Snp MPASS(ti->nhpftids != 0); 902346874Snp KASSERT(ti->hpftids_in_use == 0, 903346874Snp ("%s: no memory allocated but hpftids_in_use is %u", 904346874Snp __func__, ti->hpftids_in_use)); 905346874Snp ti->hpftid_tab = malloc(sizeof(struct filter_entry) * 906346874Snp ti->nhpftids, M_CXGBE, M_NOWAIT | M_ZERO); 907346874Snp if (ti->hpftid_tab == NULL) { 908346874Snp rc = ENOMEM; 909346874Snp goto done; 910346874Snp } 911346874Snp if (!mtx_initialized(&sc->tids.ftid_lock)) { 912346874Snp mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF); 913346874Snp cv_init(&ti->ftid_cv, "t4fcv"); 914346874Snp } 915346855Snp } else if (__predict_false(ti->ftid_tab == NULL)) { 916346874Snp MPASS(ti->nftids != 0); 917346855Snp KASSERT(ti->ftids_in_use == 0, 918346874Snp ("%s: no memory allocated but ftids_in_use is %u", 919346874Snp __func__, ti->ftids_in_use)); 920346855Snp ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids, 921346855Snp M_CXGBE, M_NOWAIT | M_ZERO); 922346855Snp if (ti->ftid_tab == NULL) { 923346855Snp rc = ENOMEM; 924346855Snp goto done; 925346855Snp } 926346874Snp if (!mtx_initialized(&sc->tids.ftid_lock)) { 927346874Snp mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF); 928346874Snp cv_init(&ti->ftid_cv, "t4fcv"); 929346874Snp } 930346855Snp } 931346855Snpdone: 932346855Snp end_synchronized_op(sc, 0); 933346855Snp if (rc != 0) 934346855Snp return (rc); 935333153Snp 936346855Snp /* 937346855Snp * Allocate L2T entry, SMT entry, etc. 938346855Snp */ 939333153Snp 940346855Snp if (t->fs.newdmac || t->fs.newvlan) { 941346855Snp /* This filter needs an L2T entry; allocate one. */ 942346915Snp l2te = t4_l2t_alloc_switching(sc, t->fs.vlan, t->fs.eport, 943346855Snp t->fs.dmac); 944346915Snp if (__predict_false(l2te == NULL)) { 945346915Snp rc = EAGAIN; 946346915Snp goto error; 947346855Snp } 948333153Snp } 949333153Snp 950346855Snp if (t->fs.newsmac) { 951346855Snp /* This filter needs an SMT entry; allocate one. */ 952346855Snp smt = t4_smt_alloc_switching(sc->smt, t->fs.smac); 953346855Snp if (__predict_false(smt == NULL)) { 954346915Snp rc = EAGAIN; 955346915Snp goto error; 956346855Snp } 957346855Snp rc = t4_smt_set_switching(sc, smt, 0x0, t->fs.smac); 958346915Snp if (rc) 959346915Snp goto error; 960333153Snp } 961333153Snp 962346855Snp if (t->fs.hash) 963346915Snp rc = set_hashfilter(sc, t, ftuple, l2te, smt); 964346855Snp else 965346915Snp rc = set_tcamfilter(sc, t, l2te, smt); 966333153Snp 967346915Snp if (rc != 0 && rc != EINPROGRESS) { 968346915Snperror: 969346915Snp if (l2te) 970346915Snp t4_l2t_release(l2te); 971346915Snp if (smt) 972346915Snp t4_smt_release(smt); 973346915Snp } 974346915Snp return (rc); 975346855Snp} 976346855Snp 977346855Snpstatic int 978346855Snpdel_tcamfilter(struct adapter *sc, struct t4_filter *t) 979346855Snp{ 980346855Snp struct filter_entry *f; 981346855Snp struct fw_filter_wr *fwr; 982346855Snp struct wrq_cookie cookie; 983346874Snp int rc, nfilters; 984346874Snp#ifdef INVARIANTS 985346874Snp u_int tid_base; 986346874Snp#endif 987346855Snp 988346874Snp mtx_lock(&sc->tids.ftid_lock); 989346874Snp if (separate_hpfilter_region(sc) && t->fs.prio) { 990346874Snp nfilters = sc->tids.nhpftids; 991346874Snp f = sc->tids.hpftid_tab; 992346874Snp#ifdef INVARIANTS 993346874Snp tid_base = sc->tids.hpftid_base; 994346874Snp#endif 995346874Snp } else { 996346874Snp nfilters = sc->tids.nftids; 997346874Snp f = sc->tids.ftid_tab; 998346874Snp#ifdef INVARIANTS 999346874Snp tid_base = sc->tids.ftid_base; 1000346874Snp#endif 1001346874Snp } 1002346874Snp MPASS(f != NULL); /* Caller checked this. */ 1003346874Snp if (t->idx >= nfilters) { 1004346874Snp rc = EINVAL; 1005346874Snp goto done; 1006346874Snp } 1007346874Snp f += t->idx; 1008346855Snp 1009346855Snp if (f->locked) { 1010346855Snp rc = EPERM; 1011333153Snp goto done; 1012333153Snp } 1013346855Snp if (f->pending) { 1014346855Snp rc = EBUSY; 1015333153Snp goto done; 1016333153Snp } 1017346855Snp if (f->valid == 0) { 1018333153Snp rc = EINVAL; 1019333153Snp goto done; 1020333153Snp } 1021346874Snp MPASS(f->tid == tid_base + t->idx); 1022346876Snp fwr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*fwr), 16), &cookie); 1023346855Snp if (fwr == NULL) { 1024346855Snp rc = ENOMEM; 1025333153Snp goto done; 1026333153Snp } 1027333153Snp 1028346855Snp bzero(fwr, sizeof (*fwr)); 1029346855Snp t4_mk_filtdelwr(f->tid, fwr, sc->sge.fwq.abs_id); 1030346855Snp f->pending = 1; 1031346876Snp commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie); 1032346855Snp t->fs = f->fs; /* extra info for the caller */ 1033333153Snp 1034346855Snp for (;;) { 1035346855Snp if (f->pending == 0) { 1036346855Snp rc = f->valid ? EIO : 0; 1037346855Snp break; 1038333153Snp } 1039346855Snp if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) { 1040346855Snp rc = EINPROGRESS; 1041346855Snp break; 1042346855Snp } 1043333153Snp } 1044346855Snpdone: 1045346855Snp mtx_unlock(&sc->tids.ftid_lock); 1046346855Snp return (rc); 1047346855Snp} 1048333153Snp 1049346855Snpint 1050346855Snpdel_filter(struct adapter *sc, struct t4_filter *t) 1051346855Snp{ 1052333153Snp 1053346855Snp /* No filters possible if not initialized yet. */ 1054346855Snp if (!(sc->flags & FULL_INIT_DONE)) 1055346855Snp return (EINVAL); 1056333153Snp 1057346855Snp /* 1058346855Snp * The checks for tid tables ensure that the locks that del_* will reach 1059346855Snp * for are initialized. 1060346855Snp */ 1061346855Snp if (t->fs.hash) { 1062346877Snp if (sc->tids.hftid_hash_4t != NULL) 1063346855Snp return (del_hashfilter(sc, t)); 1064346874Snp } else if (separate_hpfilter_region(sc) && t->fs.prio) { 1065346874Snp if (sc->tids.hpftid_tab != NULL) 1066346874Snp return (del_tcamfilter(sc, t)); 1067346855Snp } else { 1068346855Snp if (sc->tids.ftid_tab != NULL) 1069346855Snp return (del_tcamfilter(sc, t)); 1070333153Snp } 1071333153Snp 1072346855Snp return (EINVAL); 1073346855Snp} 1074333153Snp 1075346855Snp/* 1076346855Snp * Release secondary resources associated with the filter. 1077346855Snp */ 1078346855Snpstatic void 1079346855Snpfree_filter_resources(struct filter_entry *f) 1080346855Snp{ 1081333153Snp 1082346855Snp if (f->l2te) { 1083346855Snp t4_l2t_release(f->l2te); 1084346855Snp f->l2te = NULL; 1085333153Snp } 1086346855Snp if (f->smt) { 1087346855Snp t4_smt_release(f->smt); 1088346855Snp f->smt = NULL; 1089346855Snp } 1090333153Snp} 1091333153Snp 1092333153Snpstatic int 1093346855Snpset_tcb_field(struct adapter *sc, u_int tid, uint16_t word, uint64_t mask, 1094346855Snp uint64_t val, int no_reply) 1095333153Snp{ 1096333153Snp struct wrq_cookie cookie; 1097346855Snp struct cpl_set_tcb_field *req; 1098333153Snp 1099346876Snp req = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*req), 16), &cookie); 1100346855Snp if (req == NULL) 1101333153Snp return (ENOMEM); 1102346855Snp bzero(req, sizeof(*req)); 1103346855Snp INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid); 1104346855Snp if (no_reply == 0) { 1105346855Snp req->reply_ctrl = htobe16(V_QUEUENO(sc->sge.fwq.abs_id) | 1106346855Snp V_NO_REPLY(0)); 1107346855Snp } else 1108346855Snp req->reply_ctrl = htobe16(V_NO_REPLY(1)); 1109346855Snp req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(CPL_COOKIE_HASHFILTER)); 1110346855Snp req->mask = htobe64(mask); 1111346855Snp req->val = htobe64(val); 1112346876Snp commit_wrq_wr(&sc->sge.ctrlq[0], req, &cookie); 1113333153Snp 1114346855Snp return (0); 1115346855Snp} 1116333153Snp 1117346855Snp/* Set one of the t_flags bits in the TCB. */ 1118346855Snpstatic inline int 1119346855Snpset_tcb_tflag(struct adapter *sc, int tid, u_int bit_pos, u_int val, 1120346855Snp u_int no_reply) 1121346855Snp{ 1122346855Snp 1123346855Snp return (set_tcb_field(sc, tid, W_TCB_T_FLAGS, 1ULL << bit_pos, 1124346855Snp (uint64_t)val << bit_pos, no_reply)); 1125346855Snp} 1126346855Snp 1127346855Snpint 1128346855Snpt4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1129346855Snp{ 1130346855Snp struct adapter *sc = iq->adapter; 1131346855Snp const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 1132346855Snp u_int tid = GET_TID(rpl); 1133346874Snp u_int rc, idx; 1134346855Snp struct filter_entry *f; 1135346855Snp 1136346855Snp KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 1137346855Snp rss->opcode)); 1138346855Snp 1139346874Snp 1140346874Snp if (is_hpftid(sc, tid)) { 1141346874Snp idx = tid - sc->tids.hpftid_base; 1142346874Snp f = &sc->tids.hpftid_tab[idx]; 1143346874Snp } else if (is_ftid(sc, tid)) { 1144346874Snp idx = tid - sc->tids.ftid_base; 1145346874Snp f = &sc->tids.ftid_tab[idx]; 1146346874Snp } else 1147346874Snp panic("%s: FW reply for invalid TID %d.", __func__, tid); 1148346874Snp 1149346874Snp MPASS(f->tid == tid); 1150346855Snp rc = G_COOKIE(rpl->cookie); 1151346855Snp 1152346855Snp mtx_lock(&sc->tids.ftid_lock); 1153346855Snp KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.", 1154346874Snp __func__, rc, tid)); 1155346855Snp switch(rc) { 1156346855Snp case FW_FILTER_WR_FLT_ADDED: 1157346855Snp /* set-filter succeeded */ 1158346855Snp f->valid = 1; 1159346855Snp if (f->fs.newsmac) { 1160346855Snp MPASS(f->smt != NULL); 1161346855Snp set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1); 1162346855Snp set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL, 1163346855Snp V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), 1164346855Snp V_TCB_SMAC_SEL(f->smt->idx), 1); 1165346855Snp /* XXX: wait for reply to TCB update before !pending */ 1166346855Snp } 1167346855Snp break; 1168346855Snp case FW_FILTER_WR_FLT_DELETED: 1169346855Snp /* del-filter succeeded */ 1170346855Snp MPASS(f->valid == 1); 1171346855Snp f->valid = 0; 1172346855Snp /* Fall through */ 1173346855Snp case FW_FILTER_WR_SMT_TBL_FULL: 1174346855Snp /* set-filter failed due to lack of SMT space. */ 1175346855Snp MPASS(f->valid == 0); 1176346855Snp free_filter_resources(f); 1177346874Snp if (separate_hpfilter_region(sc) && f->fs.prio) 1178346874Snp sc->tids.hpftids_in_use--; 1179346874Snp else 1180346874Snp sc->tids.ftids_in_use--; 1181346855Snp break; 1182346855Snp case FW_FILTER_WR_SUCCESS: 1183346855Snp case FW_FILTER_WR_EINVAL: 1184346855Snp default: 1185346855Snp panic("%s: unexpected reply %d for filter[%d].", __func__, rc, 1186346855Snp idx); 1187346855Snp } 1188346855Snp f->pending = 0; 1189346855Snp cv_broadcast(&sc->tids.ftid_cv); 1190346855Snp mtx_unlock(&sc->tids.ftid_lock); 1191346855Snp 1192333153Snp return (0); 1193333153Snp} 1194333153Snp 1195346855Snp/* 1196346855Snp * This is the reply to the Active Open that created the filter. Additional TCB 1197346855Snp * updates may be required to complete the filter configuration. 1198346855Snp */ 1199333153Snpint 1200346855Snpt4_hashfilter_ao_rpl(struct sge_iq *iq, const struct rss_header *rss, 1201346855Snp struct mbuf *m) 1202333153Snp{ 1203346855Snp struct adapter *sc = iq->adapter; 1204346855Snp const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); 1205346855Snp u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status))); 1206346855Snp u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status)); 1207346855Snp struct filter_entry *f = lookup_atid(sc, atid); 1208346855Snp 1209346855Snp KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1210346855Snp 1211346855Snp mtx_lock(&sc->tids.hftid_lock); 1212346855Snp KASSERT(f->pending, ("%s: hashfilter[%p] isn't pending.", __func__, f)); 1213346855Snp KASSERT(f->tid == -1, ("%s: hashfilter[%p] has tid %d already.", 1214346855Snp __func__, f, f->tid)); 1215346855Snp if (status == CPL_ERR_NONE) { 1216346855Snp f->tid = GET_TID(cpl); 1217346877Snp MPASS(lookup_hftid(sc, f->tid) == NULL); 1218346877Snp insert_hftid(sc, f); 1219346855Snp /* 1220346855Snp * Leave the filter pending until it is fully set up, which will 1221346855Snp * be indicated by the reply to the last TCB update. No need to 1222346855Snp * unblock the ioctl thread either. 1223346855Snp */ 1224346855Snp if (configure_hashfilter_tcb(sc, f) == EINPROGRESS) 1225346855Snp goto done; 1226346855Snp f->valid = 1; 1227346855Snp f->pending = 0; 1228346855Snp } else { 1229346855Snp /* provide errno instead of tid to ioctl */ 1230346855Snp f->tid = act_open_rpl_status_to_errno(status); 1231346855Snp f->valid = 0; 1232346951Snp f->pending = 0; 1233346855Snp if (act_open_has_tid(status)) 1234346876Snp release_tid(sc, GET_TID(cpl), &sc->sge.ctrlq[0]); 1235346855Snp free_filter_resources(f); 1236346877Snp remove_hf(sc, f); 1237346855Snp if (f->locked == 0) 1238346855Snp free(f, M_CXGBE); 1239346855Snp } 1240346855Snp cv_broadcast(&sc->tids.hftid_cv); 1241346855Snpdone: 1242346855Snp mtx_unlock(&sc->tids.hftid_lock); 1243346855Snp 1244346855Snp free_atid(sc, atid); 1245346855Snp return (0); 1246346855Snp} 1247346855Snp 1248346855Snpint 1249346855Snpt4_hashfilter_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, 1250346855Snp struct mbuf *m) 1251346855Snp{ 1252346855Snp struct adapter *sc = iq->adapter; 1253346855Snp const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 1254346855Snp u_int tid = GET_TID(rpl); 1255333153Snp struct filter_entry *f; 1256333153Snp 1257346855Snp mtx_lock(&sc->tids.hftid_lock); 1258346855Snp f = lookup_hftid(sc, tid); 1259346855Snp KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__)); 1260346855Snp KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__, 1261346855Snp f, tid)); 1262346855Snp KASSERT(f->valid == 0, ("%s: hashfilter %p [%u] is valid already.", 1263346855Snp __func__, f, tid)); 1264346855Snp f->pending = 0; 1265346855Snp if (rpl->status == 0) { 1266346855Snp f->valid = 1; 1267346855Snp } else { 1268346855Snp f->tid = EIO; 1269346855Snp f->valid = 0; 1270346855Snp free_filter_resources(f); 1271346877Snp remove_hftid(sc, f); 1272346877Snp remove_hf(sc, f); 1273346876Snp release_tid(sc, tid, &sc->sge.ctrlq[0]); 1274346855Snp if (f->locked == 0) 1275346855Snp free(f, M_CXGBE); 1276346855Snp } 1277346855Snp cv_broadcast(&sc->tids.hftid_cv); 1278346855Snp mtx_unlock(&sc->tids.hftid_lock); 1279333153Snp 1280346855Snp return (0); 1281346855Snp} 1282333153Snp 1283346855Snpint 1284346855Snpt4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss, 1285346855Snp struct mbuf *m) 1286346855Snp{ 1287346855Snp struct adapter *sc = iq->adapter; 1288346855Snp const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); 1289346855Snp unsigned int tid = GET_TID(cpl); 1290346855Snp struct filter_entry *f; 1291346855Snp 1292346855Snp mtx_lock(&sc->tids.hftid_lock); 1293346855Snp f = lookup_hftid(sc, tid); 1294346855Snp KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__)); 1295346855Snp KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__, 1296346855Snp f, tid)); 1297346855Snp KASSERT(f->valid, ("%s: hashfilter %p [%u] isn't valid.", __func__, f, 1298346855Snp tid)); 1299346855Snp f->pending = 0; 1300346855Snp if (cpl->status == 0) { 1301346855Snp f->valid = 0; 1302346855Snp free_filter_resources(f); 1303346877Snp remove_hftid(sc, f); 1304346877Snp remove_hf(sc, f); 1305346876Snp release_tid(sc, tid, &sc->sge.ctrlq[0]); 1306346855Snp if (f->locked == 0) 1307346855Snp free(f, M_CXGBE); 1308333153Snp } 1309346855Snp cv_broadcast(&sc->tids.hftid_cv); 1310346855Snp mtx_unlock(&sc->tids.hftid_lock); 1311333153Snp 1312346855Snp return (0); 1313346855Snp} 1314346855Snp 1315346855Snpstatic int 1316346874Snpget_tcamfilter(struct adapter *sc, struct t4_filter *t) 1317346874Snp{ 1318346874Snp int i, nfilters; 1319346874Snp struct filter_entry *f; 1320346874Snp u_int in_use; 1321346874Snp#ifdef INVARIANTS 1322346874Snp u_int tid_base; 1323346874Snp#endif 1324346874Snp 1325346874Snp MPASS(!t->fs.hash); 1326346874Snp 1327346874Snp if (separate_hpfilter_region(sc) && t->fs.prio) { 1328346874Snp nfilters = sc->tids.nhpftids; 1329346874Snp f = sc->tids.hpftid_tab; 1330346874Snp in_use = sc->tids.hpftids_in_use; 1331346874Snp#ifdef INVARIANTS 1332346874Snp tid_base = sc->tids.hpftid_base; 1333346874Snp#endif 1334346874Snp } else { 1335346874Snp nfilters = sc->tids.nftids; 1336346874Snp f = sc->tids.ftid_tab; 1337346874Snp in_use = sc->tids.ftids_in_use; 1338346874Snp#ifdef INVARIANTS 1339346874Snp tid_base = sc->tids.ftid_base; 1340346874Snp#endif 1341346874Snp } 1342346874Snp 1343346874Snp if (in_use == 0 || f == NULL || t->idx >= nfilters) { 1344346874Snp t->idx = 0xffffffff; 1345346874Snp return (0); 1346346874Snp } 1347346874Snp 1348346874Snp f += t->idx; 1349346874Snp mtx_lock(&sc->tids.ftid_lock); 1350346874Snp for (i = t->idx; i < nfilters; i++, f++) { 1351346874Snp if (f->valid) { 1352346874Snp MPASS(f->tid == tid_base + i); 1353346874Snp t->idx = i; 1354346874Snp t->l2tidx = f->l2te ? f->l2te->idx : 0; 1355346874Snp t->smtidx = f->smt ? f->smt->idx : 0; 1356346874Snp if (f->fs.hitcnts) 1357346874Snp t->hits = get_filter_hits(sc, f->tid); 1358346874Snp else 1359346874Snp t->hits = UINT64_MAX; 1360346874Snp t->fs = f->fs; 1361346874Snp 1362346874Snp goto done; 1363346874Snp } 1364346874Snp } 1365346874Snp t->idx = 0xffffffff; 1366346874Snpdone: 1367346874Snp mtx_unlock(&sc->tids.ftid_lock); 1368346874Snp return (0); 1369346874Snp} 1370346874Snp 1371346874Snpstatic int 1372346855Snpget_hashfilter(struct adapter *sc, struct t4_filter *t) 1373346855Snp{ 1374346877Snp struct tid_info *ti = &sc->tids; 1375346877Snp int tid; 1376346855Snp struct filter_entry *f; 1377346877Snp const int inv_tid = ti->ntids + ti->tid_base; 1378346855Snp 1379346874Snp MPASS(t->fs.hash); 1380346874Snp 1381346877Snp if (ti->tids_in_use == 0 || ti->hftid_hash_tid == NULL || 1382346877Snp t->idx >= inv_tid) { 1383346855Snp t->idx = 0xffffffff; 1384346855Snp return (0); 1385346855Snp } 1386346877Snp if (t->idx < ti->tid_base) 1387346877Snp t->idx = ti->tid_base; 1388346855Snp 1389346877Snp mtx_lock(&ti->hftid_lock); 1390346877Snp for (tid = t->idx; tid < inv_tid; tid++) { 1391346877Snp f = lookup_hftid(sc, tid); 1392346855Snp if (f != NULL && f->valid) { 1393346877Snp t->idx = tid; 1394346855Snp t->l2tidx = f->l2te ? f->l2te->idx : 0; 1395346855Snp t->smtidx = f->smt ? f->smt->idx : 0; 1396346855Snp if (f->fs.hitcnts) 1397346877Snp t->hits = get_filter_hits(sc, tid); 1398346855Snp else 1399346855Snp t->hits = UINT64_MAX; 1400346855Snp t->fs = f->fs; 1401346855Snp 1402346855Snp goto done; 1403346855Snp } 1404346855Snp } 1405346855Snp t->idx = 0xffffffff; 1406346855Snpdone: 1407346877Snp mtx_unlock(&ti->hftid_lock); 1408346855Snp return (0); 1409346855Snp} 1410346855Snp 1411346855Snpstatic void 1412346855Snpmk_act_open_req6(struct adapter *sc, struct filter_entry *f, int atid, 1413346855Snp uint64_t ftuple, struct cpl_act_open_req6 *cpl) 1414346855Snp{ 1415346855Snp struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl; 1416346855Snp struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl; 1417346855Snp 1418346855Snp /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */ 1419346855Snp MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6); 1420346855Snp MPASS(atid >= 0); 1421346855Snp 1422346855Snp if (chip_id(sc) == CHELSIO_T5) { 1423346855Snp INIT_TP_WR(cpl5, 0); 1424346855Snp } else { 1425346855Snp INIT_TP_WR(cpl6, 0); 1426346855Snp cpl6->rsvd2 = 0; 1427346855Snp cpl6->opt3 = 0; 1428346855Snp } 1429346855Snp 1430346855Snp OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 1431346855Snp V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | 1432346855Snp V_TID_COOKIE(CPL_COOKIE_HASHFILTER))); 1433346855Snp cpl->local_port = htobe16(f->fs.val.dport); 1434346855Snp cpl->peer_port = htobe16(f->fs.val.sport); 1435346855Snp cpl->local_ip_hi = *(uint64_t *)(&f->fs.val.dip); 1436346855Snp cpl->local_ip_lo = *(((uint64_t *)&f->fs.val.dip) + 1); 1437346855Snp cpl->peer_ip_hi = *(uint64_t *)(&f->fs.val.sip); 1438346855Snp cpl->peer_ip_lo = *(((uint64_t *)&f->fs.val.sip) + 1); 1439346855Snp cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE || 1440346855Snp f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) | 1441346855Snp V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) | 1442346855Snp V_NO_CONG(f->fs.rpttid) | 1443346855Snp V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) | 1444346855Snp F_TCAM_BYPASS | F_NON_OFFLOAD); 1445346855Snp 1446346855Snp cpl6->params = htobe64(V_FILTER_TUPLE(ftuple)); 1447346855Snp cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) | 1448346855Snp V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) | 1449346855Snp V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID | 1450346855Snp F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) | 1451346855Snp V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) | 1452346855Snp V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1))); 1453346855Snp} 1454346855Snp 1455346855Snpstatic void 1456346855Snpmk_act_open_req(struct adapter *sc, struct filter_entry *f, int atid, 1457346855Snp uint64_t ftuple, struct cpl_act_open_req *cpl) 1458346855Snp{ 1459346855Snp struct cpl_t5_act_open_req *cpl5 = (void *)cpl; 1460346855Snp struct cpl_t6_act_open_req *cpl6 = (void *)cpl; 1461346855Snp 1462346855Snp /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */ 1463346855Snp MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6); 1464346855Snp MPASS(atid >= 0); 1465346855Snp 1466346855Snp if (chip_id(sc) == CHELSIO_T5) { 1467346855Snp INIT_TP_WR(cpl5, 0); 1468346855Snp } else { 1469346855Snp INIT_TP_WR(cpl6, 0); 1470346855Snp cpl6->rsvd2 = 0; 1471346855Snp cpl6->opt3 = 0; 1472346855Snp } 1473346855Snp 1474346855Snp OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 1475346855Snp V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | 1476346855Snp V_TID_COOKIE(CPL_COOKIE_HASHFILTER))); 1477346855Snp cpl->local_port = htobe16(f->fs.val.dport); 1478346855Snp cpl->peer_port = htobe16(f->fs.val.sport); 1479346855Snp cpl->local_ip = f->fs.val.dip[0] | f->fs.val.dip[1] << 8 | 1480346855Snp f->fs.val.dip[2] << 16 | f->fs.val.dip[3] << 24; 1481346855Snp cpl->peer_ip = f->fs.val.sip[0] | f->fs.val.sip[1] << 8 | 1482346855Snp f->fs.val.sip[2] << 16 | f->fs.val.sip[3] << 24; 1483346855Snp cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE || 1484346855Snp f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) | 1485346855Snp V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) | 1486346855Snp V_NO_CONG(f->fs.rpttid) | 1487346855Snp V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) | 1488346855Snp F_TCAM_BYPASS | F_NON_OFFLOAD); 1489346855Snp 1490346855Snp cpl6->params = htobe64(V_FILTER_TUPLE(ftuple)); 1491346855Snp cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) | 1492346855Snp V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) | 1493346855Snp V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID | 1494346855Snp F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) | 1495346855Snp V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) | 1496346855Snp V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1))); 1497346855Snp} 1498346855Snp 1499346855Snpstatic int 1500346855Snpact_open_cpl_len16(struct adapter *sc, int isipv6) 1501346855Snp{ 1502346855Snp int idx; 1503346855Snp static const int sz_table[3][2] = { 1504346855Snp { 1505346855Snp howmany(sizeof (struct cpl_act_open_req), 16), 1506346855Snp howmany(sizeof (struct cpl_act_open_req6), 16) 1507346855Snp }, 1508346855Snp { 1509346855Snp howmany(sizeof (struct cpl_t5_act_open_req), 16), 1510346855Snp howmany(sizeof (struct cpl_t5_act_open_req6), 16) 1511346855Snp }, 1512346855Snp { 1513346855Snp howmany(sizeof (struct cpl_t6_act_open_req), 16), 1514346855Snp howmany(sizeof (struct cpl_t6_act_open_req6), 16) 1515346855Snp }, 1516346855Snp }; 1517346855Snp 1518346855Snp MPASS(chip_id(sc) >= CHELSIO_T4); 1519346855Snp idx = min(chip_id(sc) - CHELSIO_T4, 2); 1520346855Snp 1521346855Snp return (sz_table[idx][!!isipv6]); 1522346855Snp} 1523346855Snp 1524346855Snpstatic int 1525346855Snpset_hashfilter(struct adapter *sc, struct t4_filter *t, uint64_t ftuple, 1526346855Snp struct l2t_entry *l2te, struct smt_entry *smt) 1527346855Snp{ 1528346855Snp void *wr; 1529346855Snp struct wrq_cookie cookie; 1530346855Snp struct filter_entry *f; 1531346855Snp int rc, atid = -1; 1532346877Snp uint32_t hash; 1533346855Snp 1534346855Snp MPASS(t->fs.hash); 1535346855Snp /* Already validated against fconf, iconf */ 1536346855Snp MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0); 1537346855Snp MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0); 1538346855Snp 1539346877Snp hash = hf_hashfn_4t(&t->fs); 1540346877Snp 1541346855Snp mtx_lock(&sc->tids.hftid_lock); 1542346877Snp if (lookup_hf(sc, &t->fs, hash) != NULL) { 1543346877Snp rc = EEXIST; 1544346877Snp goto done; 1545346877Snp } 1546346855Snp 1547346855Snp f = malloc(sizeof(*f), M_CXGBE, M_ZERO | M_NOWAIT); 1548346855Snp if (__predict_false(f == NULL)) { 1549346855Snp rc = ENOMEM; 1550333153Snp goto done; 1551333153Snp } 1552346855Snp f->fs = t->fs; 1553346855Snp f->l2te = l2te; 1554346855Snp f->smt = smt; 1555333153Snp 1556346855Snp atid = alloc_atid(sc, f); 1557346855Snp if (__predict_false(atid) == -1) { 1558346855Snp free(f, M_CXGBE); 1559333153Snp rc = EAGAIN; 1560333153Snp goto done; 1561333153Snp } 1562346855Snp MPASS(atid >= 0); 1563333153Snp 1564346876Snp wr = start_wrq_wr(&sc->sge.ctrlq[0], act_open_cpl_len16(sc, f->fs.type), 1565346855Snp &cookie); 1566346855Snp if (wr == NULL) { 1567346855Snp free_atid(sc, atid); 1568346855Snp free(f, M_CXGBE); 1569346855Snp rc = ENOMEM; 1570346855Snp goto done; 1571346855Snp } 1572346855Snp if (f->fs.type) 1573346855Snp mk_act_open_req6(sc, f, atid, ftuple, wr); 1574346855Snp else 1575346855Snp mk_act_open_req(sc, f, atid, ftuple, wr); 1576333153Snp 1577346855Snp f->locked = 1; /* ithread mustn't free f if ioctl is still around. */ 1578346855Snp f->pending = 1; 1579346855Snp f->tid = -1; 1580346877Snp insert_hf(sc, f, hash); 1581346876Snp commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie); 1582346855Snp 1583346855Snp for (;;) { 1584346855Snp MPASS(f->locked); 1585346855Snp if (f->pending == 0) { 1586346855Snp if (f->valid) { 1587346855Snp rc = 0; 1588346855Snp f->locked = 0; 1589346855Snp t->idx = f->tid; 1590346855Snp } else { 1591346855Snp rc = f->tid; 1592346855Snp free(f, M_CXGBE); 1593346855Snp } 1594346855Snp break; 1595346855Snp } 1596346855Snp if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) { 1597346855Snp f->locked = 0; 1598346855Snp rc = EINPROGRESS; 1599346855Snp break; 1600346855Snp } 1601346855Snp } 1602346855Snpdone: 1603346855Snp mtx_unlock(&sc->tids.hftid_lock); 1604346855Snp return (rc); 1605346855Snp} 1606346855Snp 1607346855Snp/* SET_TCB_FIELD sent as a ULP command looks like this */ 1608346855Snp#define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ 1609346855Snp sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) 1610346855Snp 1611346855Snpstatic void * 1612346855Snpmk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, uint64_t word, uint64_t mask, 1613346855Snp uint64_t val, uint32_t tid, uint32_t qid) 1614346855Snp{ 1615346855Snp struct ulptx_idata *ulpsc; 1616346855Snp struct cpl_set_tcb_field_core *req; 1617346855Snp 1618346855Snp ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 1619346855Snp ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16)); 1620346855Snp 1621346855Snp ulpsc = (struct ulptx_idata *)(ulpmc + 1); 1622346855Snp ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 1623346855Snp ulpsc->len = htobe32(sizeof(*req)); 1624346855Snp 1625346855Snp req = (struct cpl_set_tcb_field_core *)(ulpsc + 1); 1626346855Snp OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1627346855Snp req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(qid)); 1628346855Snp req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0)); 1629346855Snp req->mask = htobe64(mask); 1630346855Snp req->val = htobe64(val); 1631346855Snp 1632346855Snp ulpsc = (struct ulptx_idata *)(req + 1); 1633346855Snp if (LEN__SET_TCB_FIELD_ULP % 16) { 1634346855Snp ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1635346855Snp ulpsc->len = htobe32(0); 1636346855Snp return (ulpsc + 1); 1637346855Snp } 1638346855Snp return (ulpsc); 1639346855Snp} 1640346855Snp 1641346855Snp/* ABORT_REQ sent as a ULP command looks like this */ 1642346855Snp#define LEN__ABORT_REQ_ULP (sizeof(struct ulp_txpkt) + \ 1643346855Snp sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_req_core)) 1644346855Snp 1645346855Snpstatic void * 1646346855Snpmk_abort_req_ulp(struct ulp_txpkt *ulpmc, uint32_t tid) 1647346855Snp{ 1648346855Snp struct ulptx_idata *ulpsc; 1649346855Snp struct cpl_abort_req_core *req; 1650346855Snp 1651346855Snp ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 1652346855Snp ulpmc->len = htobe32(howmany(LEN__ABORT_REQ_ULP, 16)); 1653346855Snp 1654346855Snp ulpsc = (struct ulptx_idata *)(ulpmc + 1); 1655346855Snp ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 1656346855Snp ulpsc->len = htobe32(sizeof(*req)); 1657346855Snp 1658346855Snp req = (struct cpl_abort_req_core *)(ulpsc + 1); 1659346855Snp OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); 1660346855Snp req->rsvd0 = htonl(0); 1661346855Snp req->rsvd1 = 0; 1662346855Snp req->cmd = CPL_ABORT_NO_RST; 1663346855Snp 1664346855Snp ulpsc = (struct ulptx_idata *)(req + 1); 1665346855Snp if (LEN__ABORT_REQ_ULP % 16) { 1666346855Snp ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1667346855Snp ulpsc->len = htobe32(0); 1668346855Snp return (ulpsc + 1); 1669346855Snp } 1670346855Snp return (ulpsc); 1671346855Snp} 1672346855Snp 1673346855Snp/* ABORT_RPL sent as a ULP command looks like this */ 1674346855Snp#define LEN__ABORT_RPL_ULP (sizeof(struct ulp_txpkt) + \ 1675346855Snp sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_rpl_core)) 1676346855Snp 1677346855Snpstatic void * 1678346855Snpmk_abort_rpl_ulp(struct ulp_txpkt *ulpmc, uint32_t tid) 1679346855Snp{ 1680346855Snp struct ulptx_idata *ulpsc; 1681346855Snp struct cpl_abort_rpl_core *rpl; 1682346855Snp 1683346855Snp ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 1684346855Snp ulpmc->len = htobe32(howmany(LEN__ABORT_RPL_ULP, 16)); 1685346855Snp 1686346855Snp ulpsc = (struct ulptx_idata *)(ulpmc + 1); 1687346855Snp ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 1688346855Snp ulpsc->len = htobe32(sizeof(*rpl)); 1689346855Snp 1690346855Snp rpl = (struct cpl_abort_rpl_core *)(ulpsc + 1); 1691346855Snp OPCODE_TID(rpl) = htobe32(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); 1692346855Snp rpl->rsvd0 = htonl(0); 1693346855Snp rpl->rsvd1 = 0; 1694346855Snp rpl->cmd = CPL_ABORT_NO_RST; 1695346855Snp 1696346855Snp ulpsc = (struct ulptx_idata *)(rpl + 1); 1697346855Snp if (LEN__ABORT_RPL_ULP % 16) { 1698346855Snp ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1699346855Snp ulpsc->len = htobe32(0); 1700346855Snp return (ulpsc + 1); 1701346855Snp } 1702346855Snp return (ulpsc); 1703346855Snp} 1704346855Snp 1705346855Snpstatic inline int 1706346855Snpdel_hashfilter_wrlen(void) 1707346855Snp{ 1708346855Snp 1709346855Snp return (sizeof(struct work_request_hdr) + 1710346855Snp roundup2(LEN__SET_TCB_FIELD_ULP, 16) + 1711346855Snp roundup2(LEN__ABORT_REQ_ULP, 16) + 1712346855Snp roundup2(LEN__ABORT_RPL_ULP, 16)); 1713346855Snp} 1714346855Snp 1715346855Snpstatic void 1716346855Snpmk_del_hashfilter_wr(int tid, struct work_request_hdr *wrh, int wrlen, int qid) 1717346855Snp{ 1718346855Snp struct ulp_txpkt *ulpmc; 1719346855Snp 1720346855Snp INIT_ULPTX_WRH(wrh, wrlen, 0, 0); 1721346855Snp ulpmc = (struct ulp_txpkt *)(wrh + 1); 1722346855Snp ulpmc = mk_set_tcb_field_ulp(ulpmc, W_TCB_RSS_INFO, 1723346855Snp V_TCB_RSS_INFO(M_TCB_RSS_INFO), V_TCB_RSS_INFO(qid), tid, 0); 1724346855Snp ulpmc = mk_abort_req_ulp(ulpmc, tid); 1725346855Snp ulpmc = mk_abort_rpl_ulp(ulpmc, tid); 1726346855Snp} 1727346855Snp 1728346855Snpstatic int 1729346855Snpdel_hashfilter(struct adapter *sc, struct t4_filter *t) 1730346855Snp{ 1731346877Snp struct tid_info *ti = &sc->tids; 1732346855Snp void *wr; 1733346855Snp struct filter_entry *f; 1734346855Snp struct wrq_cookie cookie; 1735346855Snp int rc; 1736346855Snp const int wrlen = del_hashfilter_wrlen(); 1737346877Snp const int inv_tid = ti->ntids + ti->tid_base; 1738346855Snp 1739346877Snp MPASS(sc->tids.hftid_hash_4t != NULL); 1740346855Snp MPASS(sc->tids.ntids > 0); 1741346855Snp 1742346877Snp if (t->idx < sc->tids.tid_base || t->idx >= inv_tid) 1743346855Snp return (EINVAL); 1744346855Snp 1745346877Snp mtx_lock(&ti->hftid_lock); 1746346855Snp f = lookup_hftid(sc, t->idx); 1747346855Snp if (f == NULL || f->valid == 0) { 1748346855Snp rc = EINVAL; 1749333153Snp goto done; 1750333153Snp } 1751346855Snp MPASS(f->tid == t->idx); 1752333153Snp if (f->locked) { 1753333153Snp rc = EPERM; 1754333153Snp goto done; 1755333153Snp } 1756346855Snp if (f->pending) { 1757346855Snp rc = EBUSY; 1758346855Snp goto done; 1759333153Snp } 1760346876Snp wr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(wrlen, 16), &cookie); 1761346855Snp if (wr == NULL) { 1762346855Snp rc = ENOMEM; 1763346855Snp goto done; 1764346855Snp } 1765333153Snp 1766346855Snp mk_del_hashfilter_wr(t->idx, wr, wrlen, sc->sge.fwq.abs_id); 1767346855Snp f->locked = 1; 1768346855Snp f->pending = 1; 1769346876Snp commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie); 1770346855Snp t->fs = f->fs; /* extra info for the caller */ 1771333153Snp 1772346855Snp for (;;) { 1773346855Snp MPASS(f->locked); 1774346855Snp if (f->pending == 0) { 1775346855Snp if (f->valid) { 1776346855Snp f->locked = 0; 1777346855Snp rc = EIO; 1778346855Snp } else { 1779346855Snp rc = 0; 1780346855Snp free(f, M_CXGBE); 1781333153Snp } 1782346855Snp break; 1783333153Snp } 1784346877Snp if (cv_wait_sig(&ti->hftid_cv, &ti->hftid_lock) != 0) { 1785346855Snp f->locked = 0; 1786346855Snp rc = EINPROGRESS; 1787346855Snp break; 1788346855Snp } 1789333153Snp } 1790346855Snpdone: 1791346877Snp mtx_unlock(&ti->hftid_lock); 1792333153Snp return (rc); 1793333153Snp} 1794333153Snp 1795346855Snp#define WORD_MASK 0xffffffff 1796333153Snpstatic void 1797346855Snpset_nat_params(struct adapter *sc, struct filter_entry *f, const bool dip, 1798346855Snp const bool sip, const bool dp, const bool sp) 1799333153Snp{ 1800333153Snp 1801346855Snp if (dip) { 1802346855Snp if (f->fs.type) { 1803346855Snp set_tcb_field(sc, f->tid, W_TCB_SND_UNA_RAW, WORD_MASK, 1804346855Snp f->fs.nat_dip[15] | f->fs.nat_dip[14] << 8 | 1805346855Snp f->fs.nat_dip[13] << 16 | f->fs.nat_dip[12] << 24, 1); 1806346855Snp 1807346855Snp set_tcb_field(sc, f->tid, 1808346855Snp W_TCB_SND_UNA_RAW + 1, WORD_MASK, 1809346855Snp f->fs.nat_dip[11] | f->fs.nat_dip[10] << 8 | 1810346855Snp f->fs.nat_dip[9] << 16 | f->fs.nat_dip[8] << 24, 1); 1811346855Snp 1812346855Snp set_tcb_field(sc, f->tid, 1813346855Snp W_TCB_SND_UNA_RAW + 2, WORD_MASK, 1814346855Snp f->fs.nat_dip[7] | f->fs.nat_dip[6] << 8 | 1815346855Snp f->fs.nat_dip[5] << 16 | f->fs.nat_dip[4] << 24, 1); 1816346855Snp 1817346855Snp set_tcb_field(sc, f->tid, 1818346855Snp W_TCB_SND_UNA_RAW + 3, WORD_MASK, 1819346855Snp f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 | 1820346855Snp f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1); 1821346855Snp } else { 1822346855Snp set_tcb_field(sc, f->tid, 1823346855Snp W_TCB_RX_FRAG3_LEN_RAW, WORD_MASK, 1824346855Snp f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 | 1825346855Snp f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1); 1826346855Snp } 1827346855Snp } 1828346855Snp 1829346855Snp if (sip) { 1830346855Snp if (f->fs.type) { 1831346855Snp set_tcb_field(sc, f->tid, 1832346855Snp W_TCB_RX_FRAG2_PTR_RAW, WORD_MASK, 1833346855Snp f->fs.nat_sip[15] | f->fs.nat_sip[14] << 8 | 1834346855Snp f->fs.nat_sip[13] << 16 | f->fs.nat_sip[12] << 24, 1); 1835346855Snp 1836346855Snp set_tcb_field(sc, f->tid, 1837346855Snp W_TCB_RX_FRAG2_PTR_RAW + 1, WORD_MASK, 1838346855Snp f->fs.nat_sip[11] | f->fs.nat_sip[10] << 8 | 1839346855Snp f->fs.nat_sip[9] << 16 | f->fs.nat_sip[8] << 24, 1); 1840346855Snp 1841346855Snp set_tcb_field(sc, f->tid, 1842346855Snp W_TCB_RX_FRAG2_PTR_RAW + 2, WORD_MASK, 1843346855Snp f->fs.nat_sip[7] | f->fs.nat_sip[6] << 8 | 1844346855Snp f->fs.nat_sip[5] << 16 | f->fs.nat_sip[4] << 24, 1); 1845346855Snp 1846346855Snp set_tcb_field(sc, f->tid, 1847346855Snp W_TCB_RX_FRAG2_PTR_RAW + 3, WORD_MASK, 1848346855Snp f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 | 1849346855Snp f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1); 1850346855Snp 1851346855Snp } else { 1852346855Snp set_tcb_field(sc, f->tid, 1853346855Snp W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW, WORD_MASK, 1854346855Snp f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 | 1855346855Snp f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1); 1856346855Snp } 1857346855Snp } 1858346855Snp 1859346855Snp set_tcb_field(sc, f->tid, W_TCB_PDU_HDR_LEN, WORD_MASK, 1860346855Snp (dp ? f->fs.nat_dport : 0) | (sp ? f->fs.nat_sport << 16 : 0), 1); 1861333153Snp} 1862333153Snp 1863346855Snp/* 1864346855Snp * Returns EINPROGRESS to indicate that at least one TCB update was sent and the 1865346855Snp * last of the series of updates requested a reply. The reply informs the 1866346855Snp * driver that the filter is fully setup. 1867346855Snp */ 1868346855Snpstatic int 1869346855Snpconfigure_hashfilter_tcb(struct adapter *sc, struct filter_entry *f) 1870333153Snp{ 1871346855Snp int updated = 0; 1872333153Snp 1873346855Snp MPASS(f->tid < sc->tids.ntids); 1874346855Snp MPASS(f->fs.hash); 1875346855Snp MPASS(f->pending); 1876346855Snp MPASS(f->valid == 0); 1877333153Snp 1878346855Snp if (f->fs.newdmac) { 1879346855Snp set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECE, 1, 1); 1880346855Snp updated++; 1881346855Snp } 1882333153Snp 1883346855Snp if (f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) { 1884346855Snp set_tcb_tflag(sc, f->tid, S_TF_CCTRL_RFR, 1, 1); 1885346855Snp updated++; 1886346855Snp } 1887333153Snp 1888346855Snp if (f->fs.newsmac) { 1889346855Snp MPASS(f->smt != NULL); 1890346855Snp set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1); 1891346855Snp set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL, 1892346855Snp V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), V_TCB_SMAC_SEL(f->smt->idx), 1893346855Snp 1); 1894346855Snp updated++; 1895333153Snp } 1896333153Snp 1897346855Snp switch(f->fs.nat_mode) { 1898346855Snp case NAT_MODE_NONE: 1899346855Snp break; 1900346855Snp case NAT_MODE_DIP: 1901346855Snp set_nat_params(sc, f, true, false, false, false); 1902346855Snp updated++; 1903346855Snp break; 1904346855Snp case NAT_MODE_DIP_DP: 1905346855Snp set_nat_params(sc, f, true, false, true, false); 1906346855Snp updated++; 1907346855Snp break; 1908346855Snp case NAT_MODE_DIP_DP_SIP: 1909346855Snp set_nat_params(sc, f, true, true, true, false); 1910346855Snp updated++; 1911346855Snp break; 1912346855Snp case NAT_MODE_DIP_DP_SP: 1913346855Snp set_nat_params(sc, f, true, false, true, true); 1914346855Snp updated++; 1915346855Snp break; 1916346855Snp case NAT_MODE_SIP_SP: 1917346855Snp set_nat_params(sc, f, false, true, false, true); 1918346855Snp updated++; 1919346855Snp break; 1920346855Snp case NAT_MODE_DIP_SIP_SP: 1921346855Snp set_nat_params(sc, f, true, true, false, true); 1922346855Snp updated++; 1923346855Snp break; 1924346855Snp case NAT_MODE_ALL: 1925346855Snp set_nat_params(sc, f, true, true, true, true); 1926346855Snp updated++; 1927346855Snp break; 1928346855Snp default: 1929346855Snp MPASS(0); /* should have been validated earlier */ 1930346855Snp break; 1931346855Snp 1932346855Snp } 1933346855Snp 1934346855Snp if (f->fs.nat_seq_chk) { 1935346855Snp set_tcb_field(sc, f->tid, W_TCB_RCV_NXT, 1936346855Snp V_TCB_RCV_NXT(M_TCB_RCV_NXT), 1937346855Snp V_TCB_RCV_NXT(f->fs.nat_seq_chk), 1); 1938346855Snp updated++; 1939346855Snp } 1940346855Snp 1941346855Snp if (is_t5(sc) && f->fs.action == FILTER_DROP) { 1942346855Snp /* 1943346855Snp * Migrating = 1, Non-offload = 0 to get a T5 hashfilter to drop. 1944346855Snp */ 1945346855Snp set_tcb_field(sc, f->tid, W_TCB_T_FLAGS, V_TF_NON_OFFLOAD(1) | 1946346855Snp V_TF_MIGRATING(1), V_TF_MIGRATING(1), 1); 1947346855Snp updated++; 1948346855Snp } 1949346855Snp 1950346855Snp /* 1951346855Snp * Enable switching after all secondary resources (L2T entry, SMT entry, 1952346855Snp * etc.) are setup so that any switched packet will use correct 1953346855Snp * values. 1954346855Snp */ 1955346855Snp if (f->fs.action == FILTER_SWITCH) { 1956346855Snp set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECN, 1, 1); 1957346855Snp updated++; 1958346855Snp } 1959346855Snp 1960346855Snp if (f->fs.hitcnts || updated > 0) { 1961346855Snp set_tcb_field(sc, f->tid, W_TCB_TIMESTAMP, 1962346855Snp V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) | 1963346855Snp V_TCB_T_RTT_TS_RECENT_AGE(M_TCB_T_RTT_TS_RECENT_AGE), 1964346855Snp V_TCB_TIMESTAMP(0ULL) | V_TCB_T_RTT_TS_RECENT_AGE(0ULL), 0); 1965346855Snp return (EINPROGRESS); 1966346855Snp } 1967346855Snp 1968333153Snp return (0); 1969333153Snp} 1970