1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/t4_filter.c 346951 2019-04-30 08:01:59Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/eventhandler.h> 36#include <sys/fnv_hash.h> 37#include <sys/systm.h> 38#include <sys/kernel.h> 39#include <sys/module.h> 40#include <sys/bus.h> 41#include <sys/lock.h> 42#include <sys/mutex.h> 43#include <sys/rwlock.h> 44#include <sys/socket.h> 45#include <sys/sbuf.h> 46#include <netinet/in.h> 47 48#include "common/common.h" 49#include "common/t4_msg.h" 50#include "common/t4_regs.h" 51#include "common/t4_regs_values.h" 52#include "common/t4_tcb.h" 53#include "t4_l2t.h" 54#include "t4_smt.h" 55 56struct filter_entry { 57 LIST_ENTRY(filter_entry) link_4t; 58 LIST_ENTRY(filter_entry) link_tid; 59 60 uint32_t valid:1; /* filter allocated and valid */ 61 uint32_t locked:1; /* filter is administratively locked or busy */ 62 uint32_t pending:1; /* filter action is pending firmware reply */ 63 int tid; /* tid of the filter TCB */ 64 struct l2t_entry *l2te; /* L2 table entry for DMAC rewrite */ 65 struct smt_entry *smt; /* SMT entry for SMAC rewrite */ 66 67 struct t4_filter_specification fs; 68}; 69 70static void free_filter_resources(struct filter_entry *); 71static int get_tcamfilter(struct adapter *, struct t4_filter *); 72static int get_hashfilter(struct adapter *, struct t4_filter *); 73static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t, 74 struct l2t_entry *, struct smt_entry *); 75static int del_hashfilter(struct adapter *, struct t4_filter *); 76static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *); 77 78static inline bool 79separate_hpfilter_region(struct adapter *sc) 80{ 81 82 return (chip_id(sc) >= CHELSIO_T6); 83} 84 85static inline uint32_t 86hf_hashfn_4t(struct t4_filter_specification *fs) 87{ 88 struct t4_filter_tuple *ft = &fs->val; 89 uint32_t hash; 90 91 if (fs->type) { 92 /* IPv6 */ 93 hash = fnv_32_buf(&ft->sip[0], 16, FNV1_32_INIT); 94 hash = fnv_32_buf(&ft->dip[0], 16, hash); 95 } else { 96 hash = fnv_32_buf(&ft->sip[0], 4, FNV1_32_INIT); 97 hash = fnv_32_buf(&ft->dip[0], 4, hash); 98 } 99 hash = fnv_32_buf(&ft->sport, sizeof(ft->sport), hash); 100 hash = fnv_32_buf(&ft->dport, sizeof(ft->dport), hash); 101 102 return (hash); 103} 104 105static inline uint32_t 106hf_hashfn_tid(int tid) 107{ 108 109 return (fnv_32_buf(&tid, sizeof(tid), FNV1_32_INIT)); 110} 111 112static int 113alloc_hftid_hash(struct tid_info *t, int flags) 114{ 115 int n; 116 117 MPASS(t->ntids > 0); 118 MPASS(t->hftid_hash_4t == NULL); 119 MPASS(t->hftid_hash_tid == NULL); 120 121 n = max(t->ntids / 1024, 16); 122 t->hftid_hash_4t = hashinit_flags(n, M_CXGBE, &t->hftid_4t_mask, flags); 123 if (t->hftid_hash_4t == NULL) 124 return (ENOMEM); 125 t->hftid_hash_tid = hashinit_flags(n, M_CXGBE, &t->hftid_tid_mask, 126 flags); 127 if (t->hftid_hash_tid == NULL) { 128 hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask); 129 t->hftid_hash_4t = NULL; 130 return (ENOMEM); 131 } 132 133 mtx_init(&t->hftid_lock, "T4 hashfilters", 0, MTX_DEF); 134 cv_init(&t->hftid_cv, "t4hfcv"); 135 136 return (0); 137} 138 139void 140free_hftid_hash(struct tid_info *t) 141{ 142 struct filter_entry *f, *ftmp; 143 LIST_HEAD(, filter_entry) *head; 144 int i; 145#ifdef INVARIANTS 146 int n = 0; 147#endif 148 149 if (t->tids_in_use > 0) { 150 /* Remove everything from the tid hash. */ 151 head = t->hftid_hash_tid; 152 for (i = 0; i <= t->hftid_tid_mask; i++) { 153 LIST_FOREACH_SAFE(f, &head[i], link_tid, ftmp) { 154 LIST_REMOVE(f, link_tid); 155 } 156 } 157 158 /* Remove and then free each filter in the 4t hash. */ 159 head = t->hftid_hash_4t; 160 for (i = 0; i <= t->hftid_4t_mask; i++) { 161 LIST_FOREACH_SAFE(f, &head[i], link_4t, ftmp) { 162#ifdef INVARIANTS 163 n += f->fs.type ? 2 : 1; 164#endif 165 LIST_REMOVE(f, link_4t); 166 free(f, M_CXGBE); 167 } 168 } 169 MPASS(t->tids_in_use == n); 170 t->tids_in_use = 0; 171 } 172 173 if (t->hftid_hash_4t) { 174 hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask); 175 t->hftid_hash_4t = NULL; 176 } 177 if (t->hftid_hash_tid) { 178 hashdestroy(t->hftid_hash_tid, M_CXGBE, t->hftid_tid_mask); 179 t->hftid_hash_tid = NULL; 180 } 181 if (mtx_initialized(&t->hftid_lock)) { 182 mtx_destroy(&t->hftid_lock); 183 cv_destroy(&t->hftid_cv); 184 } 185} 186 187static void 188insert_hf(struct adapter *sc, struct filter_entry *f, uint32_t hash) 189{ 190 struct tid_info *t = &sc->tids; 191 LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t; 192 193 MPASS(head != NULL); 194 if (hash == 0) 195 hash = hf_hashfn_4t(&f->fs); 196 LIST_INSERT_HEAD(&head[hash & t->hftid_4t_mask], f, link_4t); 197 atomic_add_int(&t->tids_in_use, f->fs.type ? 2 : 1); 198} 199 200static void 201insert_hftid(struct adapter *sc, struct filter_entry *f) 202{ 203 struct tid_info *t = &sc->tids; 204 LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid; 205 uint32_t hash; 206 207 MPASS(f->tid >= t->tid_base); 208 MPASS(f->tid - t->tid_base < t->ntids); 209 mtx_assert(&t->hftid_lock, MA_OWNED); 210 211 hash = hf_hashfn_tid(f->tid); 212 LIST_INSERT_HEAD(&head[hash & t->hftid_tid_mask], f, link_tid); 213} 214 215static bool 216filter_eq(struct t4_filter_specification *fs1, 217 struct t4_filter_specification *fs2) 218{ 219 int n; 220 221 MPASS(fs1->hash && fs2->hash); 222 223 if (fs1->type != fs2->type) 224 return (false); 225 226 n = fs1->type ? 16 : 4; 227 if (bcmp(&fs1->val.sip[0], &fs2->val.sip[0], n) || 228 bcmp(&fs1->val.dip[0], &fs2->val.dip[0], n) || 229 fs1->val.sport != fs2->val.sport || 230 fs1->val.dport != fs2->val.dport) 231 return (false); 232 233 /* 234 * We know the masks are the same because all hashfilter masks have to 235 * conform to the global tp->hash_filter_mask and the driver has 236 * verified that already. 237 */ 238 239 if ((fs1->mask.pfvf_vld || fs1->mask.ovlan_vld) && 240 fs1->val.vnic != fs2->val.vnic) 241 return (false); 242 if (fs1->mask.vlan_vld && fs1->val.vlan != fs2->val.vlan) 243 return (false); 244 if (fs1->mask.macidx && fs1->val.macidx != fs2->val.macidx) 245 return (false); 246 if (fs1->mask.frag && fs1->val.frag != fs2->val.frag) 247 return (false); 248 if (fs1->mask.matchtype && fs1->val.matchtype != fs2->val.matchtype) 249 return (false); 250 if (fs1->mask.iport && fs1->val.iport != fs2->val.iport) 251 return (false); 252 if (fs1->mask.fcoe && fs1->val.fcoe != fs2->val.fcoe) 253 return (false); 254 if (fs1->mask.proto && fs1->val.proto != fs2->val.proto) 255 return (false); 256 if (fs1->mask.tos && fs1->val.tos != fs2->val.tos) 257 return (false); 258 if (fs1->mask.ethtype && fs1->val.ethtype != fs2->val.ethtype) 259 return (false); 260 261 return (true); 262} 263 264static struct filter_entry * 265lookup_hf(struct adapter *sc, struct t4_filter_specification *fs, uint32_t hash) 266{ 267 struct tid_info *t = &sc->tids; 268 LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t; 269 struct filter_entry *f; 270 271 mtx_assert(&t->hftid_lock, MA_OWNED); 272 MPASS(head != NULL); 273 274 if (hash == 0) 275 hash = hf_hashfn_4t(fs); 276 277 LIST_FOREACH(f, &head[hash & t->hftid_4t_mask], link_4t) { 278 if (filter_eq(&f->fs, fs)) 279 return (f); 280 } 281 282 return (NULL); 283} 284 285static struct filter_entry * 286lookup_hftid(struct adapter *sc, int tid) 287{ 288 struct tid_info *t = &sc->tids; 289 LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid; 290 struct filter_entry *f; 291 uint32_t hash; 292 293 mtx_assert(&t->hftid_lock, MA_OWNED); 294 MPASS(head != NULL); 295 296 hash = hf_hashfn_tid(tid); 297 LIST_FOREACH(f, &head[hash & t->hftid_tid_mask], link_tid) { 298 if (f->tid == tid) 299 return (f); 300 } 301 302 return (NULL); 303} 304 305static void 306remove_hf(struct adapter *sc, struct filter_entry *f) 307{ 308 struct tid_info *t = &sc->tids; 309 310 mtx_assert(&t->hftid_lock, MA_OWNED); 311 312 LIST_REMOVE(f, link_4t); 313 atomic_subtract_int(&t->tids_in_use, f->fs.type ? 2 : 1); 314} 315 316static void 317remove_hftid(struct adapter *sc, struct filter_entry *f) 318{ 319#ifdef INVARIANTS 320 struct tid_info *t = &sc->tids; 321 322 mtx_assert(&t->hftid_lock, MA_OWNED); 323#endif 324 325 LIST_REMOVE(f, link_tid); 326} 327 328static uint32_t 329mode_to_fconf(uint32_t mode) 330{ 331 uint32_t fconf = 0; 332 333 if (mode & T4_FILTER_IP_FRAGMENT) 334 fconf |= F_FRAGMENTATION; 335 336 if (mode & T4_FILTER_MPS_HIT_TYPE) 337 fconf |= F_MPSHITTYPE; 338 339 if (mode & T4_FILTER_MAC_IDX) 340 fconf |= F_MACMATCH; 341 342 if (mode & T4_FILTER_ETH_TYPE) 343 fconf |= F_ETHERTYPE; 344 345 if (mode & T4_FILTER_IP_PROTO) 346 fconf |= F_PROTOCOL; 347 348 if (mode & T4_FILTER_IP_TOS) 349 fconf |= F_TOS; 350 351 if (mode & T4_FILTER_VLAN) 352 fconf |= F_VLAN; 353 354 if (mode & T4_FILTER_VNIC) 355 fconf |= F_VNIC_ID; 356 357 if (mode & T4_FILTER_PORT) 358 fconf |= F_PORT; 359 360 if (mode & T4_FILTER_FCoE) 361 fconf |= F_FCOE; 362 363 return (fconf); 364} 365 366static uint32_t 367mode_to_iconf(uint32_t mode) 368{ 369 370 if (mode & T4_FILTER_IC_VNIC) 371 return (F_VNIC); 372 return (0); 373} 374 375static int 376check_fspec_against_fconf_iconf(struct adapter *sc, 377 struct t4_filter_specification *fs) 378{ 379 struct tp_params *tpp = &sc->params.tp; 380 uint32_t fconf = 0; 381 382 if (fs->val.frag || fs->mask.frag) 383 fconf |= F_FRAGMENTATION; 384 385 if (fs->val.matchtype || fs->mask.matchtype) 386 fconf |= F_MPSHITTYPE; 387 388 if (fs->val.macidx || fs->mask.macidx) 389 fconf |= F_MACMATCH; 390 391 if (fs->val.ethtype || fs->mask.ethtype) 392 fconf |= F_ETHERTYPE; 393 394 if (fs->val.proto || fs->mask.proto) 395 fconf |= F_PROTOCOL; 396 397 if (fs->val.tos || fs->mask.tos) 398 fconf |= F_TOS; 399 400 if (fs->val.vlan_vld || fs->mask.vlan_vld) 401 fconf |= F_VLAN; 402 403 if (fs->val.ovlan_vld || fs->mask.ovlan_vld) { 404 fconf |= F_VNIC_ID; 405 if (tpp->ingress_config & F_VNIC) 406 return (EINVAL); 407 } 408 409 if (fs->val.pfvf_vld || fs->mask.pfvf_vld) { 410 fconf |= F_VNIC_ID; 411 if ((tpp->ingress_config & F_VNIC) == 0) 412 return (EINVAL); 413 } 414 415 if (fs->val.iport || fs->mask.iport) 416 fconf |= F_PORT; 417 418 if (fs->val.fcoe || fs->mask.fcoe) 419 fconf |= F_FCOE; 420 421 if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map) 422 return (E2BIG); 423 424 return (0); 425} 426 427int 428get_filter_mode(struct adapter *sc, uint32_t *mode) 429{ 430 struct tp_params *tp = &sc->params.tp; 431 uint64_t mask; 432 433 /* Non-zero incoming value in mode means "hashfilter mode". */ 434 mask = *mode ? tp->hash_filter_mask : UINT64_MAX; 435 436 /* Always */ 437 *mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 438 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 439 440#define CHECK_FIELD(fconf_bit, field_shift, field_mask, mode_bit) do { \ 441 if (tp->vlan_pri_map & (fconf_bit)) { \ 442 MPASS(tp->field_shift >= 0); \ 443 if ((mask >> tp->field_shift & field_mask) == field_mask) \ 444 *mode |= (mode_bit); \ 445 } \ 446} while (0) 447 448 CHECK_FIELD(F_FRAGMENTATION, frag_shift, M_FT_FRAGMENTATION, T4_FILTER_IP_FRAGMENT); 449 CHECK_FIELD(F_MPSHITTYPE, matchtype_shift, M_FT_MPSHITTYPE, T4_FILTER_MPS_HIT_TYPE); 450 CHECK_FIELD(F_MACMATCH, macmatch_shift, M_FT_MACMATCH, T4_FILTER_MAC_IDX); 451 CHECK_FIELD(F_ETHERTYPE, ethertype_shift, M_FT_ETHERTYPE, T4_FILTER_ETH_TYPE); 452 CHECK_FIELD(F_PROTOCOL, protocol_shift, M_FT_PROTOCOL, T4_FILTER_IP_PROTO); 453 CHECK_FIELD(F_TOS, tos_shift, M_FT_TOS, T4_FILTER_IP_TOS); 454 CHECK_FIELD(F_VLAN, vlan_shift, M_FT_VLAN, T4_FILTER_VLAN); 455 CHECK_FIELD(F_VNIC_ID, vnic_shift, M_FT_VNIC_ID , T4_FILTER_VNIC); 456 if (tp->ingress_config & F_VNIC) 457 *mode |= T4_FILTER_IC_VNIC; 458 CHECK_FIELD(F_PORT, port_shift, M_FT_PORT , T4_FILTER_PORT); 459 CHECK_FIELD(F_FCOE, fcoe_shift, M_FT_FCOE , T4_FILTER_FCoE); 460#undef CHECK_FIELD 461 462 return (0); 463} 464 465int 466set_filter_mode(struct adapter *sc, uint32_t mode) 467{ 468 struct tp_params *tpp = &sc->params.tp; 469 uint32_t fconf, iconf; 470 int rc; 471 472 iconf = mode_to_iconf(mode); 473 if ((iconf ^ tpp->ingress_config) & F_VNIC) { 474 /* 475 * For now we just complain if A_TP_INGRESS_CONFIG is not 476 * already set to the correct value for the requested filter 477 * mode. It's not clear if it's safe to write to this register 478 * on the fly. (And we trust the cached value of the register). 479 * 480 * check_fspec_against_fconf_iconf and other code that looks at 481 * tp->vlan_pri_map and tp->ingress_config needs to be reviewed 482 * thorougly before allowing dynamic filter mode changes. 483 */ 484 return (EBUSY); 485 } 486 487 fconf = mode_to_fconf(mode); 488 489 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 490 "t4setfm"); 491 if (rc) 492 return (rc); 493 494 if (sc->tids.ftids_in_use > 0 || sc->tids.hpftids_in_use > 0) { 495 rc = EBUSY; 496 goto done; 497 } 498 499#ifdef TCP_OFFLOAD 500 if (uld_active(sc, ULD_TOM)) { 501 rc = EBUSY; 502 goto done; 503 } 504#endif 505 506 rc = -t4_set_filter_mode(sc, fconf, true); 507done: 508 end_synchronized_op(sc, LOCK_HELD); 509 return (rc); 510} 511 512static inline uint64_t 513get_filter_hits(struct adapter *sc, uint32_t tid) 514{ 515 uint32_t tcb_addr; 516 517 tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE; 518 519 if (is_t4(sc)) { 520 uint64_t hits; 521 522 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8); 523 return (be64toh(hits)); 524 } else { 525 uint32_t hits; 526 527 read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4); 528 return (be32toh(hits)); 529 } 530} 531 532int 533get_filter(struct adapter *sc, struct t4_filter *t) 534{ 535 if (t->fs.hash) 536 return (get_hashfilter(sc, t)); 537 else 538 return (get_tcamfilter(sc, t)); 539} 540 541static int 542set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te, 543 struct smt_entry *smt) 544{ 545 struct filter_entry *f; 546 struct fw_filter2_wr *fwr; 547 u_int vnic_vld, vnic_vld_mask; 548 struct wrq_cookie cookie; 549 int i, rc, busy, locked; 550 u_int tid; 551 const int ntids = t->fs.type ? 4 : 1; 552 553 MPASS(!t->fs.hash); 554 /* Already validated against fconf, iconf */ 555 MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0); 556 MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0); 557 558 if (separate_hpfilter_region(sc) && t->fs.prio) { 559 MPASS(t->idx < sc->tids.nhpftids); 560 f = &sc->tids.hpftid_tab[t->idx]; 561 tid = sc->tids.hpftid_base + t->idx; 562 } else { 563 MPASS(t->idx < sc->tids.nftids); 564 f = &sc->tids.ftid_tab[t->idx]; 565 tid = sc->tids.ftid_base + t->idx; 566 } 567 rc = busy = locked = 0; 568 mtx_lock(&sc->tids.ftid_lock); 569 for (i = 0; i < ntids; i++) { 570 busy += f[i].pending + f[i].valid; 571 locked += f[i].locked; 572 } 573 if (locked > 0) 574 rc = EPERM; 575 else if (busy > 0) 576 rc = EBUSY; 577 else { 578 int len16; 579 580 if (sc->params.filter2_wr_support) 581 len16 = howmany(sizeof(struct fw_filter2_wr), 16); 582 else 583 len16 = howmany(sizeof(struct fw_filter_wr), 16); 584 fwr = start_wrq_wr(&sc->sge.ctrlq[0], len16, &cookie); 585 if (__predict_false(fwr == NULL)) 586 rc = ENOMEM; 587 else { 588 f->pending = 1; 589 if (separate_hpfilter_region(sc) && t->fs.prio) 590 sc->tids.hpftids_in_use++; 591 else 592 sc->tids.ftids_in_use++; 593 } 594 } 595 mtx_unlock(&sc->tids.ftid_lock); 596 if (rc != 0) 597 return (rc); 598 599 /* 600 * Can't fail now. A set-filter WR will definitely be sent. 601 */ 602 603 f->tid = tid; 604 f->fs = t->fs; 605 f->l2te = l2te; 606 f->smt = smt; 607 608 if (t->fs.val.pfvf_vld || t->fs.val.ovlan_vld) 609 vnic_vld = 1; 610 else 611 vnic_vld = 0; 612 if (t->fs.mask.pfvf_vld || t->fs.mask.ovlan_vld) 613 vnic_vld_mask = 1; 614 else 615 vnic_vld_mask = 0; 616 617 bzero(fwr, sizeof(*fwr)); 618 if (sc->params.filter2_wr_support) 619 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER2_WR)); 620 else 621 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 622 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 623 fwr->tid_to_iq = 624 htobe32(V_FW_FILTER_WR_TID(f->tid) | 625 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 626 V_FW_FILTER_WR_NOREPLY(0) | 627 V_FW_FILTER_WR_IQ(f->fs.iq)); 628 fwr->del_filter_to_l2tix = 629 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 630 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 631 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 632 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 633 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 634 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 635 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 636 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 637 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 638 f->fs.newvlan == VLAN_REWRITE) | 639 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 640 f->fs.newvlan == VLAN_REWRITE) | 641 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 642 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 643 V_FW_FILTER_WR_PRIO(f->fs.prio) | 644 V_FW_FILTER_WR_L2TIX(f->l2te ? f->l2te->idx : 0)); 645 fwr->ethtype = htobe16(f->fs.val.ethtype); 646 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 647 fwr->frag_to_ovlan_vldm = 648 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 649 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 650 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 651 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) | 652 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 653 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask)); 654 fwr->smac_sel = 0; 655 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 656 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 657 fwr->maci_to_matchtypem = 658 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 659 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 660 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 661 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 662 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 663 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 664 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 665 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 666 fwr->ptcl = f->fs.val.proto; 667 fwr->ptclm = f->fs.mask.proto; 668 fwr->ttyp = f->fs.val.tos; 669 fwr->ttypm = f->fs.mask.tos; 670 fwr->ivlan = htobe16(f->fs.val.vlan); 671 fwr->ivlanm = htobe16(f->fs.mask.vlan); 672 fwr->ovlan = htobe16(f->fs.val.vnic); 673 fwr->ovlanm = htobe16(f->fs.mask.vnic); 674 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 675 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 676 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 677 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 678 fwr->lp = htobe16(f->fs.val.dport); 679 fwr->lpm = htobe16(f->fs.mask.dport); 680 fwr->fp = htobe16(f->fs.val.sport); 681 fwr->fpm = htobe16(f->fs.mask.sport); 682 /* sma = 0 tells the fw to use SMAC_SEL for source MAC address */ 683 bzero(fwr->sma, sizeof (fwr->sma)); 684 if (sc->params.filter2_wr_support) { 685 fwr->filter_type_swapmac = 686 V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac); 687 fwr->natmode_to_ulp_type = 688 V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ? 689 ULP_MODE_TCPDDP : ULP_MODE_NONE) | 690 V_FW_FILTER2_WR_NATFLAGCHECK(f->fs.nat_flag_chk) | 691 V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode); 692 memcpy(fwr->newlip, f->fs.nat_dip, sizeof(fwr->newlip)); 693 memcpy(fwr->newfip, f->fs.nat_sip, sizeof(fwr->newfip)); 694 fwr->newlport = htobe16(f->fs.nat_dport); 695 fwr->newfport = htobe16(f->fs.nat_sport); 696 fwr->natseqcheck = htobe32(f->fs.nat_seq_chk); 697 } 698 commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie); 699 700 /* Wait for response. */ 701 mtx_lock(&sc->tids.ftid_lock); 702 for (;;) { 703 if (f->pending == 0) { 704 rc = f->valid ? 0 : EIO; 705 break; 706 } 707 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) { 708 rc = EINPROGRESS; 709 break; 710 } 711 } 712 mtx_unlock(&sc->tids.ftid_lock); 713 return (rc); 714} 715 716static int 717hashfilter_ntuple(struct adapter *sc, const struct t4_filter_specification *fs, 718 uint64_t *ftuple) 719{ 720 struct tp_params *tp = &sc->params.tp; 721 uint64_t fmask; 722 723 *ftuple = fmask = 0; 724 725 /* 726 * Initialize each of the fields which we care about which are present 727 * in the Compressed Filter Tuple. 728 */ 729 if (tp->vlan_shift >= 0 && fs->mask.vlan) { 730 *ftuple |= (F_FT_VLAN_VLD | fs->val.vlan) << tp->vlan_shift; 731 fmask |= M_FT_VLAN << tp->vlan_shift; 732 } 733 734 if (tp->port_shift >= 0 && fs->mask.iport) { 735 *ftuple |= (uint64_t)fs->val.iport << tp->port_shift; 736 fmask |= M_FT_PORT << tp->port_shift; 737 } 738 739 if (tp->protocol_shift >= 0 && fs->mask.proto) { 740 *ftuple |= (uint64_t)fs->val.proto << tp->protocol_shift; 741 fmask |= M_FT_PROTOCOL << tp->protocol_shift; 742 } 743 744 if (tp->tos_shift >= 0 && fs->mask.tos) { 745 *ftuple |= (uint64_t)(fs->val.tos) << tp->tos_shift; 746 fmask |= M_FT_TOS << tp->tos_shift; 747 } 748 749 if (tp->vnic_shift >= 0 && fs->mask.vnic) { 750 /* F_VNIC in ingress config was already validated. */ 751 if (tp->ingress_config & F_VNIC) 752 MPASS(fs->mask.pfvf_vld); 753 else 754 MPASS(fs->mask.ovlan_vld); 755 756 *ftuple |= ((1ULL << 16) | fs->val.vnic) << tp->vnic_shift; 757 fmask |= M_FT_VNIC_ID << tp->vnic_shift; 758 } 759 760 if (tp->macmatch_shift >= 0 && fs->mask.macidx) { 761 *ftuple |= (uint64_t)(fs->val.macidx) << tp->macmatch_shift; 762 fmask |= M_FT_MACMATCH << tp->macmatch_shift; 763 } 764 765 if (tp->ethertype_shift >= 0 && fs->mask.ethtype) { 766 *ftuple |= (uint64_t)(fs->val.ethtype) << tp->ethertype_shift; 767 fmask |= M_FT_ETHERTYPE << tp->ethertype_shift; 768 } 769 770 if (tp->matchtype_shift >= 0 && fs->mask.matchtype) { 771 *ftuple |= (uint64_t)(fs->val.matchtype) << tp->matchtype_shift; 772 fmask |= M_FT_MPSHITTYPE << tp->matchtype_shift; 773 } 774 775 if (tp->frag_shift >= 0 && fs->mask.frag) { 776 *ftuple |= (uint64_t)(fs->val.frag) << tp->frag_shift; 777 fmask |= M_FT_FRAGMENTATION << tp->frag_shift; 778 } 779 780 if (tp->fcoe_shift >= 0 && fs->mask.fcoe) { 781 *ftuple |= (uint64_t)(fs->val.fcoe) << tp->fcoe_shift; 782 fmask |= M_FT_FCOE << tp->fcoe_shift; 783 } 784 785 /* A hashfilter must conform to the filterMask. */ 786 if (fmask != tp->hash_filter_mask) 787 return (EINVAL); 788 789 return (0); 790} 791 792static bool 793is_4tuple_specified(struct t4_filter_specification *fs) 794{ 795 int i; 796 const int n = fs->type ? 16 : 4; 797 798 if (fs->mask.sport != 0xffff || fs->mask.dport != 0xffff) 799 return (false); 800 801 for (i = 0; i < n; i++) { 802 if (fs->mask.sip[i] != 0xff) 803 return (false); 804 if (fs->mask.dip[i] != 0xff) 805 return (false); 806 } 807 808 return (true); 809} 810 811int 812set_filter(struct adapter *sc, struct t4_filter *t) 813{ 814 struct tid_info *ti = &sc->tids; 815 struct l2t_entry *l2te = NULL; 816 struct smt_entry *smt = NULL; 817 uint64_t ftuple; 818 int rc; 819 820 /* 821 * Basic filter checks first. 822 */ 823 824 if (t->fs.hash) { 825 if (!is_hashfilter(sc) || ti->ntids == 0) 826 return (ENOTSUP); 827 /* Hardware, not user, selects a tid for hashfilters. */ 828 if (t->idx != (uint32_t)-1) 829 return (EINVAL); 830 /* T5 can't count hashfilter hits. */ 831 if (is_t5(sc) && t->fs.hitcnts) 832 return (EINVAL); 833 if (!is_4tuple_specified(&t->fs)) 834 return (EINVAL); 835 rc = hashfilter_ntuple(sc, &t->fs, &ftuple); 836 if (rc != 0) 837 return (rc); 838 } else { 839 if (separate_hpfilter_region(sc) && t->fs.prio) { 840 if (ti->nhpftids == 0) 841 return (ENOTSUP); 842 if (t->idx >= ti->nhpftids) 843 return (EINVAL); 844 } else { 845 if (ti->nftids == 0) 846 return (ENOTSUP); 847 if (t->idx >= ti->nftids) 848 return (EINVAL); 849 } 850 /* IPv6 filter idx must be 4 aligned */ 851 if (t->fs.type == 1 && 852 ((t->idx & 0x3) || t->idx + 4 >= ti->nftids)) 853 return (EINVAL); 854 } 855 856 /* T4 doesn't support VLAN tag removal or rewrite, swapmac, and NAT. */ 857 if (is_t4(sc) && t->fs.action == FILTER_SWITCH && 858 (t->fs.newvlan == VLAN_REMOVE || t->fs.newvlan == VLAN_REWRITE || 859 t->fs.swapmac || t->fs.nat_mode)) 860 return (ENOTSUP); 861 862 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= sc->params.nports) 863 return (EINVAL); 864 if (t->fs.val.iport >= sc->params.nports) 865 return (EINVAL); 866 867 /* Can't specify an iq if not steering to it */ 868 if (!t->fs.dirsteer && t->fs.iq) 869 return (EINVAL); 870 871 /* Validate against the global filter mode and ingress config */ 872 rc = check_fspec_against_fconf_iconf(sc, &t->fs); 873 if (rc != 0) 874 return (rc); 875 876 /* 877 * Basic checks passed. Make sure the queues and tid tables are setup. 878 */ 879 880 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 881 if (rc) 882 return (rc); 883 if (!(sc->flags & FULL_INIT_DONE) && 884 ((rc = adapter_full_init(sc)) != 0)) { 885 end_synchronized_op(sc, 0); 886 return (rc); 887 } 888 if (t->fs.hash) { 889 if (__predict_false(ti->hftid_hash_4t == NULL)) { 890 rc = alloc_hftid_hash(&sc->tids, HASH_NOWAIT); 891 if (rc != 0) 892 goto done; 893 } 894 if (__predict_false(sc->tids.atid_tab == NULL)) { 895 rc = alloc_atid_tab(&sc->tids, M_NOWAIT); 896 if (rc != 0) 897 goto done; 898 } 899 } else if (separate_hpfilter_region(sc) && t->fs.prio && 900 __predict_false(ti->hpftid_tab == NULL)) { 901 MPASS(ti->nhpftids != 0); 902 KASSERT(ti->hpftids_in_use == 0, 903 ("%s: no memory allocated but hpftids_in_use is %u", 904 __func__, ti->hpftids_in_use)); 905 ti->hpftid_tab = malloc(sizeof(struct filter_entry) * 906 ti->nhpftids, M_CXGBE, M_NOWAIT | M_ZERO); 907 if (ti->hpftid_tab == NULL) { 908 rc = ENOMEM; 909 goto done; 910 } 911 if (!mtx_initialized(&sc->tids.ftid_lock)) { 912 mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF); 913 cv_init(&ti->ftid_cv, "t4fcv"); 914 } 915 } else if (__predict_false(ti->ftid_tab == NULL)) { 916 MPASS(ti->nftids != 0); 917 KASSERT(ti->ftids_in_use == 0, 918 ("%s: no memory allocated but ftids_in_use is %u", 919 __func__, ti->ftids_in_use)); 920 ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids, 921 M_CXGBE, M_NOWAIT | M_ZERO); 922 if (ti->ftid_tab == NULL) { 923 rc = ENOMEM; 924 goto done; 925 } 926 if (!mtx_initialized(&sc->tids.ftid_lock)) { 927 mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF); 928 cv_init(&ti->ftid_cv, "t4fcv"); 929 } 930 } 931done: 932 end_synchronized_op(sc, 0); 933 if (rc != 0) 934 return (rc); 935 936 /* 937 * Allocate L2T entry, SMT entry, etc. 938 */ 939 940 if (t->fs.newdmac || t->fs.newvlan) { 941 /* This filter needs an L2T entry; allocate one. */ 942 l2te = t4_l2t_alloc_switching(sc, t->fs.vlan, t->fs.eport, 943 t->fs.dmac); 944 if (__predict_false(l2te == NULL)) { 945 rc = EAGAIN; 946 goto error; 947 } 948 } 949 950 if (t->fs.newsmac) { 951 /* This filter needs an SMT entry; allocate one. */ 952 smt = t4_smt_alloc_switching(sc->smt, t->fs.smac); 953 if (__predict_false(smt == NULL)) { 954 rc = EAGAIN; 955 goto error; 956 } 957 rc = t4_smt_set_switching(sc, smt, 0x0, t->fs.smac); 958 if (rc) 959 goto error; 960 } 961 962 if (t->fs.hash) 963 rc = set_hashfilter(sc, t, ftuple, l2te, smt); 964 else 965 rc = set_tcamfilter(sc, t, l2te, smt); 966 967 if (rc != 0 && rc != EINPROGRESS) { 968error: 969 if (l2te) 970 t4_l2t_release(l2te); 971 if (smt) 972 t4_smt_release(smt); 973 } 974 return (rc); 975} 976 977static int 978del_tcamfilter(struct adapter *sc, struct t4_filter *t) 979{ 980 struct filter_entry *f; 981 struct fw_filter_wr *fwr; 982 struct wrq_cookie cookie; 983 int rc, nfilters; 984#ifdef INVARIANTS 985 u_int tid_base; 986#endif 987 988 mtx_lock(&sc->tids.ftid_lock); 989 if (separate_hpfilter_region(sc) && t->fs.prio) { 990 nfilters = sc->tids.nhpftids; 991 f = sc->tids.hpftid_tab; 992#ifdef INVARIANTS 993 tid_base = sc->tids.hpftid_base; 994#endif 995 } else { 996 nfilters = sc->tids.nftids; 997 f = sc->tids.ftid_tab; 998#ifdef INVARIANTS 999 tid_base = sc->tids.ftid_base; 1000#endif 1001 } 1002 MPASS(f != NULL); /* Caller checked this. */ 1003 if (t->idx >= nfilters) { 1004 rc = EINVAL; 1005 goto done; 1006 } 1007 f += t->idx; 1008 1009 if (f->locked) { 1010 rc = EPERM; 1011 goto done; 1012 } 1013 if (f->pending) { 1014 rc = EBUSY; 1015 goto done; 1016 } 1017 if (f->valid == 0) { 1018 rc = EINVAL; 1019 goto done; 1020 } 1021 MPASS(f->tid == tid_base + t->idx); 1022 fwr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*fwr), 16), &cookie); 1023 if (fwr == NULL) { 1024 rc = ENOMEM; 1025 goto done; 1026 } 1027 1028 bzero(fwr, sizeof (*fwr)); 1029 t4_mk_filtdelwr(f->tid, fwr, sc->sge.fwq.abs_id); 1030 f->pending = 1; 1031 commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie); 1032 t->fs = f->fs; /* extra info for the caller */ 1033 1034 for (;;) { 1035 if (f->pending == 0) { 1036 rc = f->valid ? EIO : 0; 1037 break; 1038 } 1039 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) { 1040 rc = EINPROGRESS; 1041 break; 1042 } 1043 } 1044done: 1045 mtx_unlock(&sc->tids.ftid_lock); 1046 return (rc); 1047} 1048 1049int 1050del_filter(struct adapter *sc, struct t4_filter *t) 1051{ 1052 1053 /* No filters possible if not initialized yet. */ 1054 if (!(sc->flags & FULL_INIT_DONE)) 1055 return (EINVAL); 1056 1057 /* 1058 * The checks for tid tables ensure that the locks that del_* will reach 1059 * for are initialized. 1060 */ 1061 if (t->fs.hash) { 1062 if (sc->tids.hftid_hash_4t != NULL) 1063 return (del_hashfilter(sc, t)); 1064 } else if (separate_hpfilter_region(sc) && t->fs.prio) { 1065 if (sc->tids.hpftid_tab != NULL) 1066 return (del_tcamfilter(sc, t)); 1067 } else { 1068 if (sc->tids.ftid_tab != NULL) 1069 return (del_tcamfilter(sc, t)); 1070 } 1071 1072 return (EINVAL); 1073} 1074 1075/* 1076 * Release secondary resources associated with the filter. 1077 */ 1078static void 1079free_filter_resources(struct filter_entry *f) 1080{ 1081 1082 if (f->l2te) { 1083 t4_l2t_release(f->l2te); 1084 f->l2te = NULL; 1085 } 1086 if (f->smt) { 1087 t4_smt_release(f->smt); 1088 f->smt = NULL; 1089 } 1090} 1091 1092static int 1093set_tcb_field(struct adapter *sc, u_int tid, uint16_t word, uint64_t mask, 1094 uint64_t val, int no_reply) 1095{ 1096 struct wrq_cookie cookie; 1097 struct cpl_set_tcb_field *req; 1098 1099 req = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*req), 16), &cookie); 1100 if (req == NULL) 1101 return (ENOMEM); 1102 bzero(req, sizeof(*req)); 1103 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid); 1104 if (no_reply == 0) { 1105 req->reply_ctrl = htobe16(V_QUEUENO(sc->sge.fwq.abs_id) | 1106 V_NO_REPLY(0)); 1107 } else 1108 req->reply_ctrl = htobe16(V_NO_REPLY(1)); 1109 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(CPL_COOKIE_HASHFILTER)); 1110 req->mask = htobe64(mask); 1111 req->val = htobe64(val); 1112 commit_wrq_wr(&sc->sge.ctrlq[0], req, &cookie); 1113 1114 return (0); 1115} 1116 1117/* Set one of the t_flags bits in the TCB. */ 1118static inline int 1119set_tcb_tflag(struct adapter *sc, int tid, u_int bit_pos, u_int val, 1120 u_int no_reply) 1121{ 1122 1123 return (set_tcb_field(sc, tid, W_TCB_T_FLAGS, 1ULL << bit_pos, 1124 (uint64_t)val << bit_pos, no_reply)); 1125} 1126 1127int 1128t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1129{ 1130 struct adapter *sc = iq->adapter; 1131 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 1132 u_int tid = GET_TID(rpl); 1133 u_int rc, idx; 1134 struct filter_entry *f; 1135 1136 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 1137 rss->opcode)); 1138 1139 1140 if (is_hpftid(sc, tid)) { 1141 idx = tid - sc->tids.hpftid_base; 1142 f = &sc->tids.hpftid_tab[idx]; 1143 } else if (is_ftid(sc, tid)) { 1144 idx = tid - sc->tids.ftid_base; 1145 f = &sc->tids.ftid_tab[idx]; 1146 } else 1147 panic("%s: FW reply for invalid TID %d.", __func__, tid); 1148 1149 MPASS(f->tid == tid); 1150 rc = G_COOKIE(rpl->cookie); 1151 1152 mtx_lock(&sc->tids.ftid_lock); 1153 KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.", 1154 __func__, rc, tid)); 1155 switch(rc) { 1156 case FW_FILTER_WR_FLT_ADDED: 1157 /* set-filter succeeded */ 1158 f->valid = 1; 1159 if (f->fs.newsmac) { 1160 MPASS(f->smt != NULL); 1161 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1); 1162 set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL, 1163 V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), 1164 V_TCB_SMAC_SEL(f->smt->idx), 1); 1165 /* XXX: wait for reply to TCB update before !pending */ 1166 } 1167 break; 1168 case FW_FILTER_WR_FLT_DELETED: 1169 /* del-filter succeeded */ 1170 MPASS(f->valid == 1); 1171 f->valid = 0; 1172 /* Fall through */ 1173 case FW_FILTER_WR_SMT_TBL_FULL: 1174 /* set-filter failed due to lack of SMT space. */ 1175 MPASS(f->valid == 0); 1176 free_filter_resources(f); 1177 if (separate_hpfilter_region(sc) && f->fs.prio) 1178 sc->tids.hpftids_in_use--; 1179 else 1180 sc->tids.ftids_in_use--; 1181 break; 1182 case FW_FILTER_WR_SUCCESS: 1183 case FW_FILTER_WR_EINVAL: 1184 default: 1185 panic("%s: unexpected reply %d for filter[%d].", __func__, rc, 1186 idx); 1187 } 1188 f->pending = 0; 1189 cv_broadcast(&sc->tids.ftid_cv); 1190 mtx_unlock(&sc->tids.ftid_lock); 1191 1192 return (0); 1193} 1194 1195/* 1196 * This is the reply to the Active Open that created the filter. Additional TCB 1197 * updates may be required to complete the filter configuration. 1198 */ 1199int 1200t4_hashfilter_ao_rpl(struct sge_iq *iq, const struct rss_header *rss, 1201 struct mbuf *m) 1202{ 1203 struct adapter *sc = iq->adapter; 1204 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); 1205 u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status))); 1206 u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status)); 1207 struct filter_entry *f = lookup_atid(sc, atid); 1208 1209 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1210 1211 mtx_lock(&sc->tids.hftid_lock); 1212 KASSERT(f->pending, ("%s: hashfilter[%p] isn't pending.", __func__, f)); 1213 KASSERT(f->tid == -1, ("%s: hashfilter[%p] has tid %d already.", 1214 __func__, f, f->tid)); 1215 if (status == CPL_ERR_NONE) { 1216 f->tid = GET_TID(cpl); 1217 MPASS(lookup_hftid(sc, f->tid) == NULL); 1218 insert_hftid(sc, f); 1219 /* 1220 * Leave the filter pending until it is fully set up, which will 1221 * be indicated by the reply to the last TCB update. No need to 1222 * unblock the ioctl thread either. 1223 */ 1224 if (configure_hashfilter_tcb(sc, f) == EINPROGRESS) 1225 goto done; 1226 f->valid = 1; 1227 f->pending = 0; 1228 } else { 1229 /* provide errno instead of tid to ioctl */ 1230 f->tid = act_open_rpl_status_to_errno(status); 1231 f->valid = 0; 1232 f->pending = 0; 1233 if (act_open_has_tid(status)) 1234 release_tid(sc, GET_TID(cpl), &sc->sge.ctrlq[0]); 1235 free_filter_resources(f); 1236 remove_hf(sc, f); 1237 if (f->locked == 0) 1238 free(f, M_CXGBE); 1239 } 1240 cv_broadcast(&sc->tids.hftid_cv); 1241done: 1242 mtx_unlock(&sc->tids.hftid_lock); 1243 1244 free_atid(sc, atid); 1245 return (0); 1246} 1247 1248int 1249t4_hashfilter_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, 1250 struct mbuf *m) 1251{ 1252 struct adapter *sc = iq->adapter; 1253 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 1254 u_int tid = GET_TID(rpl); 1255 struct filter_entry *f; 1256 1257 mtx_lock(&sc->tids.hftid_lock); 1258 f = lookup_hftid(sc, tid); 1259 KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__)); 1260 KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__, 1261 f, tid)); 1262 KASSERT(f->valid == 0, ("%s: hashfilter %p [%u] is valid already.", 1263 __func__, f, tid)); 1264 f->pending = 0; 1265 if (rpl->status == 0) { 1266 f->valid = 1; 1267 } else { 1268 f->tid = EIO; 1269 f->valid = 0; 1270 free_filter_resources(f); 1271 remove_hftid(sc, f); 1272 remove_hf(sc, f); 1273 release_tid(sc, tid, &sc->sge.ctrlq[0]); 1274 if (f->locked == 0) 1275 free(f, M_CXGBE); 1276 } 1277 cv_broadcast(&sc->tids.hftid_cv); 1278 mtx_unlock(&sc->tids.hftid_lock); 1279 1280 return (0); 1281} 1282 1283int 1284t4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss, 1285 struct mbuf *m) 1286{ 1287 struct adapter *sc = iq->adapter; 1288 const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); 1289 unsigned int tid = GET_TID(cpl); 1290 struct filter_entry *f; 1291 1292 mtx_lock(&sc->tids.hftid_lock); 1293 f = lookup_hftid(sc, tid); 1294 KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__)); 1295 KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__, 1296 f, tid)); 1297 KASSERT(f->valid, ("%s: hashfilter %p [%u] isn't valid.", __func__, f, 1298 tid)); 1299 f->pending = 0; 1300 if (cpl->status == 0) { 1301 f->valid = 0; 1302 free_filter_resources(f); 1303 remove_hftid(sc, f); 1304 remove_hf(sc, f); 1305 release_tid(sc, tid, &sc->sge.ctrlq[0]); 1306 if (f->locked == 0) 1307 free(f, M_CXGBE); 1308 } 1309 cv_broadcast(&sc->tids.hftid_cv); 1310 mtx_unlock(&sc->tids.hftid_lock); 1311 1312 return (0); 1313} 1314 1315static int 1316get_tcamfilter(struct adapter *sc, struct t4_filter *t) 1317{ 1318 int i, nfilters; 1319 struct filter_entry *f; 1320 u_int in_use; 1321#ifdef INVARIANTS 1322 u_int tid_base; 1323#endif 1324 1325 MPASS(!t->fs.hash); 1326 1327 if (separate_hpfilter_region(sc) && t->fs.prio) { 1328 nfilters = sc->tids.nhpftids; 1329 f = sc->tids.hpftid_tab; 1330 in_use = sc->tids.hpftids_in_use; 1331#ifdef INVARIANTS 1332 tid_base = sc->tids.hpftid_base; 1333#endif 1334 } else { 1335 nfilters = sc->tids.nftids; 1336 f = sc->tids.ftid_tab; 1337 in_use = sc->tids.ftids_in_use; 1338#ifdef INVARIANTS 1339 tid_base = sc->tids.ftid_base; 1340#endif 1341 } 1342 1343 if (in_use == 0 || f == NULL || t->idx >= nfilters) { 1344 t->idx = 0xffffffff; 1345 return (0); 1346 } 1347 1348 f += t->idx; 1349 mtx_lock(&sc->tids.ftid_lock); 1350 for (i = t->idx; i < nfilters; i++, f++) { 1351 if (f->valid) { 1352 MPASS(f->tid == tid_base + i); 1353 t->idx = i; 1354 t->l2tidx = f->l2te ? f->l2te->idx : 0; 1355 t->smtidx = f->smt ? f->smt->idx : 0; 1356 if (f->fs.hitcnts) 1357 t->hits = get_filter_hits(sc, f->tid); 1358 else 1359 t->hits = UINT64_MAX; 1360 t->fs = f->fs; 1361 1362 goto done; 1363 } 1364 } 1365 t->idx = 0xffffffff; 1366done: 1367 mtx_unlock(&sc->tids.ftid_lock); 1368 return (0); 1369} 1370 1371static int 1372get_hashfilter(struct adapter *sc, struct t4_filter *t) 1373{ 1374 struct tid_info *ti = &sc->tids; 1375 int tid; 1376 struct filter_entry *f; 1377 const int inv_tid = ti->ntids + ti->tid_base; 1378 1379 MPASS(t->fs.hash); 1380 1381 if (ti->tids_in_use == 0 || ti->hftid_hash_tid == NULL || 1382 t->idx >= inv_tid) { 1383 t->idx = 0xffffffff; 1384 return (0); 1385 } 1386 if (t->idx < ti->tid_base) 1387 t->idx = ti->tid_base; 1388 1389 mtx_lock(&ti->hftid_lock); 1390 for (tid = t->idx; tid < inv_tid; tid++) { 1391 f = lookup_hftid(sc, tid); 1392 if (f != NULL && f->valid) { 1393 t->idx = tid; 1394 t->l2tidx = f->l2te ? f->l2te->idx : 0; 1395 t->smtidx = f->smt ? f->smt->idx : 0; 1396 if (f->fs.hitcnts) 1397 t->hits = get_filter_hits(sc, tid); 1398 else 1399 t->hits = UINT64_MAX; 1400 t->fs = f->fs; 1401 1402 goto done; 1403 } 1404 } 1405 t->idx = 0xffffffff; 1406done: 1407 mtx_unlock(&ti->hftid_lock); 1408 return (0); 1409} 1410 1411static void 1412mk_act_open_req6(struct adapter *sc, struct filter_entry *f, int atid, 1413 uint64_t ftuple, struct cpl_act_open_req6 *cpl) 1414{ 1415 struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl; 1416 struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl; 1417 1418 /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */ 1419 MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6); 1420 MPASS(atid >= 0); 1421 1422 if (chip_id(sc) == CHELSIO_T5) { 1423 INIT_TP_WR(cpl5, 0); 1424 } else { 1425 INIT_TP_WR(cpl6, 0); 1426 cpl6->rsvd2 = 0; 1427 cpl6->opt3 = 0; 1428 } 1429 1430 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 1431 V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | 1432 V_TID_COOKIE(CPL_COOKIE_HASHFILTER))); 1433 cpl->local_port = htobe16(f->fs.val.dport); 1434 cpl->peer_port = htobe16(f->fs.val.sport); 1435 cpl->local_ip_hi = *(uint64_t *)(&f->fs.val.dip); 1436 cpl->local_ip_lo = *(((uint64_t *)&f->fs.val.dip) + 1); 1437 cpl->peer_ip_hi = *(uint64_t *)(&f->fs.val.sip); 1438 cpl->peer_ip_lo = *(((uint64_t *)&f->fs.val.sip) + 1); 1439 cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE || 1440 f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) | 1441 V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) | 1442 V_NO_CONG(f->fs.rpttid) | 1443 V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) | 1444 F_TCAM_BYPASS | F_NON_OFFLOAD); 1445 1446 cpl6->params = htobe64(V_FILTER_TUPLE(ftuple)); 1447 cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) | 1448 V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) | 1449 V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID | 1450 F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) | 1451 V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) | 1452 V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1))); 1453} 1454 1455static void 1456mk_act_open_req(struct adapter *sc, struct filter_entry *f, int atid, 1457 uint64_t ftuple, struct cpl_act_open_req *cpl) 1458{ 1459 struct cpl_t5_act_open_req *cpl5 = (void *)cpl; 1460 struct cpl_t6_act_open_req *cpl6 = (void *)cpl; 1461 1462 /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */ 1463 MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6); 1464 MPASS(atid >= 0); 1465 1466 if (chip_id(sc) == CHELSIO_T5) { 1467 INIT_TP_WR(cpl5, 0); 1468 } else { 1469 INIT_TP_WR(cpl6, 0); 1470 cpl6->rsvd2 = 0; 1471 cpl6->opt3 = 0; 1472 } 1473 1474 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 1475 V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | 1476 V_TID_COOKIE(CPL_COOKIE_HASHFILTER))); 1477 cpl->local_port = htobe16(f->fs.val.dport); 1478 cpl->peer_port = htobe16(f->fs.val.sport); 1479 cpl->local_ip = f->fs.val.dip[0] | f->fs.val.dip[1] << 8 | 1480 f->fs.val.dip[2] << 16 | f->fs.val.dip[3] << 24; 1481 cpl->peer_ip = f->fs.val.sip[0] | f->fs.val.sip[1] << 8 | 1482 f->fs.val.sip[2] << 16 | f->fs.val.sip[3] << 24; 1483 cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE || 1484 f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) | 1485 V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) | 1486 V_NO_CONG(f->fs.rpttid) | 1487 V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) | 1488 F_TCAM_BYPASS | F_NON_OFFLOAD); 1489 1490 cpl6->params = htobe64(V_FILTER_TUPLE(ftuple)); 1491 cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) | 1492 V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) | 1493 V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID | 1494 F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) | 1495 V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) | 1496 V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1))); 1497} 1498 1499static int 1500act_open_cpl_len16(struct adapter *sc, int isipv6) 1501{ 1502 int idx; 1503 static const int sz_table[3][2] = { 1504 { 1505 howmany(sizeof (struct cpl_act_open_req), 16), 1506 howmany(sizeof (struct cpl_act_open_req6), 16) 1507 }, 1508 { 1509 howmany(sizeof (struct cpl_t5_act_open_req), 16), 1510 howmany(sizeof (struct cpl_t5_act_open_req6), 16) 1511 }, 1512 { 1513 howmany(sizeof (struct cpl_t6_act_open_req), 16), 1514 howmany(sizeof (struct cpl_t6_act_open_req6), 16) 1515 }, 1516 }; 1517 1518 MPASS(chip_id(sc) >= CHELSIO_T4); 1519 idx = min(chip_id(sc) - CHELSIO_T4, 2); 1520 1521 return (sz_table[idx][!!isipv6]); 1522} 1523 1524static int 1525set_hashfilter(struct adapter *sc, struct t4_filter *t, uint64_t ftuple, 1526 struct l2t_entry *l2te, struct smt_entry *smt) 1527{ 1528 void *wr; 1529 struct wrq_cookie cookie; 1530 struct filter_entry *f; 1531 int rc, atid = -1; 1532 uint32_t hash; 1533 1534 MPASS(t->fs.hash); 1535 /* Already validated against fconf, iconf */ 1536 MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0); 1537 MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0); 1538 1539 hash = hf_hashfn_4t(&t->fs); 1540 1541 mtx_lock(&sc->tids.hftid_lock); 1542 if (lookup_hf(sc, &t->fs, hash) != NULL) { 1543 rc = EEXIST; 1544 goto done; 1545 } 1546 1547 f = malloc(sizeof(*f), M_CXGBE, M_ZERO | M_NOWAIT); 1548 if (__predict_false(f == NULL)) { 1549 rc = ENOMEM; 1550 goto done; 1551 } 1552 f->fs = t->fs; 1553 f->l2te = l2te; 1554 f->smt = smt; 1555 1556 atid = alloc_atid(sc, f); 1557 if (__predict_false(atid) == -1) { 1558 free(f, M_CXGBE); 1559 rc = EAGAIN; 1560 goto done; 1561 } 1562 MPASS(atid >= 0); 1563 1564 wr = start_wrq_wr(&sc->sge.ctrlq[0], act_open_cpl_len16(sc, f->fs.type), 1565 &cookie); 1566 if (wr == NULL) { 1567 free_atid(sc, atid); 1568 free(f, M_CXGBE); 1569 rc = ENOMEM; 1570 goto done; 1571 } 1572 if (f->fs.type) 1573 mk_act_open_req6(sc, f, atid, ftuple, wr); 1574 else 1575 mk_act_open_req(sc, f, atid, ftuple, wr); 1576 1577 f->locked = 1; /* ithread mustn't free f if ioctl is still around. */ 1578 f->pending = 1; 1579 f->tid = -1; 1580 insert_hf(sc, f, hash); 1581 commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie); 1582 1583 for (;;) { 1584 MPASS(f->locked); 1585 if (f->pending == 0) { 1586 if (f->valid) { 1587 rc = 0; 1588 f->locked = 0; 1589 t->idx = f->tid; 1590 } else { 1591 rc = f->tid; 1592 free(f, M_CXGBE); 1593 } 1594 break; 1595 } 1596 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) { 1597 f->locked = 0; 1598 rc = EINPROGRESS; 1599 break; 1600 } 1601 } 1602done: 1603 mtx_unlock(&sc->tids.hftid_lock); 1604 return (rc); 1605} 1606 1607/* SET_TCB_FIELD sent as a ULP command looks like this */ 1608#define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ 1609 sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) 1610 1611static void * 1612mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, uint64_t word, uint64_t mask, 1613 uint64_t val, uint32_t tid, uint32_t qid) 1614{ 1615 struct ulptx_idata *ulpsc; 1616 struct cpl_set_tcb_field_core *req; 1617 1618 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 1619 ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16)); 1620 1621 ulpsc = (struct ulptx_idata *)(ulpmc + 1); 1622 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 1623 ulpsc->len = htobe32(sizeof(*req)); 1624 1625 req = (struct cpl_set_tcb_field_core *)(ulpsc + 1); 1626 OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1627 req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(qid)); 1628 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0)); 1629 req->mask = htobe64(mask); 1630 req->val = htobe64(val); 1631 1632 ulpsc = (struct ulptx_idata *)(req + 1); 1633 if (LEN__SET_TCB_FIELD_ULP % 16) { 1634 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1635 ulpsc->len = htobe32(0); 1636 return (ulpsc + 1); 1637 } 1638 return (ulpsc); 1639} 1640 1641/* ABORT_REQ sent as a ULP command looks like this */ 1642#define LEN__ABORT_REQ_ULP (sizeof(struct ulp_txpkt) + \ 1643 sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_req_core)) 1644 1645static void * 1646mk_abort_req_ulp(struct ulp_txpkt *ulpmc, uint32_t tid) 1647{ 1648 struct ulptx_idata *ulpsc; 1649 struct cpl_abort_req_core *req; 1650 1651 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 1652 ulpmc->len = htobe32(howmany(LEN__ABORT_REQ_ULP, 16)); 1653 1654 ulpsc = (struct ulptx_idata *)(ulpmc + 1); 1655 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 1656 ulpsc->len = htobe32(sizeof(*req)); 1657 1658 req = (struct cpl_abort_req_core *)(ulpsc + 1); 1659 OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); 1660 req->rsvd0 = htonl(0); 1661 req->rsvd1 = 0; 1662 req->cmd = CPL_ABORT_NO_RST; 1663 1664 ulpsc = (struct ulptx_idata *)(req + 1); 1665 if (LEN__ABORT_REQ_ULP % 16) { 1666 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1667 ulpsc->len = htobe32(0); 1668 return (ulpsc + 1); 1669 } 1670 return (ulpsc); 1671} 1672 1673/* ABORT_RPL sent as a ULP command looks like this */ 1674#define LEN__ABORT_RPL_ULP (sizeof(struct ulp_txpkt) + \ 1675 sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_rpl_core)) 1676 1677static void * 1678mk_abort_rpl_ulp(struct ulp_txpkt *ulpmc, uint32_t tid) 1679{ 1680 struct ulptx_idata *ulpsc; 1681 struct cpl_abort_rpl_core *rpl; 1682 1683 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 1684 ulpmc->len = htobe32(howmany(LEN__ABORT_RPL_ULP, 16)); 1685 1686 ulpsc = (struct ulptx_idata *)(ulpmc + 1); 1687 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 1688 ulpsc->len = htobe32(sizeof(*rpl)); 1689 1690 rpl = (struct cpl_abort_rpl_core *)(ulpsc + 1); 1691 OPCODE_TID(rpl) = htobe32(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); 1692 rpl->rsvd0 = htonl(0); 1693 rpl->rsvd1 = 0; 1694 rpl->cmd = CPL_ABORT_NO_RST; 1695 1696 ulpsc = (struct ulptx_idata *)(rpl + 1); 1697 if (LEN__ABORT_RPL_ULP % 16) { 1698 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1699 ulpsc->len = htobe32(0); 1700 return (ulpsc + 1); 1701 } 1702 return (ulpsc); 1703} 1704 1705static inline int 1706del_hashfilter_wrlen(void) 1707{ 1708 1709 return (sizeof(struct work_request_hdr) + 1710 roundup2(LEN__SET_TCB_FIELD_ULP, 16) + 1711 roundup2(LEN__ABORT_REQ_ULP, 16) + 1712 roundup2(LEN__ABORT_RPL_ULP, 16)); 1713} 1714 1715static void 1716mk_del_hashfilter_wr(int tid, struct work_request_hdr *wrh, int wrlen, int qid) 1717{ 1718 struct ulp_txpkt *ulpmc; 1719 1720 INIT_ULPTX_WRH(wrh, wrlen, 0, 0); 1721 ulpmc = (struct ulp_txpkt *)(wrh + 1); 1722 ulpmc = mk_set_tcb_field_ulp(ulpmc, W_TCB_RSS_INFO, 1723 V_TCB_RSS_INFO(M_TCB_RSS_INFO), V_TCB_RSS_INFO(qid), tid, 0); 1724 ulpmc = mk_abort_req_ulp(ulpmc, tid); 1725 ulpmc = mk_abort_rpl_ulp(ulpmc, tid); 1726} 1727 1728static int 1729del_hashfilter(struct adapter *sc, struct t4_filter *t) 1730{ 1731 struct tid_info *ti = &sc->tids; 1732 void *wr; 1733 struct filter_entry *f; 1734 struct wrq_cookie cookie; 1735 int rc; 1736 const int wrlen = del_hashfilter_wrlen(); 1737 const int inv_tid = ti->ntids + ti->tid_base; 1738 1739 MPASS(sc->tids.hftid_hash_4t != NULL); 1740 MPASS(sc->tids.ntids > 0); 1741 1742 if (t->idx < sc->tids.tid_base || t->idx >= inv_tid) 1743 return (EINVAL); 1744 1745 mtx_lock(&ti->hftid_lock); 1746 f = lookup_hftid(sc, t->idx); 1747 if (f == NULL || f->valid == 0) { 1748 rc = EINVAL; 1749 goto done; 1750 } 1751 MPASS(f->tid == t->idx); 1752 if (f->locked) { 1753 rc = EPERM; 1754 goto done; 1755 } 1756 if (f->pending) { 1757 rc = EBUSY; 1758 goto done; 1759 } 1760 wr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(wrlen, 16), &cookie); 1761 if (wr == NULL) { 1762 rc = ENOMEM; 1763 goto done; 1764 } 1765 1766 mk_del_hashfilter_wr(t->idx, wr, wrlen, sc->sge.fwq.abs_id); 1767 f->locked = 1; 1768 f->pending = 1; 1769 commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie); 1770 t->fs = f->fs; /* extra info for the caller */ 1771 1772 for (;;) { 1773 MPASS(f->locked); 1774 if (f->pending == 0) { 1775 if (f->valid) { 1776 f->locked = 0; 1777 rc = EIO; 1778 } else { 1779 rc = 0; 1780 free(f, M_CXGBE); 1781 } 1782 break; 1783 } 1784 if (cv_wait_sig(&ti->hftid_cv, &ti->hftid_lock) != 0) { 1785 f->locked = 0; 1786 rc = EINPROGRESS; 1787 break; 1788 } 1789 } 1790done: 1791 mtx_unlock(&ti->hftid_lock); 1792 return (rc); 1793} 1794 1795#define WORD_MASK 0xffffffff 1796static void 1797set_nat_params(struct adapter *sc, struct filter_entry *f, const bool dip, 1798 const bool sip, const bool dp, const bool sp) 1799{ 1800 1801 if (dip) { 1802 if (f->fs.type) { 1803 set_tcb_field(sc, f->tid, W_TCB_SND_UNA_RAW, WORD_MASK, 1804 f->fs.nat_dip[15] | f->fs.nat_dip[14] << 8 | 1805 f->fs.nat_dip[13] << 16 | f->fs.nat_dip[12] << 24, 1); 1806 1807 set_tcb_field(sc, f->tid, 1808 W_TCB_SND_UNA_RAW + 1, WORD_MASK, 1809 f->fs.nat_dip[11] | f->fs.nat_dip[10] << 8 | 1810 f->fs.nat_dip[9] << 16 | f->fs.nat_dip[8] << 24, 1); 1811 1812 set_tcb_field(sc, f->tid, 1813 W_TCB_SND_UNA_RAW + 2, WORD_MASK, 1814 f->fs.nat_dip[7] | f->fs.nat_dip[6] << 8 | 1815 f->fs.nat_dip[5] << 16 | f->fs.nat_dip[4] << 24, 1); 1816 1817 set_tcb_field(sc, f->tid, 1818 W_TCB_SND_UNA_RAW + 3, WORD_MASK, 1819 f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 | 1820 f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1); 1821 } else { 1822 set_tcb_field(sc, f->tid, 1823 W_TCB_RX_FRAG3_LEN_RAW, WORD_MASK, 1824 f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 | 1825 f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1); 1826 } 1827 } 1828 1829 if (sip) { 1830 if (f->fs.type) { 1831 set_tcb_field(sc, f->tid, 1832 W_TCB_RX_FRAG2_PTR_RAW, WORD_MASK, 1833 f->fs.nat_sip[15] | f->fs.nat_sip[14] << 8 | 1834 f->fs.nat_sip[13] << 16 | f->fs.nat_sip[12] << 24, 1); 1835 1836 set_tcb_field(sc, f->tid, 1837 W_TCB_RX_FRAG2_PTR_RAW + 1, WORD_MASK, 1838 f->fs.nat_sip[11] | f->fs.nat_sip[10] << 8 | 1839 f->fs.nat_sip[9] << 16 | f->fs.nat_sip[8] << 24, 1); 1840 1841 set_tcb_field(sc, f->tid, 1842 W_TCB_RX_FRAG2_PTR_RAW + 2, WORD_MASK, 1843 f->fs.nat_sip[7] | f->fs.nat_sip[6] << 8 | 1844 f->fs.nat_sip[5] << 16 | f->fs.nat_sip[4] << 24, 1); 1845 1846 set_tcb_field(sc, f->tid, 1847 W_TCB_RX_FRAG2_PTR_RAW + 3, WORD_MASK, 1848 f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 | 1849 f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1); 1850 1851 } else { 1852 set_tcb_field(sc, f->tid, 1853 W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW, WORD_MASK, 1854 f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 | 1855 f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1); 1856 } 1857 } 1858 1859 set_tcb_field(sc, f->tid, W_TCB_PDU_HDR_LEN, WORD_MASK, 1860 (dp ? f->fs.nat_dport : 0) | (sp ? f->fs.nat_sport << 16 : 0), 1); 1861} 1862 1863/* 1864 * Returns EINPROGRESS to indicate that at least one TCB update was sent and the 1865 * last of the series of updates requested a reply. The reply informs the 1866 * driver that the filter is fully setup. 1867 */ 1868static int 1869configure_hashfilter_tcb(struct adapter *sc, struct filter_entry *f) 1870{ 1871 int updated = 0; 1872 1873 MPASS(f->tid < sc->tids.ntids); 1874 MPASS(f->fs.hash); 1875 MPASS(f->pending); 1876 MPASS(f->valid == 0); 1877 1878 if (f->fs.newdmac) { 1879 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECE, 1, 1); 1880 updated++; 1881 } 1882 1883 if (f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) { 1884 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_RFR, 1, 1); 1885 updated++; 1886 } 1887 1888 if (f->fs.newsmac) { 1889 MPASS(f->smt != NULL); 1890 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1); 1891 set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL, 1892 V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), V_TCB_SMAC_SEL(f->smt->idx), 1893 1); 1894 updated++; 1895 } 1896 1897 switch(f->fs.nat_mode) { 1898 case NAT_MODE_NONE: 1899 break; 1900 case NAT_MODE_DIP: 1901 set_nat_params(sc, f, true, false, false, false); 1902 updated++; 1903 break; 1904 case NAT_MODE_DIP_DP: 1905 set_nat_params(sc, f, true, false, true, false); 1906 updated++; 1907 break; 1908 case NAT_MODE_DIP_DP_SIP: 1909 set_nat_params(sc, f, true, true, true, false); 1910 updated++; 1911 break; 1912 case NAT_MODE_DIP_DP_SP: 1913 set_nat_params(sc, f, true, false, true, true); 1914 updated++; 1915 break; 1916 case NAT_MODE_SIP_SP: 1917 set_nat_params(sc, f, false, true, false, true); 1918 updated++; 1919 break; 1920 case NAT_MODE_DIP_SIP_SP: 1921 set_nat_params(sc, f, true, true, false, true); 1922 updated++; 1923 break; 1924 case NAT_MODE_ALL: 1925 set_nat_params(sc, f, true, true, true, true); 1926 updated++; 1927 break; 1928 default: 1929 MPASS(0); /* should have been validated earlier */ 1930 break; 1931 1932 } 1933 1934 if (f->fs.nat_seq_chk) { 1935 set_tcb_field(sc, f->tid, W_TCB_RCV_NXT, 1936 V_TCB_RCV_NXT(M_TCB_RCV_NXT), 1937 V_TCB_RCV_NXT(f->fs.nat_seq_chk), 1); 1938 updated++; 1939 } 1940 1941 if (is_t5(sc) && f->fs.action == FILTER_DROP) { 1942 /* 1943 * Migrating = 1, Non-offload = 0 to get a T5 hashfilter to drop. 1944 */ 1945 set_tcb_field(sc, f->tid, W_TCB_T_FLAGS, V_TF_NON_OFFLOAD(1) | 1946 V_TF_MIGRATING(1), V_TF_MIGRATING(1), 1); 1947 updated++; 1948 } 1949 1950 /* 1951 * Enable switching after all secondary resources (L2T entry, SMT entry, 1952 * etc.) are setup so that any switched packet will use correct 1953 * values. 1954 */ 1955 if (f->fs.action == FILTER_SWITCH) { 1956 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECN, 1, 1); 1957 updated++; 1958 } 1959 1960 if (f->fs.hitcnts || updated > 0) { 1961 set_tcb_field(sc, f->tid, W_TCB_TIMESTAMP, 1962 V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) | 1963 V_TCB_T_RTT_TS_RECENT_AGE(M_TCB_T_RTT_TS_RECENT_AGE), 1964 V_TCB_TIMESTAMP(0ULL) | V_TCB_T_RTT_TS_RECENT_AGE(0ULL), 0); 1965 return (EINPROGRESS); 1966 } 1967 1968 return (0); 1969} 1970