1// SPDX-License-Identifier: GPL-2.0-only 2/**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2005-2018 Solarflare Communications Inc. 5 * Copyright 2019-2020 Xilinx Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 as published 9 * by the Free Software Foundation, incorporated herein by reference. 10 */ 11 12#include "mcdi_filters.h" 13#include "mcdi.h" 14#include "nic.h" 15#include "rx_common.h" 16 17/* The maximum size of a shared RSS context */ 18/* TODO: this should really be from the mcdi protocol export */ 19#define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL 20 21#define EFX_EF10_FILTER_ID_INVALID 0xffff 22 23/* An arbitrary search limit for the software hash table */ 24#define EFX_EF10_FILTER_SEARCH_LIMIT 200 25 26static struct efx_filter_spec * 27efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table, 28 unsigned int filter_idx) 29{ 30 return (struct efx_filter_spec *)(table->entry[filter_idx].spec & 31 ~EFX_EF10_FILTER_FLAGS); 32} 33 34static unsigned int 35efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table, 36 unsigned int filter_idx) 37{ 38 return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS; 39} 40 41static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id) 42{ 43 WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID); 44 return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1); 45} 46 47static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id) 48{ 49 return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2); 50} 51 52static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx) 53{ 54 return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx; 55} 56 57/* 58 * Decide whether a filter should be exclusive or else should allow 59 * delivery to additional recipients. Currently we decide that 60 * filters for specific local unicast MAC and IP addresses are 61 * exclusive. 62 */ 63static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec) 64{ 65 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC && 66 !is_multicast_ether_addr(spec->loc_mac)) 67 return true; 68 69 if ((spec->match_flags & 70 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) == 71 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) { 72 if (spec->ether_type == htons(ETH_P_IP) && 73 !ipv4_is_multicast(spec->loc_host[0])) 74 return true; 75 if (spec->ether_type == htons(ETH_P_IPV6) && 76 ((const u8 *)spec->loc_host)[0] != 0xff) 77 return true; 78 } 79 80 return false; 81} 82 83static void 84efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table, 85 unsigned int filter_idx, 86 const struct efx_filter_spec *spec, 87 unsigned int flags) 88{ 89 table->entry[filter_idx].spec = (unsigned long)spec | flags; 90} 91 92static void 93efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx, 94 const struct efx_filter_spec *spec, 95 efx_dword_t *inbuf) 96{ 97 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); 98 u32 match_fields = 0, uc_match, mc_match; 99 100 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 101 efx_mcdi_filter_is_exclusive(spec) ? 102 MC_CMD_FILTER_OP_IN_OP_INSERT : 103 MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE); 104 105 /* 106 * Convert match flags and values. Unlike almost 107 * everything else in MCDI, these fields are in 108 * network byte order. 109 */ 110#define COPY_VALUE(value, mcdi_field) \ 111 do { \ 112 match_fields |= \ 113 1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \ 114 mcdi_field ## _LBN; \ 115 BUILD_BUG_ON( \ 116 MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \ 117 sizeof(value)); \ 118 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \ 119 &value, sizeof(value)); \ 120 } while (0) 121#define COPY_FIELD(gen_flag, gen_field, mcdi_field) \ 122 if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) { \ 123 COPY_VALUE(spec->gen_field, mcdi_field); \ 124 } 125 /* 126 * Handle encap filters first. They will always be mismatch 127 * (unknown UC or MC) filters 128 */ 129 if (encap_type) { 130 /* 131 * ether_type and outer_ip_proto need to be variables 132 * because COPY_VALUE wants to memcpy them 133 */ 134 __be16 ether_type = 135 htons(encap_type & EFX_ENCAP_FLAG_IPV6 ? 136 ETH_P_IPV6 : ETH_P_IP); 137 u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE; 138 u8 outer_ip_proto; 139 140 switch (encap_type & EFX_ENCAP_TYPES_MASK) { 141 case EFX_ENCAP_TYPE_VXLAN: 142 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN; 143 fallthrough; 144 case EFX_ENCAP_TYPE_GENEVE: 145 COPY_VALUE(ether_type, ETHER_TYPE); 146 outer_ip_proto = IPPROTO_UDP; 147 COPY_VALUE(outer_ip_proto, IP_PROTO); 148 /* 149 * We always need to set the type field, even 150 * though we're not matching on the TNI. 151 */ 152 MCDI_POPULATE_DWORD_1(inbuf, 153 FILTER_OP_EXT_IN_VNI_OR_VSID, 154 FILTER_OP_EXT_IN_VNI_TYPE, 155 vni_type); 156 break; 157 case EFX_ENCAP_TYPE_NVGRE: 158 COPY_VALUE(ether_type, ETHER_TYPE); 159 outer_ip_proto = IPPROTO_GRE; 160 COPY_VALUE(outer_ip_proto, IP_PROTO); 161 break; 162 default: 163 WARN_ON(1); 164 } 165 166 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; 167 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; 168 } else { 169 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; 170 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; 171 } 172 173 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) 174 match_fields |= 175 is_multicast_ether_addr(spec->loc_mac) ? 176 1 << mc_match : 177 1 << uc_match; 178 COPY_FIELD(REM_HOST, rem_host, SRC_IP); 179 COPY_FIELD(LOC_HOST, loc_host, DST_IP); 180 COPY_FIELD(REM_MAC, rem_mac, SRC_MAC); 181 COPY_FIELD(REM_PORT, rem_port, SRC_PORT); 182 COPY_FIELD(LOC_MAC, loc_mac, DST_MAC); 183 COPY_FIELD(LOC_PORT, loc_port, DST_PORT); 184 COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE); 185 COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN); 186 COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN); 187 COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO); 188#undef COPY_FIELD 189#undef COPY_VALUE 190 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS, 191 match_fields); 192} 193 194static void efx_mcdi_filter_push_prep(struct efx_nic *efx, 195 const struct efx_filter_spec *spec, 196 efx_dword_t *inbuf, u64 handle, 197 struct efx_rss_context *ctx, 198 bool replacing) 199{ 200 u32 flags = spec->flags; 201 202 memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN); 203 204 /* If RSS filter, caller better have given us an RSS context */ 205 if (flags & EFX_FILTER_FLAG_RX_RSS) { 206 /* 207 * We don't have the ability to return an error, so we'll just 208 * log a warning and disable RSS for the filter. 209 */ 210 if (WARN_ON_ONCE(!ctx)) 211 flags &= ~EFX_FILTER_FLAG_RX_RSS; 212 else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)) 213 flags &= ~EFX_FILTER_FLAG_RX_RSS; 214 } 215 216 if (replacing) { 217 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 218 MC_CMD_FILTER_OP_IN_OP_REPLACE); 219 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle); 220 } else { 221 efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf); 222 } 223 224 if (flags & EFX_FILTER_FLAG_VPORT_ID) 225 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, spec->vport_id); 226 else 227 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id); 228 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST, 229 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? 230 MC_CMD_FILTER_OP_IN_RX_DEST_DROP : 231 MC_CMD_FILTER_OP_IN_RX_DEST_HOST); 232 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0); 233 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST, 234 MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT); 235 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE, 236 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? 237 0 : spec->dmaq_id); 238 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE, 239 (flags & EFX_FILTER_FLAG_RX_RSS) ? 240 MC_CMD_FILTER_OP_IN_RX_MODE_RSS : 241 MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE); 242 if (flags & EFX_FILTER_FLAG_RX_RSS) 243 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id); 244} 245 246static int efx_mcdi_filter_push(struct efx_nic *efx, 247 const struct efx_filter_spec *spec, u64 *handle, 248 struct efx_rss_context *ctx, bool replacing) 249{ 250 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 251 MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN); 252 size_t outlen; 253 int rc; 254 255 efx_mcdi_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing); 256 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 257 outbuf, sizeof(outbuf), &outlen); 258 if (rc && spec->priority != EFX_FILTER_PRI_HINT) 259 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, sizeof(inbuf), 260 outbuf, outlen, rc); 261 if (rc == 0) 262 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE); 263 if (rc == -ENOSPC) 264 rc = -EBUSY; /* to match efx_farch_filter_insert() */ 265 return rc; 266} 267 268static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec) 269{ 270 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); 271 unsigned int match_flags = spec->match_flags; 272 unsigned int uc_match, mc_match; 273 u32 mcdi_flags = 0; 274 275#define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) { \ 276 unsigned int old_match_flags = match_flags; \ 277 match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag; \ 278 if (match_flags != old_match_flags) \ 279 mcdi_flags |= \ 280 (1 << ((encap) ? \ 281 MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \ 282 mcdi_field ## _LBN : \ 283 MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\ 284 mcdi_field ## _LBN)); \ 285 } 286 /* inner or outer based on encap type */ 287 MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type); 288 MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type); 289 MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type); 290 MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type); 291 MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type); 292 MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type); 293 MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type); 294 MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type); 295 /* always outer */ 296 MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false); 297 MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false); 298#undef MAP_FILTER_TO_MCDI_FLAG 299 300 /* special handling for encap type, and mismatch */ 301 if (encap_type) { 302 match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE; 303 mcdi_flags |= 304 (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); 305 mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); 306 307 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; 308 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; 309 } else { 310 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; 311 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; 312 } 313 314 if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) { 315 match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG; 316 mcdi_flags |= 317 is_multicast_ether_addr(spec->loc_mac) ? 318 1 << mc_match : 319 1 << uc_match; 320 } 321 322 /* Did we map them all? */ 323 WARN_ON_ONCE(match_flags); 324 325 return mcdi_flags; 326} 327 328static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table, 329 const struct efx_filter_spec *spec) 330{ 331 u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec); 332 unsigned int match_pri; 333 334 for (match_pri = 0; 335 match_pri < table->rx_match_count; 336 match_pri++) 337 if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags) 338 return match_pri; 339 340 return -EPROTONOSUPPORT; 341} 342 343static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx, 344 struct efx_filter_spec *spec, 345 bool replace_equal) 346{ 347 DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); 348 struct efx_mcdi_filter_table *table; 349 struct efx_filter_spec *saved_spec; 350 struct efx_rss_context *ctx = NULL; 351 unsigned int match_pri, hash; 352 unsigned int priv_flags; 353 bool rss_locked = false; 354 bool replacing = false; 355 unsigned int depth, i; 356 int ins_index = -1; 357 DEFINE_WAIT(wait); 358 bool is_mc_recip; 359 s32 rc; 360 361 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 362 table = efx->filter_state; 363 down_write(&table->lock); 364 365 /* For now, only support RX filters */ 366 if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) != 367 EFX_FILTER_FLAG_RX) { 368 rc = -EINVAL; 369 goto out_unlock; 370 } 371 372 rc = efx_mcdi_filter_pri(table, spec); 373 if (rc < 0) 374 goto out_unlock; 375 match_pri = rc; 376 377 hash = efx_filter_spec_hash(spec); 378 is_mc_recip = efx_filter_is_mc_recipient(spec); 379 if (is_mc_recip) 380 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); 381 382 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { 383 mutex_lock(&efx->rss_lock); 384 rss_locked = true; 385 if (spec->rss_context) 386 ctx = efx_find_rss_context_entry(efx, spec->rss_context); 387 else 388 ctx = &efx->rss_context; 389 if (!ctx) { 390 rc = -ENOENT; 391 goto out_unlock; 392 } 393 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { 394 rc = -EOPNOTSUPP; 395 goto out_unlock; 396 } 397 } 398 399 /* Find any existing filters with the same match tuple or 400 * else a free slot to insert at. 401 */ 402 for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { 403 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1); 404 saved_spec = efx_mcdi_filter_entry_spec(table, i); 405 406 if (!saved_spec) { 407 if (ins_index < 0) 408 ins_index = i; 409 } else if (efx_filter_spec_equal(spec, saved_spec)) { 410 if (spec->priority < saved_spec->priority && 411 spec->priority != EFX_FILTER_PRI_AUTO) { 412 rc = -EPERM; 413 goto out_unlock; 414 } 415 if (!is_mc_recip) { 416 /* This is the only one */ 417 if (spec->priority == 418 saved_spec->priority && 419 !replace_equal) { 420 rc = -EEXIST; 421 goto out_unlock; 422 } 423 ins_index = i; 424 break; 425 } else if (spec->priority > 426 saved_spec->priority || 427 (spec->priority == 428 saved_spec->priority && 429 replace_equal)) { 430 if (ins_index < 0) 431 ins_index = i; 432 else 433 __set_bit(depth, mc_rem_map); 434 } 435 } 436 } 437 438 /* Once we reach the maximum search depth, use the first suitable 439 * slot, or return -EBUSY if there was none 440 */ 441 if (ins_index < 0) { 442 rc = -EBUSY; 443 goto out_unlock; 444 } 445 446 /* Create a software table entry if necessary. */ 447 saved_spec = efx_mcdi_filter_entry_spec(table, ins_index); 448 if (saved_spec) { 449 if (spec->priority == EFX_FILTER_PRI_AUTO && 450 saved_spec->priority >= EFX_FILTER_PRI_AUTO) { 451 /* Just make sure it won't be removed */ 452 if (saved_spec->priority > EFX_FILTER_PRI_AUTO) 453 saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO; 454 table->entry[ins_index].spec &= 455 ~EFX_EF10_FILTER_FLAG_AUTO_OLD; 456 rc = ins_index; 457 goto out_unlock; 458 } 459 replacing = true; 460 priv_flags = efx_mcdi_filter_entry_flags(table, ins_index); 461 } else { 462 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC); 463 if (!saved_spec) { 464 rc = -ENOMEM; 465 goto out_unlock; 466 } 467 *saved_spec = *spec; 468 priv_flags = 0; 469 } 470 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags); 471 472 /* Actually insert the filter on the HW */ 473 rc = efx_mcdi_filter_push(efx, spec, &table->entry[ins_index].handle, 474 ctx, replacing); 475 476 if (rc == -EINVAL && efx->must_realloc_vis) 477 /* The MC rebooted under us, causing it to reject our filter 478 * insertion as pointing to an invalid VI (spec->dmaq_id). 479 */ 480 rc = -EAGAIN; 481 482 /* Finalise the software table entry */ 483 if (rc == 0) { 484 if (replacing) { 485 /* Update the fields that may differ */ 486 if (saved_spec->priority == EFX_FILTER_PRI_AUTO) 487 saved_spec->flags |= 488 EFX_FILTER_FLAG_RX_OVER_AUTO; 489 saved_spec->priority = spec->priority; 490 saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO; 491 saved_spec->flags |= spec->flags; 492 saved_spec->rss_context = spec->rss_context; 493 saved_spec->dmaq_id = spec->dmaq_id; 494 saved_spec->vport_id = spec->vport_id; 495 } 496 } else if (!replacing) { 497 kfree(saved_spec); 498 saved_spec = NULL; 499 } else { 500 /* We failed to replace, so the old filter is still present. 501 * Roll back the software table to reflect this. In fact the 502 * efx_mcdi_filter_set_entry() call below will do the right 503 * thing, so nothing extra is needed here. 504 */ 505 } 506 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags); 507 508 /* Remove and finalise entries for lower-priority multicast 509 * recipients 510 */ 511 if (is_mc_recip) { 512 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 513 unsigned int depth, i; 514 515 memset(inbuf, 0, sizeof(inbuf)); 516 517 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { 518 if (!test_bit(depth, mc_rem_map)) 519 continue; 520 521 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1); 522 saved_spec = efx_mcdi_filter_entry_spec(table, i); 523 priv_flags = efx_mcdi_filter_entry_flags(table, i); 524 525 if (rc == 0) { 526 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 527 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 528 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 529 table->entry[i].handle); 530 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, 531 inbuf, sizeof(inbuf), 532 NULL, 0, NULL); 533 } 534 535 if (rc == 0) { 536 kfree(saved_spec); 537 saved_spec = NULL; 538 priv_flags = 0; 539 } 540 efx_mcdi_filter_set_entry(table, i, saved_spec, 541 priv_flags); 542 } 543 } 544 545 /* If successful, return the inserted filter ID */ 546 if (rc == 0) 547 rc = efx_mcdi_filter_make_filter_id(match_pri, ins_index); 548 549out_unlock: 550 if (rss_locked) 551 mutex_unlock(&efx->rss_lock); 552 up_write(&table->lock); 553 return rc; 554} 555 556s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec, 557 bool replace_equal) 558{ 559 s32 ret; 560 561 down_read(&efx->filter_sem); 562 ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal); 563 up_read(&efx->filter_sem); 564 565 return ret; 566} 567 568/* 569 * Remove a filter. 570 * If !by_index, remove by ID 571 * If by_index, remove by index 572 * Filter ID may come from userland and must be range-checked. 573 * Caller must hold efx->filter_sem for read, and efx->filter_state->lock 574 * for write. 575 */ 576static int efx_mcdi_filter_remove_internal(struct efx_nic *efx, 577 unsigned int priority_mask, 578 u32 filter_id, bool by_index) 579{ 580 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id); 581 struct efx_mcdi_filter_table *table = efx->filter_state; 582 MCDI_DECLARE_BUF(inbuf, 583 MC_CMD_FILTER_OP_IN_HANDLE_OFST + 584 MC_CMD_FILTER_OP_IN_HANDLE_LEN); 585 struct efx_filter_spec *spec; 586 DEFINE_WAIT(wait); 587 int rc; 588 589 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 590 if (!spec || 591 (!by_index && 592 efx_mcdi_filter_pri(table, spec) != 593 efx_mcdi_filter_get_unsafe_pri(filter_id))) 594 return -ENOENT; 595 596 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO && 597 priority_mask == (1U << EFX_FILTER_PRI_AUTO)) { 598 /* Just remove flags */ 599 spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO; 600 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD; 601 return 0; 602 } 603 604 if (!(priority_mask & (1U << spec->priority))) 605 return -ENOENT; 606 607 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) { 608 /* Reset to an automatic filter */ 609 610 struct efx_filter_spec new_spec = *spec; 611 612 new_spec.priority = EFX_FILTER_PRI_AUTO; 613 new_spec.flags = (EFX_FILTER_FLAG_RX | 614 (efx_rss_active(&efx->rss_context) ? 615 EFX_FILTER_FLAG_RX_RSS : 0)); 616 new_spec.dmaq_id = 0; 617 new_spec.rss_context = 0; 618 rc = efx_mcdi_filter_push(efx, &new_spec, 619 &table->entry[filter_idx].handle, 620 &efx->rss_context, 621 true); 622 623 if (rc == 0) 624 *spec = new_spec; 625 } else { 626 /* Really remove the filter */ 627 628 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 629 efx_mcdi_filter_is_exclusive(spec) ? 630 MC_CMD_FILTER_OP_IN_OP_REMOVE : 631 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 632 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 633 table->entry[filter_idx].handle); 634 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, 635 inbuf, sizeof(inbuf), NULL, 0, NULL); 636 637 if ((rc == 0) || (rc == -ENOENT)) { 638 /* Filter removed OK or didn't actually exist */ 639 kfree(spec); 640 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0); 641 } else { 642 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, 643 MC_CMD_FILTER_OP_EXT_IN_LEN, 644 NULL, 0, rc); 645 } 646 } 647 648 return rc; 649} 650 651/* Remove filters that weren't renewed. */ 652static void efx_mcdi_filter_remove_old(struct efx_nic *efx) 653{ 654 struct efx_mcdi_filter_table *table = efx->filter_state; 655 int remove_failed = 0; 656 int remove_noent = 0; 657 int rc; 658 int i; 659 660 down_write(&table->lock); 661 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) { 662 if (READ_ONCE(table->entry[i].spec) & 663 EFX_EF10_FILTER_FLAG_AUTO_OLD) { 664 rc = efx_mcdi_filter_remove_internal(efx, 665 1U << EFX_FILTER_PRI_AUTO, i, true); 666 if (rc == -ENOENT) 667 remove_noent++; 668 else if (rc) 669 remove_failed++; 670 } 671 } 672 up_write(&table->lock); 673 674 if (remove_failed) 675 netif_info(efx, drv, efx->net_dev, 676 "%s: failed to remove %d filters\n", 677 __func__, remove_failed); 678 if (remove_noent) 679 netif_info(efx, drv, efx->net_dev, 680 "%s: failed to remove %d non-existent filters\n", 681 __func__, remove_noent); 682} 683 684int efx_mcdi_filter_remove_safe(struct efx_nic *efx, 685 enum efx_filter_priority priority, 686 u32 filter_id) 687{ 688 struct efx_mcdi_filter_table *table; 689 int rc; 690 691 down_read(&efx->filter_sem); 692 table = efx->filter_state; 693 down_write(&table->lock); 694 rc = efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id, 695 false); 696 up_write(&table->lock); 697 up_read(&efx->filter_sem); 698 return rc; 699} 700 701/* Caller must hold efx->filter_sem for read */ 702static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx, 703 enum efx_filter_priority priority, 704 u32 filter_id) 705{ 706 struct efx_mcdi_filter_table *table = efx->filter_state; 707 708 if (filter_id == EFX_EF10_FILTER_ID_INVALID) 709 return; 710 711 down_write(&table->lock); 712 efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id, 713 true); 714 up_write(&table->lock); 715} 716 717int efx_mcdi_filter_get_safe(struct efx_nic *efx, 718 enum efx_filter_priority priority, 719 u32 filter_id, struct efx_filter_spec *spec) 720{ 721 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id); 722 const struct efx_filter_spec *saved_spec; 723 struct efx_mcdi_filter_table *table; 724 int rc; 725 726 down_read(&efx->filter_sem); 727 table = efx->filter_state; 728 down_read(&table->lock); 729 saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx); 730 if (saved_spec && saved_spec->priority == priority && 731 efx_mcdi_filter_pri(table, saved_spec) == 732 efx_mcdi_filter_get_unsafe_pri(filter_id)) { 733 *spec = *saved_spec; 734 rc = 0; 735 } else { 736 rc = -ENOENT; 737 } 738 up_read(&table->lock); 739 up_read(&efx->filter_sem); 740 return rc; 741} 742 743static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx, 744 struct efx_mcdi_filter_vlan *vlan, 745 bool multicast, bool rollback) 746{ 747 struct efx_mcdi_filter_table *table = efx->filter_state; 748 struct efx_mcdi_dev_addr *addr_list; 749 enum efx_filter_flags filter_flags; 750 struct efx_filter_spec spec; 751 u8 baddr[ETH_ALEN]; 752 unsigned int i, j; 753 int addr_count; 754 u16 *ids; 755 int rc; 756 757 if (multicast) { 758 addr_list = table->dev_mc_list; 759 addr_count = table->dev_mc_count; 760 ids = vlan->mc; 761 } else { 762 addr_list = table->dev_uc_list; 763 addr_count = table->dev_uc_count; 764 ids = vlan->uc; 765 } 766 767 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0; 768 769 /* Insert/renew filters */ 770 for (i = 0; i < addr_count; i++) { 771 EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID); 772 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 773 efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr); 774 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 775 if (rc < 0) { 776 if (rollback) { 777 netif_info(efx, drv, efx->net_dev, 778 "efx_mcdi_filter_insert failed rc=%d\n", 779 rc); 780 /* Fall back to promiscuous */ 781 for (j = 0; j < i; j++) { 782 efx_mcdi_filter_remove_unsafe( 783 efx, EFX_FILTER_PRI_AUTO, 784 ids[j]); 785 ids[j] = EFX_EF10_FILTER_ID_INVALID; 786 } 787 return rc; 788 } else { 789 /* keep invalid ID, and carry on */ 790 } 791 } else { 792 ids[i] = efx_mcdi_filter_get_unsafe_id(rc); 793 } 794 } 795 796 if (multicast && rollback) { 797 /* Also need an Ethernet broadcast filter */ 798 EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] != 799 EFX_EF10_FILTER_ID_INVALID); 800 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 801 eth_broadcast_addr(baddr); 802 efx_filter_set_eth_local(&spec, vlan->vid, baddr); 803 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 804 if (rc < 0) { 805 netif_warn(efx, drv, efx->net_dev, 806 "Broadcast filter insert failed rc=%d\n", rc); 807 /* Fall back to promiscuous */ 808 for (j = 0; j < i; j++) { 809 efx_mcdi_filter_remove_unsafe( 810 efx, EFX_FILTER_PRI_AUTO, 811 ids[j]); 812 ids[j] = EFX_EF10_FILTER_ID_INVALID; 813 } 814 return rc; 815 } else { 816 vlan->default_filters[EFX_EF10_BCAST] = 817 efx_mcdi_filter_get_unsafe_id(rc); 818 } 819 } 820 821 return 0; 822} 823 824static int efx_mcdi_filter_insert_def(struct efx_nic *efx, 825 struct efx_mcdi_filter_vlan *vlan, 826 enum efx_encap_type encap_type, 827 bool multicast, bool rollback) 828{ 829 struct efx_mcdi_filter_table *table = efx->filter_state; 830 enum efx_filter_flags filter_flags; 831 struct efx_filter_spec spec; 832 u8 baddr[ETH_ALEN]; 833 int rc; 834 u16 *id; 835 836 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0; 837 838 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 839 840 if (multicast) 841 efx_filter_set_mc_def(&spec); 842 else 843 efx_filter_set_uc_def(&spec); 844 845 if (encap_type) { 846 if (efx_has_cap(efx, VXLAN_NVGRE)) 847 efx_filter_set_encap_type(&spec, encap_type); 848 else 849 /* 850 * don't insert encap filters on non-supporting 851 * platforms. ID will be left as INVALID. 852 */ 853 return 0; 854 } 855 856 if (vlan->vid != EFX_FILTER_VID_UNSPEC) 857 efx_filter_set_eth_local(&spec, vlan->vid, NULL); 858 859 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 860 if (rc < 0) { 861 const char *um = multicast ? "Multicast" : "Unicast"; 862 const char *encap_name = ""; 863 const char *encap_ipv = ""; 864 865 if ((encap_type & EFX_ENCAP_TYPES_MASK) == 866 EFX_ENCAP_TYPE_VXLAN) 867 encap_name = "VXLAN "; 868 else if ((encap_type & EFX_ENCAP_TYPES_MASK) == 869 EFX_ENCAP_TYPE_NVGRE) 870 encap_name = "NVGRE "; 871 else if ((encap_type & EFX_ENCAP_TYPES_MASK) == 872 EFX_ENCAP_TYPE_GENEVE) 873 encap_name = "GENEVE "; 874 if (encap_type & EFX_ENCAP_FLAG_IPV6) 875 encap_ipv = "IPv6 "; 876 else if (encap_type) 877 encap_ipv = "IPv4 "; 878 879 /* 880 * unprivileged functions can't insert mismatch filters 881 * for encapsulated or unicast traffic, so downgrade 882 * those warnings to debug. 883 */ 884 netif_cond_dbg(efx, drv, efx->net_dev, 885 rc == -EPERM && (encap_type || !multicast), warn, 886 "%s%s%s mismatch filter insert failed rc=%d\n", 887 encap_name, encap_ipv, um, rc); 888 } else if (multicast) { 889 /* mapping from encap types to default filter IDs (multicast) */ 890 static enum efx_mcdi_filter_default_filters map[] = { 891 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF, 892 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF, 893 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF, 894 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF, 895 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = 896 EFX_EF10_VXLAN6_MCDEF, 897 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = 898 EFX_EF10_NVGRE6_MCDEF, 899 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = 900 EFX_EF10_GENEVE6_MCDEF, 901 }; 902 903 /* quick bounds check (BCAST result impossible) */ 904 BUILD_BUG_ON(EFX_EF10_BCAST != 0); 905 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { 906 WARN_ON(1); 907 return -EINVAL; 908 } 909 /* then follow map */ 910 id = &vlan->default_filters[map[encap_type]]; 911 912 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); 913 *id = efx_mcdi_filter_get_unsafe_id(rc); 914 if (!table->mc_chaining && !encap_type) { 915 /* Also need an Ethernet broadcast filter */ 916 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, 917 filter_flags, 0); 918 eth_broadcast_addr(baddr); 919 efx_filter_set_eth_local(&spec, vlan->vid, baddr); 920 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 921 if (rc < 0) { 922 netif_warn(efx, drv, efx->net_dev, 923 "Broadcast filter insert failed rc=%d\n", 924 rc); 925 if (rollback) { 926 /* Roll back the mc_def filter */ 927 efx_mcdi_filter_remove_unsafe( 928 efx, EFX_FILTER_PRI_AUTO, 929 *id); 930 *id = EFX_EF10_FILTER_ID_INVALID; 931 return rc; 932 } 933 } else { 934 EFX_WARN_ON_PARANOID( 935 vlan->default_filters[EFX_EF10_BCAST] != 936 EFX_EF10_FILTER_ID_INVALID); 937 vlan->default_filters[EFX_EF10_BCAST] = 938 efx_mcdi_filter_get_unsafe_id(rc); 939 } 940 } 941 rc = 0; 942 } else { 943 /* mapping from encap types to default filter IDs (unicast) */ 944 static enum efx_mcdi_filter_default_filters map[] = { 945 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF, 946 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF, 947 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF, 948 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF, 949 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = 950 EFX_EF10_VXLAN6_UCDEF, 951 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = 952 EFX_EF10_NVGRE6_UCDEF, 953 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = 954 EFX_EF10_GENEVE6_UCDEF, 955 }; 956 957 /* quick bounds check (BCAST result impossible) */ 958 BUILD_BUG_ON(EFX_EF10_BCAST != 0); 959 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { 960 WARN_ON(1); 961 return -EINVAL; 962 } 963 /* then follow map */ 964 id = &vlan->default_filters[map[encap_type]]; 965 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); 966 *id = rc; 967 rc = 0; 968 } 969 return rc; 970} 971 972/* 973 * Caller must hold efx->filter_sem for read if race against 974 * efx_mcdi_filter_table_remove() is possible 975 */ 976static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx, 977 struct efx_mcdi_filter_vlan *vlan) 978{ 979 struct efx_mcdi_filter_table *table = efx->filter_state; 980 981 /* 982 * Do not install unspecified VID if VLAN filtering is enabled. 983 * Do not install all specified VIDs if VLAN filtering is disabled. 984 */ 985 if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter) 986 return; 987 988 /* Insert/renew unicast filters */ 989 if (table->uc_promisc) { 990 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE, 991 false, false); 992 efx_mcdi_filter_insert_addr_list(efx, vlan, false, false); 993 } else { 994 /* 995 * If any of the filters failed to insert, fall back to 996 * promiscuous mode - add in the uc_def filter. But keep 997 * our individual unicast filters. 998 */ 999 if (efx_mcdi_filter_insert_addr_list(efx, vlan, false, false)) 1000 efx_mcdi_filter_insert_def(efx, vlan, 1001 EFX_ENCAP_TYPE_NONE, 1002 false, false); 1003 } 1004 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN, 1005 false, false); 1006 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN | 1007 EFX_ENCAP_FLAG_IPV6, 1008 false, false); 1009 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE, 1010 false, false); 1011 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE | 1012 EFX_ENCAP_FLAG_IPV6, 1013 false, false); 1014 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE, 1015 false, false); 1016 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE | 1017 EFX_ENCAP_FLAG_IPV6, 1018 false, false); 1019 1020 /* 1021 * Insert/renew multicast filters 1022 * 1023 * If changing promiscuous state with cascaded multicast filters, remove 1024 * old filters first, so that packets are dropped rather than duplicated 1025 */ 1026 if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc) 1027 efx_mcdi_filter_remove_old(efx); 1028 if (table->mc_promisc) { 1029 if (table->mc_chaining) { 1030 /* 1031 * If we failed to insert promiscuous filters, rollback 1032 * and fall back to individual multicast filters 1033 */ 1034 if (efx_mcdi_filter_insert_def(efx, vlan, 1035 EFX_ENCAP_TYPE_NONE, 1036 true, true)) { 1037 /* Changing promisc state, so remove old filters */ 1038 efx_mcdi_filter_remove_old(efx); 1039 efx_mcdi_filter_insert_addr_list(efx, vlan, 1040 true, false); 1041 } 1042 } else { 1043 /* 1044 * If we failed to insert promiscuous filters, don't 1045 * rollback. Regardless, also insert the mc_list, 1046 * unless it's incomplete due to overflow 1047 */ 1048 efx_mcdi_filter_insert_def(efx, vlan, 1049 EFX_ENCAP_TYPE_NONE, 1050 true, false); 1051 if (!table->mc_overflow) 1052 efx_mcdi_filter_insert_addr_list(efx, vlan, 1053 true, false); 1054 } 1055 } else { 1056 /* 1057 * If any filters failed to insert, rollback and fall back to 1058 * promiscuous mode - mc_def filter and maybe broadcast. If 1059 * that fails, roll back again and insert as many of our 1060 * individual multicast filters as we can. 1061 */ 1062 if (efx_mcdi_filter_insert_addr_list(efx, vlan, true, true)) { 1063 /* Changing promisc state, so remove old filters */ 1064 if (table->mc_chaining) 1065 efx_mcdi_filter_remove_old(efx); 1066 if (efx_mcdi_filter_insert_def(efx, vlan, 1067 EFX_ENCAP_TYPE_NONE, 1068 true, true)) 1069 efx_mcdi_filter_insert_addr_list(efx, vlan, 1070 true, false); 1071 } 1072 } 1073 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN, 1074 true, false); 1075 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN | 1076 EFX_ENCAP_FLAG_IPV6, 1077 true, false); 1078 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE, 1079 true, false); 1080 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE | 1081 EFX_ENCAP_FLAG_IPV6, 1082 true, false); 1083 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE, 1084 true, false); 1085 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE | 1086 EFX_ENCAP_FLAG_IPV6, 1087 true, false); 1088} 1089 1090int efx_mcdi_filter_clear_rx(struct efx_nic *efx, 1091 enum efx_filter_priority priority) 1092{ 1093 struct efx_mcdi_filter_table *table; 1094 unsigned int priority_mask; 1095 unsigned int i; 1096 int rc; 1097 1098 priority_mask = (((1U << (priority + 1)) - 1) & 1099 ~(1U << EFX_FILTER_PRI_AUTO)); 1100 1101 down_read(&efx->filter_sem); 1102 table = efx->filter_state; 1103 down_write(&table->lock); 1104 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) { 1105 rc = efx_mcdi_filter_remove_internal(efx, priority_mask, 1106 i, true); 1107 if (rc && rc != -ENOENT) 1108 break; 1109 rc = 0; 1110 } 1111 1112 up_write(&table->lock); 1113 up_read(&efx->filter_sem); 1114 return rc; 1115} 1116 1117u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx, 1118 enum efx_filter_priority priority) 1119{ 1120 struct efx_mcdi_filter_table *table; 1121 unsigned int filter_idx; 1122 s32 count = 0; 1123 1124 down_read(&efx->filter_sem); 1125 table = efx->filter_state; 1126 down_read(&table->lock); 1127 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1128 if (table->entry[filter_idx].spec && 1129 efx_mcdi_filter_entry_spec(table, filter_idx)->priority == 1130 priority) 1131 ++count; 1132 } 1133 up_read(&table->lock); 1134 up_read(&efx->filter_sem); 1135 return count; 1136} 1137 1138u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx) 1139{ 1140 struct efx_mcdi_filter_table *table = efx->filter_state; 1141 1142 return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2; 1143} 1144 1145s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx, 1146 enum efx_filter_priority priority, 1147 u32 *buf, u32 size) 1148{ 1149 struct efx_mcdi_filter_table *table; 1150 struct efx_filter_spec *spec; 1151 unsigned int filter_idx; 1152 s32 count = 0; 1153 1154 down_read(&efx->filter_sem); 1155 table = efx->filter_state; 1156 down_read(&table->lock); 1157 1158 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1159 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1160 if (spec && spec->priority == priority) { 1161 if (count == size) { 1162 count = -EMSGSIZE; 1163 break; 1164 } 1165 buf[count++] = 1166 efx_mcdi_filter_make_filter_id( 1167 efx_mcdi_filter_pri(table, spec), 1168 filter_idx); 1169 } 1170 } 1171 up_read(&table->lock); 1172 up_read(&efx->filter_sem); 1173 return count; 1174} 1175 1176static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags) 1177{ 1178 int match_flags = 0; 1179 1180#define MAP_FLAG(gen_flag, mcdi_field) do { \ 1181 u32 old_mcdi_flags = mcdi_flags; \ 1182 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ## \ 1183 mcdi_field ## _LBN); \ 1184 if (mcdi_flags != old_mcdi_flags) \ 1185 match_flags |= EFX_FILTER_MATCH_ ## gen_flag; \ 1186 } while (0) 1187 1188 if (encap) { 1189 /* encap filters must specify encap type */ 1190 match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE; 1191 /* and imply ethertype and ip proto */ 1192 mcdi_flags &= 1193 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); 1194 mcdi_flags &= 1195 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); 1196 /* VLAN tags refer to the outer packet */ 1197 MAP_FLAG(INNER_VID, INNER_VLAN); 1198 MAP_FLAG(OUTER_VID, OUTER_VLAN); 1199 /* everything else refers to the inner packet */ 1200 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST); 1201 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST); 1202 MAP_FLAG(REM_HOST, IFRM_SRC_IP); 1203 MAP_FLAG(LOC_HOST, IFRM_DST_IP); 1204 MAP_FLAG(REM_MAC, IFRM_SRC_MAC); 1205 MAP_FLAG(REM_PORT, IFRM_SRC_PORT); 1206 MAP_FLAG(LOC_MAC, IFRM_DST_MAC); 1207 MAP_FLAG(LOC_PORT, IFRM_DST_PORT); 1208 MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE); 1209 MAP_FLAG(IP_PROTO, IFRM_IP_PROTO); 1210 } else { 1211 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST); 1212 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST); 1213 MAP_FLAG(REM_HOST, SRC_IP); 1214 MAP_FLAG(LOC_HOST, DST_IP); 1215 MAP_FLAG(REM_MAC, SRC_MAC); 1216 MAP_FLAG(REM_PORT, SRC_PORT); 1217 MAP_FLAG(LOC_MAC, DST_MAC); 1218 MAP_FLAG(LOC_PORT, DST_PORT); 1219 MAP_FLAG(ETHER_TYPE, ETHER_TYPE); 1220 MAP_FLAG(INNER_VID, INNER_VLAN); 1221 MAP_FLAG(OUTER_VID, OUTER_VLAN); 1222 MAP_FLAG(IP_PROTO, IP_PROTO); 1223 } 1224#undef MAP_FLAG 1225 1226 /* Did we map them all? */ 1227 if (mcdi_flags) 1228 return -EINVAL; 1229 1230 return match_flags; 1231} 1232 1233bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table, 1234 bool encap, 1235 enum efx_filter_match_flags match_flags) 1236{ 1237 unsigned int match_pri; 1238 int mf; 1239 1240 for (match_pri = 0; 1241 match_pri < table->rx_match_count; 1242 match_pri++) { 1243 mf = efx_mcdi_filter_match_flags_from_mcdi(encap, 1244 table->rx_match_mcdi_flags[match_pri]); 1245 if (mf == match_flags) 1246 return true; 1247 } 1248 1249 return false; 1250} 1251 1252static int 1253efx_mcdi_filter_table_probe_matches(struct efx_nic *efx, 1254 struct efx_mcdi_filter_table *table, 1255 bool encap) 1256{ 1257 MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN); 1258 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX); 1259 unsigned int pd_match_pri, pd_match_count; 1260 size_t outlen; 1261 int rc; 1262 1263 /* Find out which RX filter types are supported, and their priorities */ 1264 MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP, 1265 encap ? 1266 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES : 1267 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES); 1268 rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO, 1269 inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), 1270 &outlen); 1271 if (rc) 1272 return rc; 1273 1274 pd_match_count = MCDI_VAR_ARRAY_LEN( 1275 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES); 1276 1277 for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) { 1278 u32 mcdi_flags = 1279 MCDI_ARRAY_DWORD( 1280 outbuf, 1281 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES, 1282 pd_match_pri); 1283 rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags); 1284 if (rc < 0) { 1285 netif_dbg(efx, probe, efx->net_dev, 1286 "%s: fw flags %#x pri %u not supported in driver\n", 1287 __func__, mcdi_flags, pd_match_pri); 1288 } else { 1289 netif_dbg(efx, probe, efx->net_dev, 1290 "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n", 1291 __func__, mcdi_flags, pd_match_pri, 1292 rc, table->rx_match_count); 1293 table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags; 1294 table->rx_match_count++; 1295 } 1296 } 1297 1298 return 0; 1299} 1300 1301int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining) 1302{ 1303 struct net_device *net_dev = efx->net_dev; 1304 struct efx_mcdi_filter_table *table; 1305 int rc; 1306 1307 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1308 return -EINVAL; 1309 1310 if (efx->filter_state) /* already probed */ 1311 return 0; 1312 1313 table = kzalloc(sizeof(*table), GFP_KERNEL); 1314 if (!table) 1315 return -ENOMEM; 1316 1317 table->mc_chaining = multicast_chaining; 1318 table->rx_match_count = 0; 1319 rc = efx_mcdi_filter_table_probe_matches(efx, table, false); 1320 if (rc) 1321 goto fail; 1322 if (efx_has_cap(efx, VXLAN_NVGRE)) 1323 rc = efx_mcdi_filter_table_probe_matches(efx, table, true); 1324 if (rc) 1325 goto fail; 1326 if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) && 1327 !(efx_mcdi_filter_match_supported(table, false, 1328 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) && 1329 efx_mcdi_filter_match_supported(table, false, 1330 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) { 1331 netif_info(efx, probe, net_dev, 1332 "VLAN filters are not supported in this firmware variant\n"); 1333 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1334 efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1335 net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1336 } 1337 1338 table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS, 1339 sizeof(*table->entry))); 1340 if (!table->entry) { 1341 rc = -ENOMEM; 1342 goto fail; 1343 } 1344 1345 table->mc_promisc_last = false; 1346 table->vlan_filter = 1347 !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); 1348 INIT_LIST_HEAD(&table->vlan_list); 1349 init_rwsem(&table->lock); 1350 1351 efx->filter_state = table; 1352 1353 return 0; 1354fail: 1355 kfree(table); 1356 return rc; 1357} 1358 1359void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx) 1360{ 1361 struct efx_mcdi_filter_table *table = efx->filter_state; 1362 1363 if (table) { 1364 table->must_restore_filters = true; 1365 table->must_restore_rss_contexts = true; 1366 } 1367} 1368 1369/* 1370 * Caller must hold efx->filter_sem for read if race against 1371 * efx_mcdi_filter_table_remove() is possible 1372 */ 1373void efx_mcdi_filter_table_restore(struct efx_nic *efx) 1374{ 1375 struct efx_mcdi_filter_table *table = efx->filter_state; 1376 unsigned int invalid_filters = 0, failed = 0; 1377 struct efx_mcdi_filter_vlan *vlan; 1378 struct efx_filter_spec *spec; 1379 struct efx_rss_context *ctx; 1380 unsigned int filter_idx; 1381 u32 mcdi_flags; 1382 int match_pri; 1383 int rc, i; 1384 1385 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 1386 1387 if (!table || !table->must_restore_filters) 1388 return; 1389 1390 down_write(&table->lock); 1391 mutex_lock(&efx->rss_lock); 1392 1393 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1394 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1395 if (!spec) 1396 continue; 1397 1398 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec); 1399 match_pri = 0; 1400 while (match_pri < table->rx_match_count && 1401 table->rx_match_mcdi_flags[match_pri] != mcdi_flags) 1402 ++match_pri; 1403 if (match_pri >= table->rx_match_count) { 1404 invalid_filters++; 1405 goto not_restored; 1406 } 1407 if (spec->rss_context) 1408 ctx = efx_find_rss_context_entry(efx, spec->rss_context); 1409 else 1410 ctx = &efx->rss_context; 1411 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { 1412 if (!ctx) { 1413 netif_warn(efx, drv, efx->net_dev, 1414 "Warning: unable to restore a filter with nonexistent RSS context %u.\n", 1415 spec->rss_context); 1416 invalid_filters++; 1417 goto not_restored; 1418 } 1419 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { 1420 netif_warn(efx, drv, efx->net_dev, 1421 "Warning: unable to restore a filter with RSS context %u as it was not created.\n", 1422 spec->rss_context); 1423 invalid_filters++; 1424 goto not_restored; 1425 } 1426 } 1427 1428 rc = efx_mcdi_filter_push(efx, spec, 1429 &table->entry[filter_idx].handle, 1430 ctx, false); 1431 if (rc) 1432 failed++; 1433 1434 if (rc) { 1435not_restored: 1436 list_for_each_entry(vlan, &table->vlan_list, list) 1437 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i) 1438 if (vlan->default_filters[i] == filter_idx) 1439 vlan->default_filters[i] = 1440 EFX_EF10_FILTER_ID_INVALID; 1441 1442 kfree(spec); 1443 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0); 1444 } 1445 } 1446 1447 mutex_unlock(&efx->rss_lock); 1448 up_write(&table->lock); 1449 1450 /* 1451 * This can happen validly if the MC's capabilities have changed, so 1452 * is not an error. 1453 */ 1454 if (invalid_filters) 1455 netif_dbg(efx, drv, efx->net_dev, 1456 "Did not restore %u filters that are now unsupported.\n", 1457 invalid_filters); 1458 1459 if (failed) 1460 netif_err(efx, hw, efx->net_dev, 1461 "unable to restore %u filters\n", failed); 1462 else 1463 table->must_restore_filters = false; 1464} 1465 1466void efx_mcdi_filter_table_down(struct efx_nic *efx) 1467{ 1468 struct efx_mcdi_filter_table *table = efx->filter_state; 1469 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 1470 struct efx_filter_spec *spec; 1471 unsigned int filter_idx; 1472 int rc; 1473 1474 if (!table) 1475 return; 1476 1477 efx_mcdi_filter_cleanup_vlans(efx); 1478 1479 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1480 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1481 if (!spec) 1482 continue; 1483 1484 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 1485 efx_mcdi_filter_is_exclusive(spec) ? 1486 MC_CMD_FILTER_OP_IN_OP_REMOVE : 1487 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 1488 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 1489 table->entry[filter_idx].handle); 1490 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, 1491 sizeof(inbuf), NULL, 0, NULL); 1492 if (rc) 1493 netif_info(efx, drv, efx->net_dev, 1494 "%s: filter %04x remove failed\n", 1495 __func__, filter_idx); 1496 kfree(spec); 1497 } 1498} 1499 1500void efx_mcdi_filter_table_remove(struct efx_nic *efx) 1501{ 1502 struct efx_mcdi_filter_table *table = efx->filter_state; 1503 1504 efx_mcdi_filter_table_down(efx); 1505 1506 efx->filter_state = NULL; 1507 /* 1508 * If we were called without locking, then it's not safe to free 1509 * the table as others might be using it. So we just WARN, leak 1510 * the memory, and potentially get an inconsistent filter table 1511 * state. 1512 * This should never actually happen. 1513 */ 1514 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1515 return; 1516 1517 if (!table) 1518 return; 1519 1520 vfree(table->entry); 1521 kfree(table); 1522} 1523 1524static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id) 1525{ 1526 struct efx_mcdi_filter_table *table = efx->filter_state; 1527 unsigned int filter_idx; 1528 1529 efx_rwsem_assert_write_locked(&table->lock); 1530 1531 if (*id != EFX_EF10_FILTER_ID_INVALID) { 1532 filter_idx = efx_mcdi_filter_get_unsafe_id(*id); 1533 if (!table->entry[filter_idx].spec) 1534 netif_dbg(efx, drv, efx->net_dev, 1535 "marked null spec old %04x:%04x\n", *id, 1536 filter_idx); 1537 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD; 1538 *id = EFX_EF10_FILTER_ID_INVALID; 1539 } 1540} 1541 1542/* Mark old per-VLAN filters that may need to be removed */ 1543static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx, 1544 struct efx_mcdi_filter_vlan *vlan) 1545{ 1546 struct efx_mcdi_filter_table *table = efx->filter_state; 1547 unsigned int i; 1548 1549 for (i = 0; i < table->dev_uc_count; i++) 1550 efx_mcdi_filter_mark_one_old(efx, &vlan->uc[i]); 1551 for (i = 0; i < table->dev_mc_count; i++) 1552 efx_mcdi_filter_mark_one_old(efx, &vlan->mc[i]); 1553 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 1554 efx_mcdi_filter_mark_one_old(efx, &vlan->default_filters[i]); 1555} 1556 1557/* 1558 * Mark old filters that may need to be removed. 1559 * Caller must hold efx->filter_sem for read if race against 1560 * efx_mcdi_filter_table_remove() is possible 1561 */ 1562static void efx_mcdi_filter_mark_old(struct efx_nic *efx) 1563{ 1564 struct efx_mcdi_filter_table *table = efx->filter_state; 1565 struct efx_mcdi_filter_vlan *vlan; 1566 1567 down_write(&table->lock); 1568 list_for_each_entry(vlan, &table->vlan_list, list) 1569 _efx_mcdi_filter_vlan_mark_old(efx, vlan); 1570 up_write(&table->lock); 1571} 1572 1573int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid) 1574{ 1575 struct efx_mcdi_filter_table *table = efx->filter_state; 1576 struct efx_mcdi_filter_vlan *vlan; 1577 unsigned int i; 1578 1579 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1580 return -EINVAL; 1581 1582 vlan = efx_mcdi_filter_find_vlan(efx, vid); 1583 if (WARN_ON(vlan)) { 1584 netif_err(efx, drv, efx->net_dev, 1585 "VLAN %u already added\n", vid); 1586 return -EALREADY; 1587 } 1588 1589 vlan = kzalloc(sizeof(*vlan), GFP_KERNEL); 1590 if (!vlan) 1591 return -ENOMEM; 1592 1593 vlan->vid = vid; 1594 1595 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) 1596 vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID; 1597 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) 1598 vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID; 1599 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 1600 vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID; 1601 1602 list_add_tail(&vlan->list, &table->vlan_list); 1603 1604 if (efx_dev_registered(efx)) 1605 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan); 1606 1607 return 0; 1608} 1609 1610static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx, 1611 struct efx_mcdi_filter_vlan *vlan) 1612{ 1613 unsigned int i; 1614 1615 /* See comment in efx_mcdi_filter_table_remove() */ 1616 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1617 return; 1618 1619 list_del(&vlan->list); 1620 1621 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) 1622 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 1623 vlan->uc[i]); 1624 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) 1625 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 1626 vlan->mc[i]); 1627 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 1628 if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID) 1629 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 1630 vlan->default_filters[i]); 1631 1632 kfree(vlan); 1633} 1634 1635void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid) 1636{ 1637 struct efx_mcdi_filter_vlan *vlan; 1638 1639 /* See comment in efx_mcdi_filter_table_remove() */ 1640 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1641 return; 1642 1643 vlan = efx_mcdi_filter_find_vlan(efx, vid); 1644 if (!vlan) { 1645 netif_err(efx, drv, efx->net_dev, 1646 "VLAN %u not found in filter state\n", vid); 1647 return; 1648 } 1649 1650 efx_mcdi_filter_del_vlan_internal(efx, vlan); 1651} 1652 1653struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx, 1654 u16 vid) 1655{ 1656 struct efx_mcdi_filter_table *table = efx->filter_state; 1657 struct efx_mcdi_filter_vlan *vlan; 1658 1659 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 1660 1661 list_for_each_entry(vlan, &table->vlan_list, list) { 1662 if (vlan->vid == vid) 1663 return vlan; 1664 } 1665 1666 return NULL; 1667} 1668 1669void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx) 1670{ 1671 struct efx_mcdi_filter_table *table = efx->filter_state; 1672 struct efx_mcdi_filter_vlan *vlan, *next_vlan; 1673 1674 /* See comment in efx_mcdi_filter_table_remove() */ 1675 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1676 return; 1677 1678 if (!table) 1679 return; 1680 1681 list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list) 1682 efx_mcdi_filter_del_vlan_internal(efx, vlan); 1683} 1684 1685static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx) 1686{ 1687 struct efx_mcdi_filter_table *table = efx->filter_state; 1688 struct net_device *net_dev = efx->net_dev; 1689 struct netdev_hw_addr *uc; 1690 unsigned int i; 1691 1692 table->uc_promisc = !!(net_dev->flags & IFF_PROMISC); 1693 ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr); 1694 i = 1; 1695 netdev_for_each_uc_addr(uc, net_dev) { 1696 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) { 1697 table->uc_promisc = true; 1698 break; 1699 } 1700 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr); 1701 i++; 1702 } 1703 1704 table->dev_uc_count = i; 1705} 1706 1707static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx) 1708{ 1709 struct efx_mcdi_filter_table *table = efx->filter_state; 1710 struct net_device *net_dev = efx->net_dev; 1711 struct netdev_hw_addr *mc; 1712 unsigned int i; 1713 1714 table->mc_overflow = false; 1715 table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI)); 1716 1717 i = 0; 1718 netdev_for_each_mc_addr(mc, net_dev) { 1719 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) { 1720 table->mc_promisc = true; 1721 table->mc_overflow = true; 1722 break; 1723 } 1724 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr); 1725 i++; 1726 } 1727 1728 table->dev_mc_count = i; 1729} 1730 1731/* 1732 * Caller must hold efx->filter_sem for read if race against 1733 * efx_mcdi_filter_table_remove() is possible 1734 */ 1735void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx) 1736{ 1737 struct efx_mcdi_filter_table *table = efx->filter_state; 1738 struct net_device *net_dev = efx->net_dev; 1739 struct efx_mcdi_filter_vlan *vlan; 1740 bool vlan_filter; 1741 1742 if (!efx_dev_registered(efx)) 1743 return; 1744 1745 if (!table) 1746 return; 1747 1748 efx_mcdi_filter_mark_old(efx); 1749 1750 /* 1751 * Copy/convert the address lists; add the primary station 1752 * address and broadcast address 1753 */ 1754 netif_addr_lock_bh(net_dev); 1755 efx_mcdi_filter_uc_addr_list(efx); 1756 efx_mcdi_filter_mc_addr_list(efx); 1757 netif_addr_unlock_bh(net_dev); 1758 1759 /* 1760 * If VLAN filtering changes, all old filters are finally removed. 1761 * Do it in advance to avoid conflicts for unicast untagged and 1762 * VLAN 0 tagged filters. 1763 */ 1764 vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); 1765 if (table->vlan_filter != vlan_filter) { 1766 table->vlan_filter = vlan_filter; 1767 efx_mcdi_filter_remove_old(efx); 1768 } 1769 1770 list_for_each_entry(vlan, &table->vlan_list, list) 1771 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan); 1772 1773 efx_mcdi_filter_remove_old(efx); 1774 table->mc_promisc_last = table->mc_promisc; 1775} 1776 1777#ifdef CONFIG_RFS_ACCEL 1778 1779bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, 1780 unsigned int filter_idx) 1781{ 1782 struct efx_filter_spec *spec, saved_spec; 1783 struct efx_mcdi_filter_table *table; 1784 struct efx_arfs_rule *rule = NULL; 1785 bool ret = true, force = false; 1786 u16 arfs_id; 1787 1788 down_read(&efx->filter_sem); 1789 table = efx->filter_state; 1790 down_write(&table->lock); 1791 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1792 1793 if (!spec || spec->priority != EFX_FILTER_PRI_HINT) 1794 goto out_unlock; 1795 1796 spin_lock_bh(&efx->rps_hash_lock); 1797 if (!efx->rps_hash_table) { 1798 /* In the absence of the table, we always return 0 to ARFS. */ 1799 arfs_id = 0; 1800 } else { 1801 rule = efx_rps_hash_find(efx, spec); 1802 if (!rule) 1803 /* ARFS table doesn't know of this filter, so remove it */ 1804 goto expire; 1805 arfs_id = rule->arfs_id; 1806 ret = efx_rps_check_rule(rule, filter_idx, &force); 1807 if (force) 1808 goto expire; 1809 if (!ret) { 1810 spin_unlock_bh(&efx->rps_hash_lock); 1811 goto out_unlock; 1812 } 1813 } 1814 if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id)) 1815 ret = false; 1816 else if (rule) 1817 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; 1818expire: 1819 saved_spec = *spec; /* remove operation will kfree spec */ 1820 spin_unlock_bh(&efx->rps_hash_lock); 1821 /* 1822 * At this point (since we dropped the lock), another thread might queue 1823 * up a fresh insertion request (but the actual insertion will be held 1824 * up by our possession of the filter table lock). In that case, it 1825 * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that 1826 * the rule is not removed by efx_rps_hash_del() below. 1827 */ 1828 if (ret) 1829 ret = efx_mcdi_filter_remove_internal(efx, 1U << spec->priority, 1830 filter_idx, true) == 0; 1831 /* 1832 * While we can't safely dereference rule (we dropped the lock), we can 1833 * still test it for NULL. 1834 */ 1835 if (ret && rule) { 1836 /* Expiring, so remove entry from ARFS table */ 1837 spin_lock_bh(&efx->rps_hash_lock); 1838 efx_rps_hash_del(efx, &saved_spec); 1839 spin_unlock_bh(&efx->rps_hash_lock); 1840 } 1841out_unlock: 1842 up_write(&table->lock); 1843 up_read(&efx->filter_sem); 1844 return ret; 1845} 1846 1847#endif /* CONFIG_RFS_ACCEL */ 1848 1849#define RSS_MODE_HASH_ADDRS (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\ 1850 1 << RSS_MODE_HASH_DST_ADDR_LBN) 1851#define RSS_MODE_HASH_PORTS (1 << RSS_MODE_HASH_SRC_PORT_LBN |\ 1852 1 << RSS_MODE_HASH_DST_PORT_LBN) 1853#define RSS_CONTEXT_FLAGS_DEFAULT (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\ 1854 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\ 1855 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\ 1856 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\ 1857 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\ 1858 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\ 1859 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\ 1860 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\ 1861 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\ 1862 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN) 1863 1864int efx_mcdi_get_rss_context_flags(struct efx_nic *efx, u32 context, u32 *flags) 1865{ 1866 /* 1867 * Firmware had a bug (sfc bug 61952) where it would not actually 1868 * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS. 1869 * This meant that it would always contain whatever was previously 1870 * in the MCDI buffer. Fortunately, all firmware versions with 1871 * this bug have the same default flags value for a newly-allocated 1872 * RSS context, and the only time we want to get the flags is just 1873 * after allocating. Moreover, the response has a 32-bit hole 1874 * where the context ID would be in the request, so we can use an 1875 * overlength buffer in the request and pre-fill the flags field 1876 * with what we believe the default to be. Thus if the firmware 1877 * has the bug, it will leave our pre-filled value in the flags 1878 * field of the response, and we will get the right answer. 1879 * 1880 * However, this does mean that this function should NOT be used if 1881 * the RSS context flags might not be their defaults - it is ONLY 1882 * reliably correct for a newly-allocated RSS context. 1883 */ 1884 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); 1885 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); 1886 size_t outlen; 1887 int rc; 1888 1889 /* Check we have a hole for the context ID */ 1890 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST); 1891 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context); 1892 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS, 1893 RSS_CONTEXT_FLAGS_DEFAULT); 1894 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf, 1895 sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); 1896 if (rc == 0) { 1897 if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN) 1898 rc = -EIO; 1899 else 1900 *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS); 1901 } 1902 return rc; 1903} 1904 1905/* 1906 * Attempt to enable 4-tuple UDP hashing on the specified RSS context. 1907 * If we fail, we just leave the RSS context at its default hash settings, 1908 * which is safe but may slightly reduce performance. 1909 * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we 1910 * just need to set the UDP ports flags (for both IP versions). 1911 */ 1912void efx_mcdi_set_rss_context_flags(struct efx_nic *efx, 1913 struct efx_rss_context *ctx) 1914{ 1915 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN); 1916 u32 flags; 1917 1918 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0); 1919 1920 if (efx_mcdi_get_rss_context_flags(efx, ctx->context_id, &flags) != 0) 1921 return; 1922 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 1923 ctx->context_id); 1924 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN; 1925 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN; 1926 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags); 1927 if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf), 1928 NULL, 0, NULL)) 1929 /* Succeeded, so UDP 4-tuple is now enabled */ 1930 ctx->rx_hash_udp_4tuple = true; 1931} 1932 1933static int efx_mcdi_filter_alloc_rss_context(struct efx_nic *efx, bool exclusive, 1934 struct efx_rss_context *ctx, 1935 unsigned *context_size) 1936{ 1937 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN); 1938 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 1939 size_t outlen; 1940 int rc; 1941 u32 alloc_type = exclusive ? 1942 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE : 1943 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 1944 unsigned rss_spread = exclusive ? 1945 efx->rss_spread : 1946 min(rounddown_pow_of_two(efx->rss_spread), 1947 EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE); 1948 1949 if (!exclusive && rss_spread == 1) { 1950 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID; 1951 if (context_size) 1952 *context_size = 1; 1953 return 0; 1954 } 1955 1956 if (efx_has_cap(efx, RX_RSS_LIMITED)) 1957 return -EOPNOTSUPP; 1958 1959 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 1960 efx->vport_id); 1961 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type); 1962 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread); 1963 1964 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf), 1965 outbuf, sizeof(outbuf), &outlen); 1966 if (rc != 0) 1967 return rc; 1968 1969 if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) 1970 return -EIO; 1971 1972 ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 1973 1974 if (context_size) 1975 *context_size = rss_spread; 1976 1977 if (efx_has_cap(efx, ADDITIONAL_RSS_MODES)) 1978 efx_mcdi_set_rss_context_flags(efx, ctx); 1979 1980 return 0; 1981} 1982 1983static int efx_mcdi_filter_free_rss_context(struct efx_nic *efx, u32 context) 1984{ 1985 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN); 1986 1987 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, 1988 context); 1989 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf), 1990 NULL, 0, NULL); 1991} 1992 1993static int efx_mcdi_filter_populate_rss_table(struct efx_nic *efx, u32 context, 1994 const u32 *rx_indir_table, const u8 *key) 1995{ 1996 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN); 1997 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN); 1998 int i, rc; 1999 2000 MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 2001 context); 2002 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) != 2003 MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN); 2004 2005 /* This iterates over the length of efx->rss_context.rx_indir_table, but 2006 * copies bytes from rx_indir_table. That's because the latter is a 2007 * pointer rather than an array, but should have the same length. 2008 * The efx->rss_context.rx_hash_key loop below is similar. 2009 */ 2010 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i) 2011 MCDI_PTR(tablebuf, 2012 RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] = 2013 (u8) rx_indir_table[i]; 2014 2015 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf, 2016 sizeof(tablebuf), NULL, 0, NULL); 2017 if (rc != 0) 2018 return rc; 2019 2020 MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 2021 context); 2022 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) != 2023 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 2024 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i) 2025 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i]; 2026 2027 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf, 2028 sizeof(keybuf), NULL, 0, NULL); 2029} 2030 2031void efx_mcdi_rx_free_indir_table(struct efx_nic *efx) 2032{ 2033 int rc; 2034 2035 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) { 2036 rc = efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id); 2037 WARN_ON(rc != 0); 2038 } 2039 efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID; 2040} 2041 2042static int efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic *efx, 2043 unsigned *context_size) 2044{ 2045 struct efx_mcdi_filter_table *table = efx->filter_state; 2046 int rc = efx_mcdi_filter_alloc_rss_context(efx, false, &efx->rss_context, 2047 context_size); 2048 2049 if (rc != 0) 2050 return rc; 2051 2052 table->rx_rss_context_exclusive = false; 2053 efx_set_default_rx_indir_table(efx, &efx->rss_context); 2054 return 0; 2055} 2056 2057static int efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic *efx, 2058 const u32 *rx_indir_table, 2059 const u8 *key) 2060{ 2061 struct efx_mcdi_filter_table *table = efx->filter_state; 2062 u32 old_rx_rss_context = efx->rss_context.context_id; 2063 int rc; 2064 2065 if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID || 2066 !table->rx_rss_context_exclusive) { 2067 rc = efx_mcdi_filter_alloc_rss_context(efx, true, &efx->rss_context, 2068 NULL); 2069 if (rc == -EOPNOTSUPP) 2070 return rc; 2071 else if (rc != 0) 2072 goto fail1; 2073 } 2074 2075 rc = efx_mcdi_filter_populate_rss_table(efx, efx->rss_context.context_id, 2076 rx_indir_table, key); 2077 if (rc != 0) 2078 goto fail2; 2079 2080 if (efx->rss_context.context_id != old_rx_rss_context && 2081 old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID) 2082 WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0); 2083 table->rx_rss_context_exclusive = true; 2084 if (rx_indir_table != efx->rss_context.rx_indir_table) 2085 memcpy(efx->rss_context.rx_indir_table, rx_indir_table, 2086 sizeof(efx->rss_context.rx_indir_table)); 2087 if (key != efx->rss_context.rx_hash_key) 2088 memcpy(efx->rss_context.rx_hash_key, key, 2089 efx->type->rx_hash_key_size); 2090 2091 return 0; 2092 2093fail2: 2094 if (old_rx_rss_context != efx->rss_context.context_id) { 2095 WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0); 2096 efx->rss_context.context_id = old_rx_rss_context; 2097 } 2098fail1: 2099 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc); 2100 return rc; 2101} 2102 2103int efx_mcdi_rx_push_rss_context_config(struct efx_nic *efx, 2104 struct efx_rss_context *ctx, 2105 const u32 *rx_indir_table, 2106 const u8 *key) 2107{ 2108 int rc; 2109 2110 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2111 2112 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { 2113 rc = efx_mcdi_filter_alloc_rss_context(efx, true, ctx, NULL); 2114 if (rc) 2115 return rc; 2116 } 2117 2118 if (!rx_indir_table) /* Delete this context */ 2119 return efx_mcdi_filter_free_rss_context(efx, ctx->context_id); 2120 2121 rc = efx_mcdi_filter_populate_rss_table(efx, ctx->context_id, 2122 rx_indir_table, key); 2123 if (rc) 2124 return rc; 2125 2126 memcpy(ctx->rx_indir_table, rx_indir_table, 2127 sizeof(efx->rss_context.rx_indir_table)); 2128 memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size); 2129 2130 return 0; 2131} 2132 2133int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx, 2134 struct efx_rss_context *ctx) 2135{ 2136 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN); 2137 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN); 2138 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN); 2139 size_t outlen; 2140 int rc, i; 2141 2142 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2143 2144 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN != 2145 MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN); 2146 2147 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) 2148 return -ENOENT; 2149 2150 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID, 2151 ctx->context_id); 2152 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) != 2153 MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN); 2154 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf), 2155 tablebuf, sizeof(tablebuf), &outlen); 2156 if (rc != 0) 2157 return rc; 2158 2159 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN)) 2160 return -EIO; 2161 2162 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 2163 ctx->rx_indir_table[i] = MCDI_PTR(tablebuf, 2164 RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i]; 2165 2166 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID, 2167 ctx->context_id); 2168 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) != 2169 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 2170 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf), 2171 keybuf, sizeof(keybuf), &outlen); 2172 if (rc != 0) 2173 return rc; 2174 2175 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN)) 2176 return -EIO; 2177 2178 for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i) 2179 ctx->rx_hash_key[i] = MCDI_PTR( 2180 keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i]; 2181 2182 return 0; 2183} 2184 2185int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx) 2186{ 2187 int rc; 2188 2189 mutex_lock(&efx->rss_lock); 2190 rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context); 2191 mutex_unlock(&efx->rss_lock); 2192 return rc; 2193} 2194 2195void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx) 2196{ 2197 struct efx_mcdi_filter_table *table = efx->filter_state; 2198 struct efx_rss_context *ctx; 2199 int rc; 2200 2201 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2202 2203 if (!table->must_restore_rss_contexts) 2204 return; 2205 2206 list_for_each_entry(ctx, &efx->rss_context.list, list) { 2207 /* previous NIC RSS context is gone */ 2208 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID; 2209 /* so try to allocate a new one */ 2210 rc = efx_mcdi_rx_push_rss_context_config(efx, ctx, 2211 ctx->rx_indir_table, 2212 ctx->rx_hash_key); 2213 if (rc) 2214 netif_warn(efx, probe, efx->net_dev, 2215 "failed to restore RSS context %u, rc=%d" 2216 "; RSS filters may fail to be applied\n", 2217 ctx->user_id, rc); 2218 } 2219 table->must_restore_rss_contexts = false; 2220} 2221 2222int efx_mcdi_pf_rx_push_rss_config(struct efx_nic *efx, bool user, 2223 const u32 *rx_indir_table, 2224 const u8 *key) 2225{ 2226 int rc; 2227 2228 if (efx->rss_spread == 1) 2229 return 0; 2230 2231 if (!key) 2232 key = efx->rss_context.rx_hash_key; 2233 2234 rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key); 2235 2236 if (rc == -ENOBUFS && !user) { 2237 unsigned context_size; 2238 bool mismatch = false; 2239 size_t i; 2240 2241 for (i = 0; 2242 i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch; 2243 i++) 2244 mismatch = rx_indir_table[i] != 2245 ethtool_rxfh_indir_default(i, efx->rss_spread); 2246 2247 rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, &context_size); 2248 if (rc == 0) { 2249 if (context_size != efx->rss_spread) 2250 netif_warn(efx, probe, efx->net_dev, 2251 "Could not allocate an exclusive RSS" 2252 " context; allocated a shared one of" 2253 " different size." 2254 " Wanted %u, got %u.\n", 2255 efx->rss_spread, context_size); 2256 else if (mismatch) 2257 netif_warn(efx, probe, efx->net_dev, 2258 "Could not allocate an exclusive RSS" 2259 " context; allocated a shared one but" 2260 " could not apply custom" 2261 " indirection.\n"); 2262 else 2263 netif_info(efx, probe, efx->net_dev, 2264 "Could not allocate an exclusive RSS" 2265 " context; allocated a shared one.\n"); 2266 } 2267 } 2268 return rc; 2269} 2270 2271int efx_mcdi_vf_rx_push_rss_config(struct efx_nic *efx, bool user, 2272 const u32 *rx_indir_table 2273 __attribute__ ((unused)), 2274 const u8 *key 2275 __attribute__ ((unused))) 2276{ 2277 if (user) 2278 return -EOPNOTSUPP; 2279 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) 2280 return 0; 2281 return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL); 2282} 2283 2284int efx_mcdi_push_default_indir_table(struct efx_nic *efx, 2285 unsigned int rss_spread) 2286{ 2287 int rc = 0; 2288 2289 if (efx->rss_spread == rss_spread) 2290 return 0; 2291 2292 efx->rss_spread = rss_spread; 2293 if (!efx->filter_state) 2294 return 0; 2295 2296 efx_mcdi_rx_free_indir_table(efx); 2297 if (rss_spread > 1) { 2298 efx_set_default_rx_indir_table(efx, &efx->rss_context); 2299 rc = efx->type->rx_push_rss_config(efx, false, 2300 efx->rss_context.rx_indir_table, NULL); 2301 } 2302 return rc; 2303} 2304