1/* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <net/tc_act/tc_gact.h> 34#include <linux/mlx5/fs.h> 35#include <net/vxlan.h> 36#include <net/geneve.h> 37#include <linux/bpf.h> 38#include <linux/debugfs.h> 39#include <linux/if_bridge.h> 40#include <linux/filter.h> 41#include <net/page_pool/types.h> 42#include <net/pkt_sched.h> 43#include <net/xdp_sock_drv.h> 44#include "eswitch.h" 45#include "en.h" 46#include "en/txrx.h" 47#include "en_tc.h" 48#include "en_rep.h" 49#include "en_accel/ipsec.h" 50#include "en_accel/macsec.h" 51#include "en_accel/en_accel.h" 52#include "en_accel/ktls.h" 53#include "lib/vxlan.h" 54#include "lib/clock.h" 55#include "en/port.h" 56#include "en/xdp.h" 57#include "lib/eq.h" 58#include "en/monitor_stats.h" 59#include "en/health.h" 60#include "en/params.h" 61#include "en/xsk/pool.h" 62#include "en/xsk/setup.h" 63#include "en/xsk/rx.h" 64#include "en/xsk/tx.h" 65#include "en/hv_vhca_stats.h" 66#include "en/devlink.h" 67#include "lib/mlx5.h" 68#include "en/ptp.h" 69#include "en/htb.h" 70#include "qos.h" 71#include "en/trap.h" 72#include "lib/devcom.h" 73#include "lib/sd.h" 74 75bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, 76 enum mlx5e_mpwrq_umr_mode umr_mode) 77{ 78 u16 umr_wqebbs, max_wqebbs; 79 bool striding_rq_umr; 80 81 striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 82 MLX5_CAP_ETH(mdev, reg_umr_sq); 83 if (!striding_rq_umr) 84 return false; 85 86 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); 87 max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 88 /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is 89 * calculated from mlx5e_get_max_sq_aligned_wqebbs. 90 */ 91 if (WARN_ON(umr_wqebbs > max_wqebbs)) 92 return false; 93 94 return true; 95} 96 97void mlx5e_update_carrier(struct mlx5e_priv *priv) 98{ 99 struct mlx5_core_dev *mdev = priv->mdev; 100 u8 port_state; 101 bool up; 102 103 port_state = mlx5_query_vport_state(mdev, 104 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 105 0); 106 107 up = port_state == VPORT_STATE_UP; 108 if (up == netif_carrier_ok(priv->netdev)) 109 netif_carrier_event(priv->netdev); 110 if (up) { 111 netdev_info(priv->netdev, "Link up\n"); 112 netif_carrier_on(priv->netdev); 113 } else { 114 netdev_info(priv->netdev, "Link down\n"); 115 netif_carrier_off(priv->netdev); 116 } 117} 118 119static void mlx5e_update_carrier_work(struct work_struct *work) 120{ 121 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 122 update_carrier_work); 123 124 mutex_lock(&priv->state_lock); 125 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 126 if (priv->profile->update_carrier) 127 priv->profile->update_carrier(priv); 128 mutex_unlock(&priv->state_lock); 129} 130 131static void mlx5e_update_stats_work(struct work_struct *work) 132{ 133 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 134 update_stats_work); 135 136 mutex_lock(&priv->state_lock); 137 priv->profile->update_stats(priv); 138 mutex_unlock(&priv->state_lock); 139} 140 141void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 142{ 143 if (!priv->profile->update_stats) 144 return; 145 146 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 147 return; 148 149 queue_work(priv->wq, &priv->update_stats_work); 150} 151 152static int async_event(struct notifier_block *nb, unsigned long event, void *data) 153{ 154 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 155 struct mlx5_eqe *eqe = data; 156 157 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 158 return NOTIFY_DONE; 159 160 switch (eqe->sub_type) { 161 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 162 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 163 queue_work(priv->wq, &priv->update_carrier_work); 164 break; 165 default: 166 return NOTIFY_DONE; 167 } 168 169 return NOTIFY_OK; 170} 171 172static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 173{ 174 priv->events_nb.notifier_call = async_event; 175 mlx5_notifier_register(priv->mdev, &priv->events_nb); 176} 177 178static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 179{ 180 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 181} 182 183static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data) 184{ 185 struct mlx5e_priv *slave_priv = my_data; 186 187 switch (event) { 188 case MPV_DEVCOM_MASTER_UP: 189 mlx5_devcom_comp_set_ready(slave_priv->devcom, true); 190 break; 191 case MPV_DEVCOM_MASTER_DOWN: 192 /* no need for comp set ready false since we unregister after 193 * and it hurts cleanup flow. 194 */ 195 break; 196 case MPV_DEVCOM_IPSEC_MASTER_UP: 197 case MPV_DEVCOM_IPSEC_MASTER_DOWN: 198 mlx5e_ipsec_handle_mpv_event(event, my_data, event_data); 199 break; 200 } 201 202 return 0; 203} 204 205static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data) 206{ 207 priv->devcom = mlx5_devcom_register_component(priv->mdev->priv.devc, 208 MLX5_DEVCOM_MPV, 209 *data, 210 mlx5e_devcom_event_mpv, 211 priv); 212 if (IS_ERR(priv->devcom)) 213 return PTR_ERR(priv->devcom); 214 215 if (mlx5_core_is_mp_master(priv->mdev)) { 216 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_UP, 217 MPV_DEVCOM_MASTER_UP, priv); 218 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_UP); 219 } 220 221 return 0; 222} 223 224static void mlx5e_devcom_cleanup_mpv(struct mlx5e_priv *priv) 225{ 226 if (IS_ERR_OR_NULL(priv->devcom)) 227 return; 228 229 if (mlx5_core_is_mp_master(priv->mdev)) { 230 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_DOWN, 231 MPV_DEVCOM_MASTER_DOWN, priv); 232 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_DOWN); 233 } 234 235 mlx5_devcom_unregister_component(priv->devcom); 236} 237 238static int blocking_event(struct notifier_block *nb, unsigned long event, void *data) 239{ 240 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb); 241 struct mlx5_devlink_trap_event_ctx *trap_event_ctx = data; 242 int err; 243 244 switch (event) { 245 case MLX5_DRIVER_EVENT_TYPE_TRAP: 246 err = mlx5e_handle_trap_event(priv, trap_event_ctx->trap); 247 if (err) { 248 trap_event_ctx->err = err; 249 return NOTIFY_BAD; 250 } 251 break; 252 case MLX5_DRIVER_EVENT_AFFILIATION_DONE: 253 if (mlx5e_devcom_init_mpv(priv, data)) 254 return NOTIFY_BAD; 255 break; 256 case MLX5_DRIVER_EVENT_AFFILIATION_REMOVED: 257 mlx5e_devcom_cleanup_mpv(priv); 258 break; 259 default: 260 return NOTIFY_DONE; 261 } 262 return NOTIFY_OK; 263} 264 265static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv) 266{ 267 priv->blocking_events_nb.notifier_call = blocking_event; 268 mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb); 269} 270 271static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv) 272{ 273 mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb); 274} 275 276static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode) 277{ 278 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); 279 u32 sz; 280 281 sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT); 282 283 return sz / MLX5_OCTWORD; 284} 285 286static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 287 struct mlx5e_icosq *sq, 288 struct mlx5e_umr_wqe *wqe) 289{ 290 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 291 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; 292 u16 octowords; 293 u8 ds_cnt; 294 295 ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift, 296 rq->mpwqe.umr_mode), 297 MLX5_SEND_WQE_DS); 298 299 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 300 ds_cnt); 301 cseg->umr_mkey = rq->mpwqe.umr_mkey_be; 302 303 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 304 octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode); 305 ucseg->xlt_octowords = cpu_to_be16(octowords); 306 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 307} 308 309static int mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq *rq, int node) 310{ 311 rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo), 312 GFP_KERNEL, node); 313 if (!rq->mpwqe.shampo) 314 return -ENOMEM; 315 return 0; 316} 317 318static void mlx5e_rq_shampo_hd_free(struct mlx5e_rq *rq) 319{ 320 kvfree(rq->mpwqe.shampo); 321} 322 323static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node) 324{ 325 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 326 327 shampo->bitmap = bitmap_zalloc_node(shampo->hd_per_wq, GFP_KERNEL, 328 node); 329 shampo->info = kvzalloc_node(array_size(shampo->hd_per_wq, 330 sizeof(*shampo->info)), 331 GFP_KERNEL, node); 332 shampo->pages = kvzalloc_node(array_size(shampo->hd_per_wq, 333 sizeof(*shampo->pages)), 334 GFP_KERNEL, node); 335 if (!shampo->bitmap || !shampo->info || !shampo->pages) 336 goto err_nomem; 337 338 return 0; 339 340err_nomem: 341 kvfree(shampo->info); 342 kvfree(shampo->bitmap); 343 kvfree(shampo->pages); 344 345 return -ENOMEM; 346} 347 348static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq) 349{ 350 kvfree(rq->mpwqe.shampo->bitmap); 351 kvfree(rq->mpwqe.shampo->info); 352 kvfree(rq->mpwqe.shampo->pages); 353} 354 355static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) 356{ 357 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 358 size_t alloc_size; 359 360 alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, 361 alloc_units.frag_pages, 362 rq->mpwqe.pages_per_wqe)); 363 364 rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node); 365 if (!rq->mpwqe.info) 366 return -ENOMEM; 367 368 /* For deferred page release (release right before alloc), make sure 369 * that on first round release is not called. 370 */ 371 for (int i = 0; i < wq_sz; i++) { 372 struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, i); 373 374 bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); 375 } 376 377 mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe); 378 379 return 0; 380} 381 382 383static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode) 384{ 385 switch (umr_mode) { 386 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 387 return MLX5_MKC_ACCESS_MODE_MTT; 388 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 389 return MLX5_MKC_ACCESS_MODE_KSM; 390 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 391 return MLX5_MKC_ACCESS_MODE_KLMS; 392 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 393 return MLX5_MKC_ACCESS_MODE_KSM; 394 } 395 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode); 396 return 0; 397} 398 399static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 400 u32 npages, u8 page_shift, u32 *umr_mkey, 401 dma_addr_t filler_addr, 402 enum mlx5e_mpwrq_umr_mode umr_mode, 403 u32 xsk_chunk_size) 404{ 405 struct mlx5_mtt *mtt; 406 struct mlx5_ksm *ksm; 407 struct mlx5_klm *klm; 408 u32 octwords; 409 int inlen; 410 void *mkc; 411 u32 *in; 412 int err; 413 int i; 414 415 if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED || 416 umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) && 417 !MLX5_CAP_GEN(mdev, fixed_buffer_size)) { 418 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n"); 419 return -EINVAL; 420 } 421 422 octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode); 423 424 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 425 MLX5_OCTWORD, octwords); 426 if (inlen < 0) 427 return inlen; 428 429 in = kvzalloc(inlen, GFP_KERNEL); 430 if (!in) 431 return -ENOMEM; 432 433 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 434 435 MLX5_SET(mkc, mkc, free, 1); 436 MLX5_SET(mkc, mkc, umr_en, 1); 437 MLX5_SET(mkc, mkc, lw, 1); 438 MLX5_SET(mkc, mkc, lr, 1); 439 MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode)); 440 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 441 MLX5_SET(mkc, mkc, qpn, 0xffffff); 442 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 443 MLX5_SET64(mkc, mkc, len, npages << page_shift); 444 MLX5_SET(mkc, mkc, translations_octword_size, octwords); 445 if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) 446 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2); 447 else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED) 448 MLX5_SET(mkc, mkc, log_page_size, page_shift); 449 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords); 450 451 /* Initialize the mkey with all MTTs pointing to a default 452 * page (filler_addr). When the channels are activated, UMR 453 * WQEs will redirect the RX WQEs to the actual memory from 454 * the RQ's pool, while the gaps (wqe_overflow) remain mapped 455 * to the default page. 456 */ 457 switch (umr_mode) { 458 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 459 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 460 for (i = 0; i < npages; i++) { 461 klm[i << 1] = (struct mlx5_klm) { 462 .va = cpu_to_be64(filler_addr), 463 .bcount = cpu_to_be32(xsk_chunk_size), 464 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 465 }; 466 klm[(i << 1) + 1] = (struct mlx5_klm) { 467 .va = cpu_to_be64(filler_addr), 468 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size), 469 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 470 }; 471 } 472 break; 473 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 474 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 475 for (i = 0; i < npages; i++) 476 ksm[i] = (struct mlx5_ksm) { 477 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 478 .va = cpu_to_be64(filler_addr), 479 }; 480 break; 481 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 482 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 483 for (i = 0; i < npages; i++) 484 mtt[i] = (struct mlx5_mtt) { 485 .ptag = cpu_to_be64(filler_addr), 486 }; 487 break; 488 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 489 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 490 for (i = 0; i < npages * 4; i++) { 491 ksm[i] = (struct mlx5_ksm) { 492 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 493 .va = cpu_to_be64(filler_addr), 494 }; 495 } 496 break; 497 } 498 499 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 500 501 kvfree(in); 502 return err; 503} 504 505static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, 506 u64 nentries, 507 u32 *umr_mkey) 508{ 509 int inlen; 510 void *mkc; 511 u32 *in; 512 int err; 513 514 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 515 516 in = kvzalloc(inlen, GFP_KERNEL); 517 if (!in) 518 return -ENOMEM; 519 520 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 521 522 MLX5_SET(mkc, mkc, free, 1); 523 MLX5_SET(mkc, mkc, umr_en, 1); 524 MLX5_SET(mkc, mkc, lw, 1); 525 MLX5_SET(mkc, mkc, lr, 1); 526 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 527 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 528 MLX5_SET(mkc, mkc, qpn, 0xffffff); 529 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 530 MLX5_SET(mkc, mkc, translations_octword_size, nentries); 531 MLX5_SET(mkc, mkc, length64, 1); 532 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 533 534 kvfree(in); 535 return err; 536} 537 538static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 539{ 540 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; 541 u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 542 u32 num_entries, max_num_entries; 543 u32 umr_mkey; 544 int err; 545 546 max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode); 547 548 /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */ 549 if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe, 550 &num_entries) || 551 num_entries > max_num_entries)) 552 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n", 553 __func__, wq_size, rq->mpwqe.mtts_per_wqe, 554 max_num_entries); 555 556 err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift, 557 &umr_mkey, rq->wqe_overflow.addr, 558 rq->mpwqe.umr_mode, xsk_chunk_size); 559 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); 560 return err; 561} 562 563static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, 564 struct mlx5e_rq *rq) 565{ 566 u32 max_klm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size)); 567 568 if (max_klm_size < rq->mpwqe.shampo->hd_per_wq) { 569 mlx5_core_err(mdev, "max klm list size 0x%x is smaller than shampo header buffer list size 0x%x\n", 570 max_klm_size, rq->mpwqe.shampo->hd_per_wq); 571 return -EINVAL; 572 } 573 return mlx5e_create_umr_klm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq, 574 &rq->mpwqe.shampo->mkey); 575} 576 577static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 578{ 579 struct mlx5e_wqe_frag_info next_frag = {}; 580 struct mlx5e_wqe_frag_info *prev = NULL; 581 int i; 582 583 WARN_ON(rq->xsk_pool); 584 585 next_frag.frag_page = &rq->wqe.alloc_units->frag_pages[0]; 586 587 /* Skip first release due to deferred release. */ 588 next_frag.flags = BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 589 590 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 591 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 592 struct mlx5e_wqe_frag_info *frag = 593 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 594 int f; 595 596 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 597 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 598 /* Pages are assigned at runtime. */ 599 next_frag.frag_page++; 600 next_frag.offset = 0; 601 if (prev) 602 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 603 } 604 *frag = next_frag; 605 606 /* prepare next */ 607 next_frag.offset += frag_info[f].frag_stride; 608 prev = frag; 609 } 610 } 611 612 if (prev) 613 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 614} 615 616static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq) 617{ 618 int i; 619 620 /* Assumptions used by XSK batched allocator. */ 621 WARN_ON(rq->wqe.info.num_frags != 1); 622 WARN_ON(rq->wqe.info.log_num_frags != 0); 623 WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE); 624 625 /* Considering the above assumptions a fragment maps to a single 626 * xsk_buff. 627 */ 628 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 629 rq->wqe.frags[i].xskp = &rq->wqe.alloc_units->xsk_buffs[i]; 630 631 /* Skip first release due to deferred release as WQES are 632 * not allocated yet. 633 */ 634 rq->wqe.frags[i].flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 635 } 636} 637 638static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node) 639{ 640 int wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 641 int len = wq_sz << rq->wqe.info.log_num_frags; 642 struct mlx5e_wqe_frag_info *frags; 643 union mlx5e_alloc_units *aus; 644 int aus_sz; 645 646 if (rq->xsk_pool) 647 aus_sz = sizeof(*aus->xsk_buffs); 648 else 649 aus_sz = sizeof(*aus->frag_pages); 650 651 aus = kvzalloc_node(array_size(len, aus_sz), GFP_KERNEL, node); 652 if (!aus) 653 return -ENOMEM; 654 655 frags = kvzalloc_node(array_size(len, sizeof(*frags)), GFP_KERNEL, node); 656 if (!frags) { 657 kvfree(aus); 658 return -ENOMEM; 659 } 660 661 rq->wqe.alloc_units = aus; 662 rq->wqe.frags = frags; 663 664 if (rq->xsk_pool) 665 mlx5e_init_xsk_buffs(rq); 666 else 667 mlx5e_init_frags_partition(rq); 668 669 return 0; 670} 671 672static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq) 673{ 674 kvfree(rq->wqe.frags); 675 kvfree(rq->wqe.alloc_units); 676} 677 678static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) 679{ 680 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work); 681 682 mlx5e_reporter_rq_cqe_err(rq); 683} 684 685static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 686{ 687 rq->wqe_overflow.page = alloc_page(GFP_KERNEL); 688 if (!rq->wqe_overflow.page) 689 return -ENOMEM; 690 691 rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0, 692 PAGE_SIZE, rq->buff.map_dir); 693 if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) { 694 __free_page(rq->wqe_overflow.page); 695 return -ENOMEM; 696 } 697 return 0; 698} 699 700static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 701{ 702 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE, 703 rq->buff.map_dir); 704 __free_page(rq->wqe_overflow.page); 705} 706 707static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 708 u32 xdp_frag_size, struct mlx5e_rq *rq) 709{ 710 struct mlx5_core_dev *mdev = c->mdev; 711 int err; 712 713 rq->wq_type = params->rq_wq_type; 714 rq->pdev = c->pdev; 715 rq->netdev = c->netdev; 716 rq->priv = c->priv; 717 rq->tstamp = c->tstamp; 718 rq->clock = &mdev->clock; 719 rq->icosq = &c->icosq; 720 rq->ix = c->ix; 721 rq->channel = c; 722 rq->mdev = mdev; 723 rq->hw_mtu = 724 MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN * !params->scatter_fcs_en; 725 rq->xdpsq = &c->rq_xdpsq; 726 rq->stats = &c->priv->channel_stats[c->ix]->rq; 727 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); 728 err = mlx5e_rq_set_handlers(rq, params, NULL); 729 if (err) 730 return err; 731 732 return __xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id, 733 xdp_frag_size); 734} 735 736static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 737 struct mlx5e_params *params, 738 struct mlx5e_rq_param *rqp, 739 struct mlx5e_rq *rq, 740 u32 *pool_size, 741 int node) 742{ 743 void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq); 744 int wq_size; 745 int err; 746 747 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 748 return 0; 749 err = mlx5e_rq_shampo_hd_alloc(rq, node); 750 if (err) 751 goto out; 752 rq->mpwqe.shampo->hd_per_wq = 753 mlx5e_shampo_hd_per_wq(mdev, params, rqp); 754 err = mlx5e_create_rq_hd_umr_mkey(mdev, rq); 755 if (err) 756 goto err_shampo_hd; 757 err = mlx5e_rq_shampo_hd_info_alloc(rq, node); 758 if (err) 759 goto err_shampo_info; 760 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node); 761 if (!rq->hw_gro_data) { 762 err = -ENOMEM; 763 goto err_hw_gro_data; 764 } 765 rq->mpwqe.shampo->key = 766 cpu_to_be32(rq->mpwqe.shampo->mkey); 767 rq->mpwqe.shampo->hd_per_wqe = 768 mlx5e_shampo_hd_per_wqe(mdev, params, rqp); 769 wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 770 *pool_size += (rq->mpwqe.shampo->hd_per_wqe * wq_size) / 771 MLX5E_SHAMPO_WQ_HEADER_PER_PAGE; 772 return 0; 773 774err_hw_gro_data: 775 mlx5e_rq_shampo_hd_info_free(rq); 776err_shampo_info: 777 mlx5_core_destroy_mkey(mdev, rq->mpwqe.shampo->mkey); 778err_shampo_hd: 779 mlx5e_rq_shampo_hd_free(rq); 780out: 781 return err; 782} 783 784static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) 785{ 786 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 787 return; 788 789 kvfree(rq->hw_gro_data); 790 mlx5e_rq_shampo_hd_info_free(rq); 791 mlx5_core_destroy_mkey(rq->mdev, rq->mpwqe.shampo->mkey); 792 mlx5e_rq_shampo_hd_free(rq); 793} 794 795static int mlx5e_alloc_rq(struct mlx5e_params *params, 796 struct mlx5e_xsk_param *xsk, 797 struct mlx5e_rq_param *rqp, 798 int node, struct mlx5e_rq *rq) 799{ 800 struct mlx5_core_dev *mdev = rq->mdev; 801 void *rqc = rqp->rqc; 802 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 803 u32 pool_size; 804 int wq_sz; 805 int err; 806 int i; 807 808 rqp->wq.db_numa_node = node; 809 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); 810 811 if (params->xdp_prog) 812 bpf_prog_inc(params->xdp_prog); 813 RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog); 814 815 rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 816 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); 817 pool_size = 1 << params->log_rq_mtu_frames; 818 819 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 820 821 switch (rq->wq_type) { 822 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 823 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, 824 &rq->wq_ctrl); 825 if (err) 826 goto err_rq_xdp_prog; 827 828 err = mlx5e_alloc_mpwqe_rq_drop_page(rq); 829 if (err) 830 goto err_rq_wq_destroy; 831 832 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 833 834 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 835 836 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); 837 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); 838 rq->mpwqe.pages_per_wqe = 839 mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, 840 rq->mpwqe.umr_mode); 841 rq->mpwqe.umr_wqebbs = 842 mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift, 843 rq->mpwqe.umr_mode); 844 rq->mpwqe.mtts_per_wqe = 845 mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift, 846 rq->mpwqe.umr_mode); 847 848 pool_size = rq->mpwqe.pages_per_wqe << 849 mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); 850 851 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog) 852 pool_size *= 2; /* additional page per packet for the linear part */ 853 854 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 855 rq->mpwqe.num_strides = 856 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); 857 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz); 858 859 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); 860 861 err = mlx5e_create_rq_umr_mkey(mdev, rq); 862 if (err) 863 goto err_rq_drop_page; 864 865 err = mlx5e_rq_alloc_mpwqe_info(rq, node); 866 if (err) 867 goto err_rq_mkey; 868 869 err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node); 870 if (err) 871 goto err_free_mpwqe_info; 872 873 break; 874 default: /* MLX5_WQ_TYPE_CYCLIC */ 875 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, 876 &rq->wq_ctrl); 877 if (err) 878 goto err_rq_xdp_prog; 879 880 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 881 882 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 883 884 rq->wqe.info = rqp->frags_info; 885 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; 886 887 err = mlx5e_init_wqe_alloc_info(rq, node); 888 if (err) 889 goto err_rq_wq_destroy; 890 } 891 892 if (xsk) { 893 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 894 MEM_TYPE_XSK_BUFF_POOL, NULL); 895 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq); 896 } else { 897 /* Create a page_pool and register it with rxq */ 898 struct page_pool_params pp_params = { 0 }; 899 900 pp_params.order = 0; 901 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 902 pp_params.pool_size = pool_size; 903 pp_params.nid = node; 904 pp_params.dev = rq->pdev; 905 pp_params.napi = rq->cq.napi; 906 pp_params.netdev = rq->netdev; 907 pp_params.dma_dir = rq->buff.map_dir; 908 pp_params.max_len = PAGE_SIZE; 909 910 /* page_pool can be used even when there is no rq->xdp_prog, 911 * given page_pool does not handle DMA mapping there is no 912 * required state to clear. And page_pool gracefully handle 913 * elevated refcnt. 914 */ 915 rq->page_pool = page_pool_create(&pp_params); 916 if (IS_ERR(rq->page_pool)) { 917 err = PTR_ERR(rq->page_pool); 918 rq->page_pool = NULL; 919 goto err_free_by_rq_type; 920 } 921 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) 922 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 923 MEM_TYPE_PAGE_POOL, rq->page_pool); 924 } 925 if (err) 926 goto err_destroy_page_pool; 927 928 for (i = 0; i < wq_sz; i++) { 929 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 930 struct mlx5e_rx_wqe_ll *wqe = 931 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 932 u32 byte_count = 933 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 934 u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) << 935 rq->mpwqe.page_shift; 936 u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ? 937 0 : rq->buff.headroom; 938 939 wqe->data[0].addr = cpu_to_be64(dma_offset + headroom); 940 wqe->data[0].byte_count = cpu_to_be32(byte_count); 941 wqe->data[0].lkey = rq->mpwqe.umr_mkey_be; 942 } else { 943 struct mlx5e_rx_wqe_cyc *wqe = 944 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 945 int f; 946 947 for (f = 0; f < rq->wqe.info.num_frags; f++) { 948 u32 frag_size = rq->wqe.info.arr[f].frag_size | 949 MLX5_HW_START_PADDING; 950 951 wqe->data[f].byte_count = cpu_to_be32(frag_size); 952 wqe->data[f].lkey = rq->mkey_be; 953 } 954 /* check if num_frags is not a pow of two */ 955 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 956 wqe->data[f].byte_count = 0; 957 wqe->data[f].lkey = params->terminate_lkey_be; 958 wqe->data[f].addr = 0; 959 } 960 } 961 } 962 963 INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work); 964 965 switch (params->rx_cq_moderation.cq_period_mode) { 966 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: 967 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; 968 break; 969 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: 970 default: 971 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 972 } 973 974 return 0; 975 976err_destroy_page_pool: 977 page_pool_destroy(rq->page_pool); 978err_free_by_rq_type: 979 switch (rq->wq_type) { 980 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 981 mlx5e_rq_free_shampo(rq); 982err_free_mpwqe_info: 983 kvfree(rq->mpwqe.info); 984err_rq_mkey: 985 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 986err_rq_drop_page: 987 mlx5e_free_mpwqe_rq_drop_page(rq); 988 break; 989 default: /* MLX5_WQ_TYPE_CYCLIC */ 990 mlx5e_free_wqe_alloc_info(rq); 991 } 992err_rq_wq_destroy: 993 mlx5_wq_destroy(&rq->wq_ctrl); 994err_rq_xdp_prog: 995 if (params->xdp_prog) 996 bpf_prog_put(params->xdp_prog); 997 998 return err; 999} 1000 1001static void mlx5e_free_rq(struct mlx5e_rq *rq) 1002{ 1003 struct bpf_prog *old_prog; 1004 1005 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 1006 old_prog = rcu_dereference_protected(rq->xdp_prog, 1007 lockdep_is_held(&rq->priv->state_lock)); 1008 if (old_prog) 1009 bpf_prog_put(old_prog); 1010 } 1011 1012 switch (rq->wq_type) { 1013 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1014 kvfree(rq->mpwqe.info); 1015 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 1016 mlx5e_free_mpwqe_rq_drop_page(rq); 1017 mlx5e_rq_free_shampo(rq); 1018 break; 1019 default: /* MLX5_WQ_TYPE_CYCLIC */ 1020 mlx5e_free_wqe_alloc_info(rq); 1021 } 1022 1023 xdp_rxq_info_unreg(&rq->xdp_rxq); 1024 page_pool_destroy(rq->page_pool); 1025 mlx5_wq_destroy(&rq->wq_ctrl); 1026} 1027 1028int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter) 1029{ 1030 struct mlx5_core_dev *mdev = rq->mdev; 1031 u8 ts_format; 1032 void *in; 1033 void *rqc; 1034 void *wq; 1035 int inlen; 1036 int err; 1037 1038 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 1039 sizeof(u64) * rq->wq_ctrl.buf.npages; 1040 in = kvzalloc(inlen, GFP_KERNEL); 1041 if (!in) 1042 return -ENOMEM; 1043 1044 ts_format = mlx5_is_real_time_rq(mdev) ? 1045 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1046 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1047 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 1048 wq = MLX5_ADDR_OF(rqc, rqc, wq); 1049 1050 memcpy(rqc, param->rqc, sizeof(param->rqc)); 1051 1052 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 1053 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 1054 MLX5_SET(rqc, rqc, ts_format, ts_format); 1055 MLX5_SET(rqc, rqc, counter_set_id, q_counter); 1056 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 1057 MLX5_ADAPTER_PAGE_SHIFT); 1058 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 1059 1060 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1061 MLX5_SET(wq, wq, log_headers_buffer_entry_num, 1062 order_base_2(rq->mpwqe.shampo->hd_per_wq)); 1063 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey); 1064 } 1065 1066 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 1067 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1068 1069 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 1070 1071 kvfree(in); 1072 1073 return err; 1074} 1075 1076static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) 1077{ 1078 struct mlx5_core_dev *mdev = rq->mdev; 1079 1080 void *in; 1081 void *rqc; 1082 int inlen; 1083 int err; 1084 1085 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1086 in = kvzalloc(inlen, GFP_KERNEL); 1087 if (!in) 1088 return -ENOMEM; 1089 1090 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY) 1091 mlx5e_rqwq_reset(rq); 1092 1093 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1094 1095 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 1096 MLX5_SET(rqc, rqc, state, next_state); 1097 1098 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1099 1100 kvfree(in); 1101 1102 return err; 1103} 1104 1105static void mlx5e_flush_rq_cq(struct mlx5e_rq *rq) 1106{ 1107 struct mlx5_cqwq *cqwq = &rq->cq.wq; 1108 struct mlx5_cqe64 *cqe; 1109 1110 if (test_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state)) { 1111 while ((cqe = mlx5_cqwq_get_cqe_enahnced_comp(cqwq))) 1112 mlx5_cqwq_pop(cqwq); 1113 } else { 1114 while ((cqe = mlx5_cqwq_get_cqe(cqwq))) 1115 mlx5_cqwq_pop(cqwq); 1116 } 1117 1118 mlx5_cqwq_update_db_record(cqwq); 1119} 1120 1121int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state) 1122{ 1123 struct net_device *dev = rq->netdev; 1124 int err; 1125 1126 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); 1127 if (err) { 1128 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); 1129 return err; 1130 } 1131 1132 mlx5e_free_rx_descs(rq); 1133 mlx5e_flush_rq_cq(rq); 1134 1135 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1136 if (err) { 1137 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); 1138 return err; 1139 } 1140 1141 return 0; 1142} 1143 1144static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 1145{ 1146 struct mlx5_core_dev *mdev = rq->mdev; 1147 void *in; 1148 void *rqc; 1149 int inlen; 1150 int err; 1151 1152 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1153 in = kvzalloc(inlen, GFP_KERNEL); 1154 if (!in) 1155 return -ENOMEM; 1156 1157 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1158 1159 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 1160 MLX5_SET64(modify_rq_in, in, modify_bitmask, 1161 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 1162 MLX5_SET(rqc, rqc, vsd, vsd); 1163 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 1164 1165 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1166 1167 kvfree(in); 1168 1169 return err; 1170} 1171 1172void mlx5e_destroy_rq(struct mlx5e_rq *rq) 1173{ 1174 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 1175} 1176 1177int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 1178{ 1179 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 1180 1181 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 1182 1183 do { 1184 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 1185 return 0; 1186 1187 msleep(20); 1188 } while (time_before(jiffies, exp_time)); 1189 1190 netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 1191 rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 1192 1193 mlx5e_reporter_rx_timeout(rq); 1194 return -ETIMEDOUT; 1195} 1196 1197void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq) 1198{ 1199 struct mlx5_wq_ll *wq; 1200 u16 head; 1201 int i; 1202 1203 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 1204 return; 1205 1206 wq = &rq->mpwqe.wq; 1207 head = wq->head; 1208 1209 /* Release WQEs that are in missing state: they have been 1210 * popped from the list after completion but were not freed 1211 * due to deferred release. 1212 * Also free the linked-list reserved entry, hence the "+ 1". 1213 */ 1214 for (i = 0; i < mlx5_wq_ll_missing(wq) + 1; i++) { 1215 rq->dealloc_wqe(rq, head); 1216 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 1217 } 1218 1219 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1220 u16 len; 1221 1222 len = (rq->mpwqe.shampo->pi - rq->mpwqe.shampo->ci) & 1223 (rq->mpwqe.shampo->hd_per_wq - 1); 1224 mlx5e_shampo_dealloc_hd(rq, len, rq->mpwqe.shampo->ci, false); 1225 rq->mpwqe.shampo->pi = rq->mpwqe.shampo->ci; 1226 } 1227 1228 rq->mpwqe.actual_wq_head = wq->head; 1229 rq->mpwqe.umr_in_progress = 0; 1230 rq->mpwqe.umr_completed = 0; 1231} 1232 1233void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 1234{ 1235 __be16 wqe_ix_be; 1236 u16 wqe_ix; 1237 1238 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1239 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 1240 1241 mlx5e_free_rx_missing_descs(rq); 1242 1243 while (!mlx5_wq_ll_is_empty(wq)) { 1244 struct mlx5e_rx_wqe_ll *wqe; 1245 1246 wqe_ix_be = *wq->tail_next; 1247 wqe_ix = be16_to_cpu(wqe_ix_be); 1248 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 1249 rq->dealloc_wqe(rq, wqe_ix); 1250 mlx5_wq_ll_pop(wq, wqe_ix_be, 1251 &wqe->next.next_wqe_index); 1252 } 1253 1254 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1255 mlx5e_shampo_dealloc_hd(rq, rq->mpwqe.shampo->hd_per_wq, 1256 0, true); 1257 } else { 1258 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 1259 u16 missing = mlx5_wq_cyc_missing(wq); 1260 u16 head = mlx5_wq_cyc_get_head(wq); 1261 1262 while (!mlx5_wq_cyc_is_empty(wq)) { 1263 wqe_ix = mlx5_wq_cyc_get_tail(wq); 1264 rq->dealloc_wqe(rq, wqe_ix); 1265 mlx5_wq_cyc_pop(wq); 1266 } 1267 /* Missing slots might also contain unreleased pages due to 1268 * deferred release. 1269 */ 1270 while (missing--) { 1271 wqe_ix = mlx5_wq_cyc_ctr2ix(wq, head++); 1272 rq->dealloc_wqe(rq, wqe_ix); 1273 } 1274 } 1275 1276} 1277 1278int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, 1279 struct mlx5e_xsk_param *xsk, int node, u16 q_counter, 1280 struct mlx5e_rq *rq) 1281{ 1282 struct mlx5_core_dev *mdev = rq->mdev; 1283 int err; 1284 1285 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 1286 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state); 1287 1288 err = mlx5e_alloc_rq(params, xsk, param, node, rq); 1289 if (err) 1290 return err; 1291 1292 err = mlx5e_create_rq(rq, param, q_counter); 1293 if (err) 1294 goto err_free_rq; 1295 1296 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1297 if (err) 1298 goto err_destroy_rq; 1299 1300 if (MLX5_CAP_ETH(mdev, cqe_checksum_full)) 1301 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state); 1302 1303 if (params->rx_dim_enabled) 1304 __set_bit(MLX5E_RQ_STATE_DIM, &rq->state); 1305 1306 /* We disable csum_complete when XDP is enabled since 1307 * XDP programs might manipulate packets which will render 1308 * skb->checksum incorrect. 1309 */ 1310 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog) 1311 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state); 1312 1313 /* For CQE compression on striding RQ, use stride index provided by 1314 * HW if capability is supported. 1315 */ 1316 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) && 1317 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) 1318 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state); 1319 1320 /* For enhanced CQE compression packet processing. decompress 1321 * session according to the enhanced layout. 1322 */ 1323 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) && 1324 MLX5_CAP_GEN(mdev, enhanced_cqe_compression)) 1325 __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state); 1326 1327 return 0; 1328 1329err_destroy_rq: 1330 mlx5e_destroy_rq(rq); 1331err_free_rq: 1332 mlx5e_free_rq(rq); 1333 1334 return err; 1335} 1336 1337void mlx5e_activate_rq(struct mlx5e_rq *rq) 1338{ 1339 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1340} 1341 1342void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 1343{ 1344 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1345 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */ 1346} 1347 1348void mlx5e_close_rq(struct mlx5e_rq *rq) 1349{ 1350 cancel_work_sync(&rq->dim.work); 1351 cancel_work_sync(&rq->recover_work); 1352 mlx5e_destroy_rq(rq); 1353 mlx5e_free_rx_descs(rq); 1354 mlx5e_free_rq(rq); 1355} 1356 1357u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, 1358 struct mlx5e_priv *priv, 1359 const struct mlx5e_profile *profile, 1360 u8 lag_port, u8 tc) 1361{ 1362 if (profile->get_tisn) 1363 return profile->get_tisn(mdev, priv, lag_port, tc); 1364 1365 return mdev->mlx5e_res.hw_objs.tisn[lag_port][tc]; 1366} 1367 1368static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 1369{ 1370 kvfree(sq->db.xdpi_fifo.xi); 1371 kvfree(sq->db.wqe_info); 1372} 1373 1374static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 1375{ 1376 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 1377 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1378 int entries; 1379 size_t size; 1380 1381 /* upper bound for maximum num of entries of all xmit_modes. */ 1382 entries = roundup_pow_of_two(wq_sz * MLX5_SEND_WQEBB_NUM_DS * 1383 MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO); 1384 1385 size = array_size(sizeof(*xdpi_fifo->xi), entries); 1386 xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa); 1387 if (!xdpi_fifo->xi) 1388 return -ENOMEM; 1389 1390 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 1391 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 1392 xdpi_fifo->mask = entries - 1; 1393 1394 return 0; 1395} 1396 1397static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 1398{ 1399 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1400 size_t size; 1401 int err; 1402 1403 size = array_size(sizeof(*sq->db.wqe_info), wq_sz); 1404 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1405 if (!sq->db.wqe_info) 1406 return -ENOMEM; 1407 1408 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 1409 if (err) { 1410 mlx5e_free_xdpsq_db(sq); 1411 return err; 1412 } 1413 1414 return 0; 1415} 1416 1417static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 1418 struct mlx5e_params *params, 1419 struct xsk_buff_pool *xsk_pool, 1420 struct mlx5e_sq_param *param, 1421 struct mlx5e_xdpsq *sq, 1422 bool is_redirect) 1423{ 1424 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1425 struct mlx5_core_dev *mdev = c->mdev; 1426 struct mlx5_wq_cyc *wq = &sq->wq; 1427 int err; 1428 1429 sq->pdev = c->pdev; 1430 sq->mkey_be = c->mkey_be; 1431 sq->channel = c; 1432 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; 1433 sq->min_inline_mode = params->tx_min_inline_mode; 1434 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN; 1435 sq->xsk_pool = xsk_pool; 1436 1437 sq->stats = sq->xsk_pool ? 1438 &c->priv->channel_stats[c->ix]->xsksq : 1439 is_redirect ? 1440 &c->priv->channel_stats[c->ix]->xdpsq : 1441 &c->priv->channel_stats[c->ix]->rq_xdpsq; 1442 sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) : 1443 mlx5e_stop_room_for_max_wqe(mdev); 1444 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1445 1446 param->wq.db_numa_node = cpu_to_node(c->cpu); 1447 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1448 if (err) 1449 return err; 1450 wq->db = &wq->db[MLX5_SND_DBR]; 1451 1452 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 1453 if (err) 1454 goto err_sq_wq_destroy; 1455 1456 return 0; 1457 1458err_sq_wq_destroy: 1459 mlx5_wq_destroy(&sq->wq_ctrl); 1460 1461 return err; 1462} 1463 1464static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 1465{ 1466 mlx5e_free_xdpsq_db(sq); 1467 mlx5_wq_destroy(&sq->wq_ctrl); 1468} 1469 1470static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 1471{ 1472 kvfree(sq->db.wqe_info); 1473} 1474 1475static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 1476{ 1477 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1478 size_t size; 1479 1480 size = array_size(wq_sz, sizeof(*sq->db.wqe_info)); 1481 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1482 if (!sq->db.wqe_info) 1483 return -ENOMEM; 1484 1485 return 0; 1486} 1487 1488static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work) 1489{ 1490 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1491 recover_work); 1492 1493 mlx5e_reporter_icosq_cqe_err(sq); 1494} 1495 1496static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work) 1497{ 1498 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1499 recover_work); 1500 1501 /* Not implemented yet. */ 1502 1503 netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n"); 1504} 1505 1506static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1507 struct mlx5e_sq_param *param, 1508 struct mlx5e_icosq *sq, 1509 work_func_t recover_work_func) 1510{ 1511 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1512 struct mlx5_core_dev *mdev = c->mdev; 1513 struct mlx5_wq_cyc *wq = &sq->wq; 1514 int err; 1515 1516 sq->channel = c; 1517 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; 1518 sq->reserved_room = param->stop_room; 1519 1520 param->wq.db_numa_node = cpu_to_node(c->cpu); 1521 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1522 if (err) 1523 return err; 1524 wq->db = &wq->db[MLX5_SND_DBR]; 1525 1526 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1527 if (err) 1528 goto err_sq_wq_destroy; 1529 1530 INIT_WORK(&sq->recover_work, recover_work_func); 1531 1532 return 0; 1533 1534err_sq_wq_destroy: 1535 mlx5_wq_destroy(&sq->wq_ctrl); 1536 1537 return err; 1538} 1539 1540static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1541{ 1542 mlx5e_free_icosq_db(sq); 1543 mlx5_wq_destroy(&sq->wq_ctrl); 1544} 1545 1546void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1547{ 1548 kvfree(sq->db.wqe_info); 1549 kvfree(sq->db.skb_fifo.fifo); 1550 kvfree(sq->db.dma_fifo); 1551} 1552 1553int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1554{ 1555 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1556 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1557 1558 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1559 sizeof(*sq->db.dma_fifo)), 1560 GFP_KERNEL, numa); 1561 sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz, 1562 sizeof(*sq->db.skb_fifo.fifo)), 1563 GFP_KERNEL, numa); 1564 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1565 sizeof(*sq->db.wqe_info)), 1566 GFP_KERNEL, numa); 1567 if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) { 1568 mlx5e_free_txqsq_db(sq); 1569 return -ENOMEM; 1570 } 1571 1572 sq->dma_fifo_mask = df_sz - 1; 1573 1574 sq->db.skb_fifo.pc = &sq->skb_fifo_pc; 1575 sq->db.skb_fifo.cc = &sq->skb_fifo_cc; 1576 sq->db.skb_fifo.mask = df_sz - 1; 1577 1578 return 0; 1579} 1580 1581static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1582 int txq_ix, 1583 struct mlx5e_params *params, 1584 struct mlx5e_sq_param *param, 1585 struct mlx5e_txqsq *sq, 1586 int tc) 1587{ 1588 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1589 struct mlx5_core_dev *mdev = c->mdev; 1590 struct mlx5_wq_cyc *wq = &sq->wq; 1591 int err; 1592 1593 sq->pdev = c->pdev; 1594 sq->clock = &mdev->clock; 1595 sq->mkey_be = c->mkey_be; 1596 sq->netdev = c->netdev; 1597 sq->mdev = c->mdev; 1598 sq->channel = c; 1599 sq->priv = c->priv; 1600 sq->ch_ix = c->ix; 1601 sq->txq_ix = txq_ix; 1602 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; 1603 sq->min_inline_mode = params->tx_min_inline_mode; 1604 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1605 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1606 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1607 if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert)) 1608 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state); 1609 if (mlx5_ipsec_device_caps(c->priv->mdev)) 1610 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1611 if (param->is_mpw) 1612 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state); 1613 sq->stop_room = param->stop_room; 1614 sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev); 1615 1616 param->wq.db_numa_node = cpu_to_node(c->cpu); 1617 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1618 if (err) 1619 return err; 1620 wq->db = &wq->db[MLX5_SND_DBR]; 1621 1622 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1623 if (err) 1624 goto err_sq_wq_destroy; 1625 1626 INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work); 1627 sq->dim.mode = params->tx_cq_moderation.cq_period_mode; 1628 1629 return 0; 1630 1631err_sq_wq_destroy: 1632 mlx5_wq_destroy(&sq->wq_ctrl); 1633 1634 return err; 1635} 1636 1637void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1638{ 1639 mlx5e_free_txqsq_db(sq); 1640 mlx5_wq_destroy(&sq->wq_ctrl); 1641} 1642 1643static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1644 struct mlx5e_sq_param *param, 1645 struct mlx5e_create_sq_param *csp, 1646 u32 *sqn) 1647{ 1648 u8 ts_format; 1649 void *in; 1650 void *sqc; 1651 void *wq; 1652 int inlen; 1653 int err; 1654 1655 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1656 sizeof(u64) * csp->wq_ctrl->buf.npages; 1657 in = kvzalloc(inlen, GFP_KERNEL); 1658 if (!in) 1659 return -ENOMEM; 1660 1661 ts_format = mlx5_is_real_time_sq(mdev) ? 1662 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1663 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1664 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1665 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1666 1667 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1668 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1669 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1670 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1671 MLX5_SET(sqc, sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn); 1672 MLX5_SET(sqc, sqc, ts_format, ts_format); 1673 1674 1675 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1676 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1677 1678 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1679 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1680 1681 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1682 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); 1683 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1684 MLX5_ADAPTER_PAGE_SHIFT); 1685 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1686 1687 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1688 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1689 1690 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1691 1692 kvfree(in); 1693 1694 return err; 1695} 1696 1697int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1698 struct mlx5e_modify_sq_param *p) 1699{ 1700 u64 bitmask = 0; 1701 void *in; 1702 void *sqc; 1703 int inlen; 1704 int err; 1705 1706 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1707 in = kvzalloc(inlen, GFP_KERNEL); 1708 if (!in) 1709 return -ENOMEM; 1710 1711 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1712 1713 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1714 MLX5_SET(sqc, sqc, state, p->next_state); 1715 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1716 bitmask |= 1; 1717 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1718 } 1719 if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) { 1720 bitmask |= 1 << 2; 1721 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id); 1722 } 1723 MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask); 1724 1725 err = mlx5_core_modify_sq(mdev, sqn, in); 1726 1727 kvfree(in); 1728 1729 return err; 1730} 1731 1732static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1733{ 1734 mlx5_core_destroy_sq(mdev, sqn); 1735} 1736 1737int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1738 struct mlx5e_sq_param *param, 1739 struct mlx5e_create_sq_param *csp, 1740 u16 qos_queue_group_id, 1741 u32 *sqn) 1742{ 1743 struct mlx5e_modify_sq_param msp = {0}; 1744 int err; 1745 1746 err = mlx5e_create_sq(mdev, param, csp, sqn); 1747 if (err) 1748 return err; 1749 1750 msp.curr_state = MLX5_SQC_STATE_RST; 1751 msp.next_state = MLX5_SQC_STATE_RDY; 1752 if (qos_queue_group_id) { 1753 msp.qos_update = true; 1754 msp.qos_queue_group_id = qos_queue_group_id; 1755 } 1756 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1757 if (err) 1758 mlx5e_destroy_sq(mdev, *sqn); 1759 1760 return err; 1761} 1762 1763static int mlx5e_set_sq_maxrate(struct net_device *dev, 1764 struct mlx5e_txqsq *sq, u32 rate); 1765 1766int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, 1767 struct mlx5e_params *params, struct mlx5e_sq_param *param, 1768 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, 1769 struct mlx5e_sq_stats *sq_stats) 1770{ 1771 struct mlx5e_create_sq_param csp = {}; 1772 u32 tx_rate; 1773 int err; 1774 1775 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1776 if (err) 1777 return err; 1778 1779 sq->stats = sq_stats; 1780 1781 csp.tisn = tisn; 1782 csp.tis_lst_sz = 1; 1783 csp.cqn = sq->cq.mcq.cqn; 1784 csp.wq_ctrl = &sq->wq_ctrl; 1785 csp.min_inline_mode = sq->min_inline_mode; 1786 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn); 1787 if (err) 1788 goto err_free_txqsq; 1789 1790 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1791 if (tx_rate) 1792 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1793 1794 if (params->tx_dim_enabled) 1795 sq->state |= BIT(MLX5E_SQ_STATE_DIM); 1796 1797 return 0; 1798 1799err_free_txqsq: 1800 mlx5e_free_txqsq(sq); 1801 1802 return err; 1803} 1804 1805void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1806{ 1807 sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix); 1808 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1809 netdev_tx_reset_queue(sq->txq); 1810 netif_tx_start_queue(sq->txq); 1811 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, sq->cq.napi); 1812} 1813 1814void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1815{ 1816 __netif_tx_lock_bh(txq); 1817 netif_tx_stop_queue(txq); 1818 __netif_tx_unlock_bh(txq); 1819} 1820 1821void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1822{ 1823 struct mlx5_wq_cyc *wq = &sq->wq; 1824 1825 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, NULL); 1826 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1827 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */ 1828 1829 mlx5e_tx_disable_queue(sq->txq); 1830 1831 /* last doorbell out, godspeed .. */ 1832 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1833 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1834 struct mlx5e_tx_wqe *nop; 1835 1836 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) { 1837 .num_wqebbs = 1, 1838 }; 1839 1840 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1841 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1842 } 1843} 1844 1845void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1846{ 1847 struct mlx5_core_dev *mdev = sq->mdev; 1848 struct mlx5_rate_limit rl = {0}; 1849 1850 cancel_work_sync(&sq->dim.work); 1851 cancel_work_sync(&sq->recover_work); 1852 mlx5e_destroy_sq(mdev, sq->sqn); 1853 if (sq->rate_limit) { 1854 rl.rate = sq->rate_limit; 1855 mlx5_rl_remove_rate(mdev, &rl); 1856 } 1857 mlx5e_free_txqsq_descs(sq); 1858 mlx5e_free_txqsq(sq); 1859} 1860 1861void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1862{ 1863 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 1864 recover_work); 1865 1866 mlx5e_reporter_tx_err_cqe(sq); 1867} 1868 1869static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 1870 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq, 1871 work_func_t recover_work_func) 1872{ 1873 struct mlx5e_create_sq_param csp = {}; 1874 int err; 1875 1876 err = mlx5e_alloc_icosq(c, param, sq, recover_work_func); 1877 if (err) 1878 return err; 1879 1880 csp.cqn = sq->cq.mcq.cqn; 1881 csp.wq_ctrl = &sq->wq_ctrl; 1882 csp.min_inline_mode = params->tx_min_inline_mode; 1883 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 1884 if (err) 1885 goto err_free_icosq; 1886 1887 if (param->is_tls) { 1888 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list(); 1889 if (IS_ERR(sq->ktls_resync)) { 1890 err = PTR_ERR(sq->ktls_resync); 1891 goto err_destroy_icosq; 1892 } 1893 } 1894 return 0; 1895 1896err_destroy_icosq: 1897 mlx5e_destroy_sq(c->mdev, sq->sqn); 1898err_free_icosq: 1899 mlx5e_free_icosq(sq); 1900 1901 return err; 1902} 1903 1904void mlx5e_activate_icosq(struct mlx5e_icosq *icosq) 1905{ 1906 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1907} 1908 1909void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq) 1910{ 1911 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1912 synchronize_net(); /* Sync with NAPI. */ 1913} 1914 1915static void mlx5e_close_icosq(struct mlx5e_icosq *sq) 1916{ 1917 struct mlx5e_channel *c = sq->channel; 1918 1919 if (sq->ktls_resync) 1920 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync); 1921 mlx5e_destroy_sq(c->mdev, sq->sqn); 1922 mlx5e_free_icosq_descs(sq); 1923 mlx5e_free_icosq(sq); 1924} 1925 1926int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 1927 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool, 1928 struct mlx5e_xdpsq *sq, bool is_redirect) 1929{ 1930 struct mlx5e_create_sq_param csp = {}; 1931 int err; 1932 1933 err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect); 1934 if (err) 1935 return err; 1936 1937 csp.tis_lst_sz = 1; 1938 csp.tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 1939 c->lag_port, 0); /* tc = 0 */ 1940 csp.cqn = sq->cq.mcq.cqn; 1941 csp.wq_ctrl = &sq->wq_ctrl; 1942 csp.min_inline_mode = sq->min_inline_mode; 1943 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1944 1945 if (param->is_xdp_mb) 1946 set_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state); 1947 1948 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 1949 if (err) 1950 goto err_free_xdpsq; 1951 1952 mlx5e_set_xmit_fp(sq, param->is_mpw); 1953 1954 if (!param->is_mpw && !test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state)) { 1955 unsigned int ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + 1; 1956 unsigned int inline_hdr_sz = 0; 1957 int i; 1958 1959 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) { 1960 inline_hdr_sz = MLX5E_XDP_MIN_INLINE; 1961 ds_cnt++; 1962 } 1963 1964 /* Pre initialize fixed WQE fields */ 1965 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) { 1966 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i); 1967 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 1968 struct mlx5_wqe_eth_seg *eseg = &wqe->eth; 1969 1970 sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) { 1971 .num_wqebbs = 1, 1972 .num_pkts = 1, 1973 }; 1974 1975 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 1976 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz); 1977 } 1978 } 1979 1980 return 0; 1981 1982err_free_xdpsq: 1983 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1984 mlx5e_free_xdpsq(sq); 1985 1986 return err; 1987} 1988 1989void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) 1990{ 1991 struct mlx5e_channel *c = sq->channel; 1992 1993 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1994 synchronize_net(); /* Sync with NAPI. */ 1995 1996 mlx5e_destroy_sq(c->mdev, sq->sqn); 1997 mlx5e_free_xdpsq_descs(sq); 1998 mlx5e_free_xdpsq(sq); 1999} 2000 2001static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, 2002 struct net_device *netdev, 2003 struct workqueue_struct *workqueue, 2004 struct mlx5e_cq_param *param, 2005 struct mlx5e_cq *cq) 2006{ 2007 struct mlx5_core_cq *mcq = &cq->mcq; 2008 int err; 2009 u32 i; 2010 2011 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 2012 &cq->wq_ctrl); 2013 if (err) 2014 return err; 2015 2016 mcq->cqe_sz = 64; 2017 mcq->set_ci_db = cq->wq_ctrl.db.db; 2018 mcq->arm_db = cq->wq_ctrl.db.db + 1; 2019 *mcq->set_ci_db = 0; 2020 *mcq->arm_db = 0; 2021 mcq->vector = param->eq_ix; 2022 mcq->comp = mlx5e_completion_event; 2023 mcq->event = mlx5e_cq_error_event; 2024 2025 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 2026 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 2027 2028 cqe->op_own = 0xf1; 2029 cqe->validity_iteration_count = 0xff; 2030 } 2031 2032 cq->mdev = mdev; 2033 cq->netdev = netdev; 2034 cq->workqueue = workqueue; 2035 2036 return 0; 2037} 2038 2039static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev, 2040 struct mlx5e_cq_param *param, 2041 struct mlx5e_create_cq_param *ccp, 2042 struct mlx5e_cq *cq) 2043{ 2044 int err; 2045 2046 param->wq.buf_numa_node = ccp->node; 2047 param->wq.db_numa_node = ccp->node; 2048 param->eq_ix = ccp->ix; 2049 2050 err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, param, cq); 2051 2052 cq->napi = ccp->napi; 2053 cq->ch_stats = ccp->ch_stats; 2054 2055 return err; 2056} 2057 2058static void mlx5e_free_cq(struct mlx5e_cq *cq) 2059{ 2060 mlx5_wq_destroy(&cq->wq_ctrl); 2061} 2062 2063static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 2064{ 2065 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 2066 struct mlx5_core_dev *mdev = cq->mdev; 2067 struct mlx5_core_cq *mcq = &cq->mcq; 2068 2069 void *in; 2070 void *cqc; 2071 int inlen; 2072 int eqn; 2073 int err; 2074 2075 err = mlx5_comp_eqn_get(mdev, param->eq_ix, &eqn); 2076 if (err) 2077 return err; 2078 2079 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 2080 sizeof(u64) * cq->wq_ctrl.buf.npages; 2081 in = kvzalloc(inlen, GFP_KERNEL); 2082 if (!in) 2083 return -ENOMEM; 2084 2085 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 2086 2087 memcpy(cqc, param->cqc, sizeof(param->cqc)); 2088 2089 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 2090 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 2091 2092 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode); 2093 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 2094 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); 2095 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 2096 MLX5_ADAPTER_PAGE_SHIFT); 2097 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 2098 2099 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 2100 2101 kvfree(in); 2102 2103 if (err) 2104 return err; 2105 2106 mlx5e_cq_arm(cq); 2107 2108 return 0; 2109} 2110 2111static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 2112{ 2113 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 2114} 2115 2116int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder, 2117 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, 2118 struct mlx5e_cq *cq) 2119{ 2120 int err; 2121 2122 err = mlx5e_alloc_cq(mdev, param, ccp, cq); 2123 if (err) 2124 return err; 2125 2126 err = mlx5e_create_cq(cq, param); 2127 if (err) 2128 goto err_free_cq; 2129 2130 if (MLX5_CAP_GEN(mdev, cq_moderation)) 2131 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts); 2132 return 0; 2133 2134err_free_cq: 2135 mlx5e_free_cq(cq); 2136 2137 return err; 2138} 2139 2140void mlx5e_close_cq(struct mlx5e_cq *cq) 2141{ 2142 mlx5e_destroy_cq(cq); 2143 mlx5e_free_cq(cq); 2144} 2145 2146static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 2147 struct mlx5e_params *params, 2148 struct mlx5e_create_cq_param *ccp, 2149 struct mlx5e_channel_param *cparam) 2150{ 2151 int err; 2152 int tc; 2153 2154 for (tc = 0; tc < c->num_tc; tc++) { 2155 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->txq_sq.cqp, 2156 ccp, &c->sq[tc].cq); 2157 if (err) 2158 goto err_close_tx_cqs; 2159 } 2160 2161 return 0; 2162 2163err_close_tx_cqs: 2164 for (tc--; tc >= 0; tc--) 2165 mlx5e_close_cq(&c->sq[tc].cq); 2166 2167 return err; 2168} 2169 2170static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 2171{ 2172 int tc; 2173 2174 for (tc = 0; tc < c->num_tc; tc++) 2175 mlx5e_close_cq(&c->sq[tc].cq); 2176} 2177 2178static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq) 2179{ 2180 int tc; 2181 2182 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 2183 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count) 2184 return tc; 2185 2186 WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq); 2187 return -ENOENT; 2188} 2189 2190static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, 2191 u32 *hw_id) 2192{ 2193 int tc; 2194 2195 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { 2196 *hw_id = 0; 2197 return 0; 2198 } 2199 2200 tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix); 2201 if (tc < 0) 2202 return tc; 2203 2204 if (tc >= params->mqprio.num_tc) { 2205 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", 2206 tc, params->mqprio.num_tc); 2207 return -EINVAL; 2208 } 2209 2210 *hw_id = params->mqprio.channel.hw_id[tc]; 2211 return 0; 2212} 2213 2214static int mlx5e_open_sqs(struct mlx5e_channel *c, 2215 struct mlx5e_params *params, 2216 struct mlx5e_channel_param *cparam) 2217{ 2218 int err, tc; 2219 2220 for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) { 2221 int txq_ix = c->ix + tc * params->num_channels; 2222 u32 qos_queue_group_id; 2223 u32 tisn; 2224 2225 tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 2226 c->lag_port, tc); 2227 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id); 2228 if (err) 2229 goto err_close_sqs; 2230 2231 err = mlx5e_open_txqsq(c, tisn, txq_ix, 2232 params, &cparam->txq_sq, &c->sq[tc], tc, 2233 qos_queue_group_id, 2234 &c->priv->channel_stats[c->ix]->sq[tc]); 2235 if (err) 2236 goto err_close_sqs; 2237 } 2238 2239 return 0; 2240 2241err_close_sqs: 2242 for (tc--; tc >= 0; tc--) 2243 mlx5e_close_txqsq(&c->sq[tc]); 2244 2245 return err; 2246} 2247 2248static void mlx5e_close_sqs(struct mlx5e_channel *c) 2249{ 2250 int tc; 2251 2252 for (tc = 0; tc < c->num_tc; tc++) 2253 mlx5e_close_txqsq(&c->sq[tc]); 2254} 2255 2256static int mlx5e_set_sq_maxrate(struct net_device *dev, 2257 struct mlx5e_txqsq *sq, u32 rate) 2258{ 2259 struct mlx5e_priv *priv = netdev_priv(dev); 2260 struct mlx5_core_dev *mdev = priv->mdev; 2261 struct mlx5e_modify_sq_param msp = {0}; 2262 struct mlx5_rate_limit rl = {0}; 2263 u16 rl_index = 0; 2264 int err; 2265 2266 if (rate == sq->rate_limit) 2267 /* nothing to do */ 2268 return 0; 2269 2270 if (sq->rate_limit) { 2271 rl.rate = sq->rate_limit; 2272 /* remove current rl index to free space to next ones */ 2273 mlx5_rl_remove_rate(mdev, &rl); 2274 } 2275 2276 sq->rate_limit = 0; 2277 2278 if (rate) { 2279 rl.rate = rate; 2280 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 2281 if (err) { 2282 netdev_err(dev, "Failed configuring rate %u: %d\n", 2283 rate, err); 2284 return err; 2285 } 2286 } 2287 2288 msp.curr_state = MLX5_SQC_STATE_RDY; 2289 msp.next_state = MLX5_SQC_STATE_RDY; 2290 msp.rl_index = rl_index; 2291 msp.rl_update = true; 2292 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 2293 if (err) { 2294 netdev_err(dev, "Failed configuring rate %u: %d\n", 2295 rate, err); 2296 /* remove the rate from the table */ 2297 if (rate) 2298 mlx5_rl_remove_rate(mdev, &rl); 2299 return err; 2300 } 2301 2302 sq->rate_limit = rate; 2303 return 0; 2304} 2305 2306static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2307{ 2308 struct mlx5e_priv *priv = netdev_priv(dev); 2309 struct mlx5_core_dev *mdev = priv->mdev; 2310 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 2311 int err = 0; 2312 2313 if (!mlx5_rl_is_supported(mdev)) { 2314 netdev_err(dev, "Rate limiting is not supported on this device\n"); 2315 return -EINVAL; 2316 } 2317 2318 /* rate is given in Mb/sec, HW config is in Kb/sec */ 2319 rate = rate << 10; 2320 2321 /* Check whether rate in valid range, 0 is always valid */ 2322 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 2323 netdev_err(dev, "TX rate %u, is not in range\n", rate); 2324 return -ERANGE; 2325 } 2326 2327 mutex_lock(&priv->state_lock); 2328 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 2329 err = mlx5e_set_sq_maxrate(dev, sq, rate); 2330 if (!err) 2331 priv->tx_rates[index] = rate; 2332 mutex_unlock(&priv->state_lock); 2333 2334 return err; 2335} 2336 2337static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 2338 struct mlx5e_rq_param *rq_params) 2339{ 2340 u16 q_counter = c->priv->q_counter[c->sd_ix]; 2341 int err; 2342 2343 err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq); 2344 if (err) 2345 return err; 2346 2347 return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq); 2348} 2349 2350static int mlx5e_open_queues(struct mlx5e_channel *c, 2351 struct mlx5e_params *params, 2352 struct mlx5e_channel_param *cparam) 2353{ 2354 struct dim_cq_moder icocq_moder = {0, 0}; 2355 struct mlx5e_create_cq_param ccp; 2356 int err; 2357 2358 mlx5e_build_create_cq_param(&ccp, c); 2359 2360 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, &ccp, 2361 &c->async_icosq.cq); 2362 if (err) 2363 return err; 2364 2365 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->icosq.cqp, &ccp, 2366 &c->icosq.cq); 2367 if (err) 2368 goto err_close_async_icosq_cq; 2369 2370 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam); 2371 if (err) 2372 goto err_close_icosq_cq; 2373 2374 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, 2375 &c->xdpsq.cq); 2376 if (err) 2377 goto err_close_tx_cqs; 2378 2379 err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, 2380 &c->rq.cq); 2381 if (err) 2382 goto err_close_xdp_tx_cqs; 2383 2384 err = c->xdp ? mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, 2385 &ccp, &c->rq_xdpsq.cq) : 0; 2386 if (err) 2387 goto err_close_rx_cq; 2388 2389 spin_lock_init(&c->async_icosq_lock); 2390 2391 err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq, 2392 mlx5e_async_icosq_err_cqe_work); 2393 if (err) 2394 goto err_close_xdpsq_cq; 2395 2396 mutex_init(&c->icosq_recovery_lock); 2397 2398 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq, 2399 mlx5e_icosq_err_cqe_work); 2400 if (err) 2401 goto err_close_async_icosq; 2402 2403 err = mlx5e_open_sqs(c, params, cparam); 2404 if (err) 2405 goto err_close_icosq; 2406 2407 err = mlx5e_open_rxq_rq(c, params, &cparam->rq); 2408 if (err) 2409 goto err_close_sqs; 2410 2411 if (c->xdp) { 2412 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, 2413 &c->rq_xdpsq, false); 2414 if (err) 2415 goto err_close_rq; 2416 } 2417 2418 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true); 2419 if (err) 2420 goto err_close_xdp_sq; 2421 2422 return 0; 2423 2424err_close_xdp_sq: 2425 if (c->xdp) 2426 mlx5e_close_xdpsq(&c->rq_xdpsq); 2427 2428err_close_rq: 2429 mlx5e_close_rq(&c->rq); 2430 2431err_close_sqs: 2432 mlx5e_close_sqs(c); 2433 2434err_close_icosq: 2435 mlx5e_close_icosq(&c->icosq); 2436 2437err_close_async_icosq: 2438 mlx5e_close_icosq(&c->async_icosq); 2439 2440err_close_xdpsq_cq: 2441 if (c->xdp) 2442 mlx5e_close_cq(&c->rq_xdpsq.cq); 2443 2444err_close_rx_cq: 2445 mlx5e_close_cq(&c->rq.cq); 2446 2447err_close_xdp_tx_cqs: 2448 mlx5e_close_cq(&c->xdpsq.cq); 2449 2450err_close_tx_cqs: 2451 mlx5e_close_tx_cqs(c); 2452 2453err_close_icosq_cq: 2454 mlx5e_close_cq(&c->icosq.cq); 2455 2456err_close_async_icosq_cq: 2457 mlx5e_close_cq(&c->async_icosq.cq); 2458 2459 return err; 2460} 2461 2462static void mlx5e_close_queues(struct mlx5e_channel *c) 2463{ 2464 mlx5e_close_xdpsq(&c->xdpsq); 2465 if (c->xdp) 2466 mlx5e_close_xdpsq(&c->rq_xdpsq); 2467 /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */ 2468 cancel_work_sync(&c->icosq.recover_work); 2469 mlx5e_close_rq(&c->rq); 2470 mlx5e_close_sqs(c); 2471 mlx5e_close_icosq(&c->icosq); 2472 mutex_destroy(&c->icosq_recovery_lock); 2473 mlx5e_close_icosq(&c->async_icosq); 2474 if (c->xdp) 2475 mlx5e_close_cq(&c->rq_xdpsq.cq); 2476 mlx5e_close_cq(&c->rq.cq); 2477 mlx5e_close_cq(&c->xdpsq.cq); 2478 mlx5e_close_tx_cqs(c); 2479 mlx5e_close_cq(&c->icosq.cq); 2480 mlx5e_close_cq(&c->async_icosq.cq); 2481} 2482 2483static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) 2484{ 2485 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id); 2486 2487 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev); 2488} 2489 2490static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu) 2491{ 2492 if (ix > priv->stats_nch) { 2493 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix, 2494 priv->stats_nch); 2495 return -EINVAL; 2496 } 2497 2498 if (priv->channel_stats[ix]) 2499 return 0; 2500 2501 /* Asymmetric dynamic memory allocation. 2502 * Freed in mlx5e_priv_arrays_free, not on channel closure. 2503 */ 2504 netdev_dbg(priv->netdev, "Creating channel stats %d\n", ix); 2505 priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats), 2506 GFP_KERNEL, cpu_to_node(cpu)); 2507 if (!priv->channel_stats[ix]) 2508 return -ENOMEM; 2509 priv->stats_nch++; 2510 2511 return 0; 2512} 2513 2514void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c) 2515{ 2516 spin_lock_bh(&c->async_icosq_lock); 2517 mlx5e_trigger_irq(&c->async_icosq); 2518 spin_unlock_bh(&c->async_icosq_lock); 2519} 2520 2521void mlx5e_trigger_napi_sched(struct napi_struct *napi) 2522{ 2523 local_bh_disable(); 2524 napi_schedule(napi); 2525 local_bh_enable(); 2526} 2527 2528static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 2529 struct mlx5e_params *params, 2530 struct mlx5e_channel_param *cparam, 2531 struct xsk_buff_pool *xsk_pool, 2532 struct mlx5e_channel **cp) 2533{ 2534 struct net_device *netdev = priv->netdev; 2535 struct mlx5_core_dev *mdev; 2536 struct mlx5e_xsk_param xsk; 2537 struct mlx5e_channel *c; 2538 unsigned int irq; 2539 int vec_ix; 2540 int cpu; 2541 int err; 2542 2543 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 2544 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 2545 cpu = mlx5_comp_vector_get_cpu(mdev, vec_ix); 2546 2547 err = mlx5_comp_irqn_get(mdev, vec_ix, &irq); 2548 if (err) 2549 return err; 2550 2551 err = mlx5e_channel_stats_alloc(priv, ix, cpu); 2552 if (err) 2553 return err; 2554 2555 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 2556 if (!c) 2557 return -ENOMEM; 2558 2559 c->priv = priv; 2560 c->mdev = mdev; 2561 c->tstamp = &priv->tstamp; 2562 c->ix = ix; 2563 c->vec_ix = vec_ix; 2564 c->sd_ix = mlx5_sd_ch_ix_get_dev_ix(mdev, ix); 2565 c->cpu = cpu; 2566 c->pdev = mlx5_core_dma_dev(mdev); 2567 c->netdev = priv->netdev; 2568 c->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 2569 c->num_tc = mlx5e_get_dcb_num_tc(params); 2570 c->xdp = !!params->xdp_prog; 2571 c->stats = &priv->channel_stats[ix]->ch; 2572 c->aff_mask = irq_get_effective_affinity_mask(irq); 2573 c->lag_port = mlx5e_enumerate_lag_port(mdev, ix); 2574 2575 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll); 2576 netif_napi_set_irq(&c->napi, irq); 2577 2578 err = mlx5e_open_queues(c, params, cparam); 2579 if (unlikely(err)) 2580 goto err_napi_del; 2581 2582 if (xsk_pool) { 2583 mlx5e_build_xsk_param(xsk_pool, &xsk); 2584 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c); 2585 if (unlikely(err)) 2586 goto err_close_queues; 2587 } 2588 2589 *cp = c; 2590 2591 return 0; 2592 2593err_close_queues: 2594 mlx5e_close_queues(c); 2595 2596err_napi_del: 2597 netif_napi_del(&c->napi); 2598 2599 kvfree(c); 2600 2601 return err; 2602} 2603 2604static void mlx5e_activate_channel(struct mlx5e_channel *c) 2605{ 2606 int tc; 2607 2608 napi_enable(&c->napi); 2609 2610 for (tc = 0; tc < c->num_tc; tc++) 2611 mlx5e_activate_txqsq(&c->sq[tc]); 2612 mlx5e_activate_icosq(&c->icosq); 2613 mlx5e_activate_icosq(&c->async_icosq); 2614 2615 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2616 mlx5e_activate_xsk(c); 2617 else 2618 mlx5e_activate_rq(&c->rq); 2619 2620 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, &c->napi); 2621} 2622 2623static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 2624{ 2625 int tc; 2626 2627 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, NULL); 2628 2629 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2630 mlx5e_deactivate_xsk(c); 2631 else 2632 mlx5e_deactivate_rq(&c->rq); 2633 2634 mlx5e_deactivate_icosq(&c->async_icosq); 2635 mlx5e_deactivate_icosq(&c->icosq); 2636 for (tc = 0; tc < c->num_tc; tc++) 2637 mlx5e_deactivate_txqsq(&c->sq[tc]); 2638 mlx5e_qos_deactivate_queues(c); 2639 2640 napi_disable(&c->napi); 2641} 2642 2643static void mlx5e_close_channel(struct mlx5e_channel *c) 2644{ 2645 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2646 mlx5e_close_xsk(c); 2647 mlx5e_close_queues(c); 2648 mlx5e_qos_close_queues(c); 2649 netif_napi_del(&c->napi); 2650 2651 kvfree(c); 2652} 2653 2654int mlx5e_open_channels(struct mlx5e_priv *priv, 2655 struct mlx5e_channels *chs) 2656{ 2657 struct mlx5e_channel_param *cparam; 2658 int err = -ENOMEM; 2659 int i; 2660 2661 chs->num = chs->params.num_channels; 2662 2663 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL); 2664 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL); 2665 if (!chs->c || !cparam) 2666 goto err_free; 2667 2668 err = mlx5e_build_channel_param(priv->mdev, &chs->params, cparam); 2669 if (err) 2670 goto err_free; 2671 2672 for (i = 0; i < chs->num; i++) { 2673 struct xsk_buff_pool *xsk_pool = NULL; 2674 2675 if (chs->params.xdp_prog) 2676 xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i); 2677 2678 err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]); 2679 if (err) 2680 goto err_close_channels; 2681 } 2682 2683 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) { 2684 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp); 2685 if (err) 2686 goto err_close_channels; 2687 } 2688 2689 if (priv->htb) { 2690 err = mlx5e_qos_open_queues(priv, chs); 2691 if (err) 2692 goto err_close_ptp; 2693 } 2694 2695 mlx5e_health_channels_update(priv); 2696 kvfree(cparam); 2697 return 0; 2698 2699err_close_ptp: 2700 if (chs->ptp) 2701 mlx5e_ptp_close(chs->ptp); 2702 2703err_close_channels: 2704 for (i--; i >= 0; i--) 2705 mlx5e_close_channel(chs->c[i]); 2706 2707err_free: 2708 kfree(chs->c); 2709 kvfree(cparam); 2710 chs->num = 0; 2711 return err; 2712} 2713 2714static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 2715{ 2716 int i; 2717 2718 for (i = 0; i < chs->num; i++) 2719 mlx5e_activate_channel(chs->c[i]); 2720 2721 if (priv->htb) 2722 mlx5e_qos_activate_queues(priv); 2723 2724 for (i = 0; i < chs->num; i++) 2725 mlx5e_trigger_napi_icosq(chs->c[i]); 2726 2727 if (chs->ptp) 2728 mlx5e_ptp_activate_channel(chs->ptp); 2729} 2730 2731static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 2732{ 2733 int err = 0; 2734 int i; 2735 2736 for (i = 0; i < chs->num; i++) { 2737 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 2738 struct mlx5e_channel *c = chs->c[i]; 2739 2740 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2741 continue; 2742 2743 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout); 2744 2745 /* Don't wait on the XSK RQ, because the newer xdpsock sample 2746 * doesn't provide any Fill Ring entries at the setup stage. 2747 */ 2748 } 2749 2750 return err ? -ETIMEDOUT : 0; 2751} 2752 2753static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 2754{ 2755 int i; 2756 2757 if (chs->ptp) 2758 mlx5e_ptp_deactivate_channel(chs->ptp); 2759 2760 for (i = 0; i < chs->num; i++) 2761 mlx5e_deactivate_channel(chs->c[i]); 2762} 2763 2764void mlx5e_close_channels(struct mlx5e_channels *chs) 2765{ 2766 int i; 2767 2768 ASSERT_RTNL(); 2769 if (chs->ptp) { 2770 mlx5e_ptp_close(chs->ptp); 2771 chs->ptp = NULL; 2772 } 2773 for (i = 0; i < chs->num; i++) 2774 mlx5e_close_channel(chs->c[i]); 2775 2776 kfree(chs->c); 2777 chs->num = 0; 2778} 2779 2780static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv) 2781{ 2782 struct mlx5e_rx_res *res = priv->rx_res; 2783 2784 return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge); 2785} 2786 2787static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge); 2788 2789static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 2790 struct mlx5e_params *params, u16 mtu) 2791{ 2792 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 2793 int err; 2794 2795 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 2796 if (err) 2797 return err; 2798 2799 /* Update vport context MTU */ 2800 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 2801 return 0; 2802} 2803 2804static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 2805 struct mlx5e_params *params, u16 *mtu) 2806{ 2807 u16 hw_mtu = 0; 2808 int err; 2809 2810 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2811 if (err || !hw_mtu) /* fallback to port oper mtu */ 2812 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 2813 2814 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 2815} 2816 2817int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 2818{ 2819 struct mlx5e_params *params = &priv->channels.params; 2820 struct net_device *netdev = priv->netdev; 2821 struct mlx5_core_dev *mdev = priv->mdev; 2822 u16 mtu; 2823 int err; 2824 2825 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 2826 if (err) 2827 return err; 2828 2829 mlx5e_query_mtu(mdev, params, &mtu); 2830 if (mtu != params->sw_mtu) 2831 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 2832 __func__, mtu, params->sw_mtu); 2833 2834 params->sw_mtu = mtu; 2835 return 0; 2836} 2837 2838MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu); 2839 2840void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 2841{ 2842 struct mlx5e_params *params = &priv->channels.params; 2843 struct net_device *netdev = priv->netdev; 2844 struct mlx5_core_dev *mdev = priv->mdev; 2845 u16 max_mtu; 2846 2847 /* MTU range: 68 - hw-specific max */ 2848 netdev->min_mtu = ETH_MIN_MTU; 2849 2850 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 2851 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 2852 ETH_MAX_MTU); 2853} 2854 2855static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, 2856 struct netdev_tc_txq *tc_to_txq) 2857{ 2858 int tc, err; 2859 2860 netdev_reset_tc(netdev); 2861 2862 if (ntc == 1) 2863 return 0; 2864 2865 err = netdev_set_num_tc(netdev, ntc); 2866 if (err) { 2867 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc); 2868 return err; 2869 } 2870 2871 for (tc = 0; tc < ntc; tc++) { 2872 u16 count, offset; 2873 2874 count = tc_to_txq[tc].count; 2875 offset = tc_to_txq[tc].offset; 2876 netdev_set_tc_queue(netdev, tc, count, offset); 2877 } 2878 2879 return 0; 2880} 2881 2882int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) 2883{ 2884 int nch, ntc, num_txqs, err; 2885 int qos_queues = 0; 2886 2887 if (priv->htb) 2888 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); 2889 2890 nch = priv->channels.params.num_channels; 2891 ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); 2892 num_txqs = nch * ntc + qos_queues; 2893 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)) 2894 num_txqs += ntc; 2895 2896 netdev_dbg(priv->netdev, "Setting num_txqs %d\n", num_txqs); 2897 err = netif_set_real_num_tx_queues(priv->netdev, num_txqs); 2898 if (err) 2899 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err); 2900 2901 return err; 2902} 2903 2904static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) 2905{ 2906 struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq; 2907 struct net_device *netdev = priv->netdev; 2908 int old_num_txqs, old_ntc; 2909 int nch, ntc; 2910 int err; 2911 int i; 2912 2913 old_num_txqs = netdev->real_num_tx_queues; 2914 old_ntc = netdev->num_tc ? : 1; 2915 for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++) 2916 old_tc_to_txq[i] = netdev->tc_to_txq[i]; 2917 2918 nch = priv->channels.params.num_channels; 2919 ntc = priv->channels.params.mqprio.num_tc; 2920 tc_to_txq = priv->channels.params.mqprio.tc_to_txq; 2921 2922 err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq); 2923 if (err) 2924 goto err_out; 2925 err = mlx5e_update_tx_netdev_queues(priv); 2926 if (err) 2927 goto err_tcs; 2928 err = netif_set_real_num_rx_queues(netdev, nch); 2929 if (err) { 2930 netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); 2931 goto err_txqs; 2932 } 2933 2934 return 0; 2935 2936err_txqs: 2937 /* netif_set_real_num_rx_queues could fail only when nch increased. Only 2938 * one of nch and ntc is changed in this function. That means, the call 2939 * to netif_set_real_num_tx_queues below should not fail, because it 2940 * decreases the number of TX queues. 2941 */ 2942 WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs)); 2943 2944err_tcs: 2945 WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc, 2946 old_tc_to_txq)); 2947err_out: 2948 return err; 2949} 2950 2951static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_netdev_queues); 2952 2953static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, 2954 struct mlx5e_params *params) 2955{ 2956 int ix; 2957 2958 for (ix = 0; ix < params->num_channels; ix++) { 2959 int num_comp_vectors, irq, vec_ix; 2960 struct mlx5_core_dev *mdev; 2961 2962 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 2963 num_comp_vectors = mlx5_comp_vectors_max(mdev); 2964 cpumask_clear(priv->scratchpad.cpumask); 2965 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 2966 2967 for (irq = vec_ix; irq < num_comp_vectors; irq += params->num_channels) { 2968 int cpu = mlx5_comp_vector_get_cpu(mdev, irq); 2969 2970 cpumask_set_cpu(cpu, priv->scratchpad.cpumask); 2971 } 2972 2973 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix); 2974 } 2975} 2976 2977static int mlx5e_num_channels_changed(struct mlx5e_priv *priv) 2978{ 2979 u16 count = priv->channels.params.num_channels; 2980 int err; 2981 2982 err = mlx5e_update_netdev_queues(priv); 2983 if (err) 2984 return err; 2985 2986 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params); 2987 2988 /* This function may be called on attach, before priv->rx_res is created. */ 2989 if (priv->rx_res) { 2990 mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count); 2991 2992 if (!netif_is_rxfh_configured(priv->netdev)) 2993 mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count); 2994 } 2995 2996 return 0; 2997} 2998 2999MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed); 3000 3001static void mlx5e_build_txq_maps(struct mlx5e_priv *priv) 3002{ 3003 int i, ch, tc, num_tc; 3004 3005 ch = priv->channels.num; 3006 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 3007 3008 for (i = 0; i < ch; i++) { 3009 for (tc = 0; tc < num_tc; tc++) { 3010 struct mlx5e_channel *c = priv->channels.c[i]; 3011 struct mlx5e_txqsq *sq = &c->sq[tc]; 3012 3013 priv->txq2sq[sq->txq_ix] = sq; 3014 } 3015 } 3016 3017 if (!priv->channels.ptp) 3018 goto out; 3019 3020 if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state)) 3021 goto out; 3022 3023 for (tc = 0; tc < num_tc; tc++) { 3024 struct mlx5e_ptp *c = priv->channels.ptp; 3025 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq; 3026 3027 priv->txq2sq[sq->txq_ix] = sq; 3028 } 3029 3030out: 3031 /* Make the change to txq2sq visible before the queue is started. 3032 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 3033 * which pairs with this barrier. 3034 */ 3035 smp_wmb(); 3036} 3037 3038void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 3039{ 3040 mlx5e_build_txq_maps(priv); 3041 mlx5e_activate_channels(priv, &priv->channels); 3042 mlx5e_xdp_tx_enable(priv); 3043 3044 /* dev_watchdog() wants all TX queues to be started when the carrier is 3045 * OK, including the ones in range real_num_tx_queues..num_tx_queues-1. 3046 * Make it happy to avoid TX timeout false alarms. 3047 */ 3048 netif_tx_start_all_queues(priv->netdev); 3049 3050 if (mlx5e_is_vport_rep(priv)) 3051 mlx5e_rep_activate_channels(priv); 3052 3053 set_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3054 3055 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 3056 3057 if (priv->rx_res) 3058 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels); 3059} 3060 3061static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv) 3062{ 3063 WARN_ON_ONCE(test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)); 3064 if (current_work() != &priv->tx_timeout_work) 3065 cancel_work_sync(&priv->tx_timeout_work); 3066} 3067 3068void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 3069{ 3070 if (priv->rx_res) 3071 mlx5e_rx_res_channels_deactivate(priv->rx_res); 3072 3073 clear_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3074 mlx5e_cancel_tx_timeout_work(priv); 3075 3076 if (mlx5e_is_vport_rep(priv)) 3077 mlx5e_rep_deactivate_channels(priv); 3078 3079 /* The results of ndo_select_queue are unreliable, while netdev config 3080 * is being changed (real_num_tx_queues, num_tc). Stop all queues to 3081 * prevent ndo_start_xmit from being called, so that it can assume that 3082 * the selected queue is always valid. 3083 */ 3084 netif_tx_disable(priv->netdev); 3085 3086 mlx5e_xdp_tx_disable(priv); 3087 mlx5e_deactivate_channels(&priv->channels); 3088} 3089 3090static int mlx5e_switch_priv_params(struct mlx5e_priv *priv, 3091 struct mlx5e_params *new_params, 3092 mlx5e_fp_preactivate preactivate, 3093 void *context) 3094{ 3095 struct mlx5e_params old_params; 3096 3097 old_params = priv->channels.params; 3098 priv->channels.params = *new_params; 3099 3100 if (preactivate) { 3101 int err; 3102 3103 err = preactivate(priv, context); 3104 if (err) { 3105 priv->channels.params = old_params; 3106 return err; 3107 } 3108 } 3109 3110 return 0; 3111} 3112 3113static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 3114 struct mlx5e_channels *new_chs, 3115 mlx5e_fp_preactivate preactivate, 3116 void *context) 3117{ 3118 struct net_device *netdev = priv->netdev; 3119 struct mlx5e_channels old_chs; 3120 int carrier_ok; 3121 int err = 0; 3122 3123 carrier_ok = netif_carrier_ok(netdev); 3124 netif_carrier_off(netdev); 3125 3126 mlx5e_deactivate_priv_channels(priv); 3127 3128 old_chs = priv->channels; 3129 priv->channels = *new_chs; 3130 3131 /* New channels are ready to roll, call the preactivate hook if needed 3132 * to modify HW settings or update kernel parameters. 3133 */ 3134 if (preactivate) { 3135 err = preactivate(priv, context); 3136 if (err) { 3137 priv->channels = old_chs; 3138 goto out; 3139 } 3140 } 3141 3142 mlx5e_close_channels(&old_chs); 3143 priv->profile->update_rx(priv); 3144 3145 mlx5e_selq_apply(&priv->selq); 3146out: 3147 mlx5e_activate_priv_channels(priv); 3148 3149 /* return carrier back if needed */ 3150 if (carrier_ok) 3151 netif_carrier_on(netdev); 3152 3153 return err; 3154} 3155 3156int mlx5e_safe_switch_params(struct mlx5e_priv *priv, 3157 struct mlx5e_params *params, 3158 mlx5e_fp_preactivate preactivate, 3159 void *context, bool reset) 3160{ 3161 struct mlx5e_channels *new_chs; 3162 int err; 3163 3164 reset &= test_bit(MLX5E_STATE_OPENED, &priv->state); 3165 if (!reset) 3166 return mlx5e_switch_priv_params(priv, params, preactivate, context); 3167 3168 new_chs = kzalloc(sizeof(*new_chs), GFP_KERNEL); 3169 if (!new_chs) 3170 return -ENOMEM; 3171 new_chs->params = *params; 3172 3173 mlx5e_selq_prepare_params(&priv->selq, &new_chs->params); 3174 3175 err = mlx5e_open_channels(priv, new_chs); 3176 if (err) 3177 goto err_cancel_selq; 3178 3179 err = mlx5e_switch_priv_channels(priv, new_chs, preactivate, context); 3180 if (err) 3181 goto err_close; 3182 3183 kfree(new_chs); 3184 return 0; 3185 3186err_close: 3187 mlx5e_close_channels(new_chs); 3188 3189err_cancel_selq: 3190 mlx5e_selq_cancel(&priv->selq); 3191 kfree(new_chs); 3192 return err; 3193} 3194 3195int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 3196{ 3197 return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true); 3198} 3199 3200void mlx5e_timestamp_init(struct mlx5e_priv *priv) 3201{ 3202 priv->tstamp.tx_type = HWTSTAMP_TX_OFF; 3203 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE; 3204} 3205 3206static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev, 3207 enum mlx5_port_status state) 3208{ 3209 struct mlx5_eswitch *esw = mdev->priv.eswitch; 3210 int vport_admin_state; 3211 3212 mlx5_set_port_admin_status(mdev, state); 3213 3214 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS || 3215 !MLX5_CAP_GEN(mdev, uplink_follow)) 3216 return; 3217 3218 if (state == MLX5_PORT_UP) 3219 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; 3220 else 3221 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3222 3223 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); 3224} 3225 3226int mlx5e_open_locked(struct net_device *netdev) 3227{ 3228 struct mlx5e_priv *priv = netdev_priv(netdev); 3229 int err; 3230 3231 mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); 3232 3233 set_bit(MLX5E_STATE_OPENED, &priv->state); 3234 3235 err = mlx5e_open_channels(priv, &priv->channels); 3236 if (err) 3237 goto err_clear_state_opened_flag; 3238 3239 err = priv->profile->update_rx(priv); 3240 if (err) 3241 goto err_close_channels; 3242 3243 mlx5e_selq_apply(&priv->selq); 3244 mlx5e_activate_priv_channels(priv); 3245 mlx5e_apply_traps(priv, true); 3246 if (priv->profile->update_carrier) 3247 priv->profile->update_carrier(priv); 3248 3249 mlx5e_queue_update_stats(priv); 3250 return 0; 3251 3252err_close_channels: 3253 mlx5e_close_channels(&priv->channels); 3254err_clear_state_opened_flag: 3255 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3256 mlx5e_selq_cancel(&priv->selq); 3257 return err; 3258} 3259 3260int mlx5e_open(struct net_device *netdev) 3261{ 3262 struct mlx5e_priv *priv = netdev_priv(netdev); 3263 int err; 3264 3265 mutex_lock(&priv->state_lock); 3266 err = mlx5e_open_locked(netdev); 3267 if (!err) 3268 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP); 3269 mutex_unlock(&priv->state_lock); 3270 3271 return err; 3272} 3273 3274int mlx5e_close_locked(struct net_device *netdev) 3275{ 3276 struct mlx5e_priv *priv = netdev_priv(netdev); 3277 3278 /* May already be CLOSED in case a previous configuration operation 3279 * (e.g RX/TX queue size change) that involves close&open failed. 3280 */ 3281 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3282 return 0; 3283 3284 mlx5e_apply_traps(priv, false); 3285 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3286 3287 netif_carrier_off(priv->netdev); 3288 mlx5e_deactivate_priv_channels(priv); 3289 mlx5e_close_channels(&priv->channels); 3290 3291 return 0; 3292} 3293 3294int mlx5e_close(struct net_device *netdev) 3295{ 3296 struct mlx5e_priv *priv = netdev_priv(netdev); 3297 int err; 3298 3299 if (!netif_device_present(netdev)) 3300 return -ENODEV; 3301 3302 mutex_lock(&priv->state_lock); 3303 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN); 3304 err = mlx5e_close_locked(netdev); 3305 mutex_unlock(&priv->state_lock); 3306 3307 return err; 3308} 3309 3310static void mlx5e_free_drop_rq(struct mlx5e_rq *rq) 3311{ 3312 mlx5_wq_destroy(&rq->wq_ctrl); 3313} 3314 3315static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 3316 struct mlx5e_rq *rq, 3317 struct mlx5e_rq_param *param) 3318{ 3319 void *rqc = param->rqc; 3320 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 3321 int err; 3322 3323 param->wq.db_numa_node = param->wq.buf_numa_node; 3324 3325 err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, 3326 &rq->wq_ctrl); 3327 if (err) 3328 return err; 3329 3330 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 3331 xdp_rxq_info_unused(&rq->xdp_rxq); 3332 3333 rq->mdev = mdev; 3334 3335 return 0; 3336} 3337 3338static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv, 3339 struct mlx5e_cq *cq, 3340 struct mlx5e_cq_param *param) 3341{ 3342 struct mlx5_core_dev *mdev = priv->mdev; 3343 3344 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3345 param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3346 3347 return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, param, cq); 3348} 3349 3350int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 3351 struct mlx5e_rq *drop_rq) 3352{ 3353 struct mlx5_core_dev *mdev = priv->mdev; 3354 struct mlx5e_cq_param cq_param = {}; 3355 struct mlx5e_rq_param rq_param = {}; 3356 struct mlx5e_cq *cq = &drop_rq->cq; 3357 int err; 3358 3359 mlx5e_build_drop_rq_param(mdev, &rq_param); 3360 3361 err = mlx5e_alloc_drop_cq(priv, cq, &cq_param); 3362 if (err) 3363 return err; 3364 3365 err = mlx5e_create_cq(cq, &cq_param); 3366 if (err) 3367 goto err_free_cq; 3368 3369 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3370 if (err) 3371 goto err_destroy_cq; 3372 3373 err = mlx5e_create_rq(drop_rq, &rq_param, priv->drop_rq_q_counter); 3374 if (err) 3375 goto err_free_rq; 3376 3377 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3378 if (err) 3379 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3380 3381 return 0; 3382 3383err_free_rq: 3384 mlx5e_free_drop_rq(drop_rq); 3385 3386err_destroy_cq: 3387 mlx5e_destroy_cq(cq); 3388 3389err_free_cq: 3390 mlx5e_free_cq(cq); 3391 3392 return err; 3393} 3394 3395void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3396{ 3397 mlx5e_destroy_rq(drop_rq); 3398 mlx5e_free_drop_rq(drop_rq); 3399 mlx5e_destroy_cq(&drop_rq->cq); 3400 mlx5e_free_cq(&drop_rq->cq); 3401} 3402 3403static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3404{ 3405 if (priv->mqprio_rl) { 3406 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3407 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3408 priv->mqprio_rl = NULL; 3409 } 3410 mlx5e_accel_cleanup_tx(priv); 3411} 3412 3413static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3414{ 3415 int err; 3416 int i; 3417 3418 for (i = 0; i < chs->num; i++) { 3419 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3420 if (err) 3421 return err; 3422 } 3423 if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state)) 3424 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd); 3425 3426 return 0; 3427} 3428 3429static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3430 int ntc, int nch) 3431{ 3432 int tc; 3433 3434 memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE); 3435 3436 /* Map netdev TCs to offset 0. 3437 * We have our own UP to TXQ mapping for DCB mode of QoS 3438 */ 3439 for (tc = 0; tc < ntc; tc++) { 3440 tc_to_txq[tc] = (struct netdev_tc_txq) { 3441 .count = nch, 3442 .offset = 0, 3443 }; 3444 } 3445} 3446 3447static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3448 struct tc_mqprio_qopt *qopt) 3449{ 3450 int tc; 3451 3452 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3453 tc_to_txq[tc] = (struct netdev_tc_txq) { 3454 .count = qopt->count[tc], 3455 .offset = qopt->offset[tc], 3456 }; 3457 } 3458} 3459 3460static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) 3461{ 3462 params->mqprio.mode = TC_MQPRIO_MODE_DCB; 3463 params->mqprio.num_tc = num_tc; 3464 mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, 3465 params->num_channels); 3466} 3467 3468static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, 3469 struct mlx5e_mqprio_rl *rl) 3470{ 3471 int tc; 3472 3473 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3474 u32 hw_id = 0; 3475 3476 if (rl) 3477 mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); 3478 params->mqprio.channel.hw_id[tc] = hw_id; 3479 } 3480} 3481 3482static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, 3483 struct tc_mqprio_qopt_offload *mqprio, 3484 struct mlx5e_mqprio_rl *rl) 3485{ 3486 int tc; 3487 3488 params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; 3489 params->mqprio.num_tc = mqprio->qopt.num_tc; 3490 3491 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 3492 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; 3493 3494 mlx5e_mqprio_rl_update_params(params, rl); 3495 mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); 3496} 3497 3498static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) 3499{ 3500 mlx5e_params_mqprio_dcb_set(params, 1); 3501} 3502 3503static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, 3504 struct tc_mqprio_qopt *mqprio) 3505{ 3506 struct mlx5e_params new_params; 3507 u8 tc = mqprio->num_tc; 3508 int err; 3509 3510 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3511 3512 if (tc && tc != MLX5_MAX_NUM_TC) 3513 return -EINVAL; 3514 3515 new_params = priv->channels.params; 3516 mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1); 3517 3518 err = mlx5e_safe_switch_params(priv, &new_params, 3519 mlx5e_num_channels_changed_ctx, NULL, true); 3520 3521 if (!err && priv->mqprio_rl) { 3522 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3523 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3524 priv->mqprio_rl = NULL; 3525 } 3526 3527 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3528 mlx5e_get_dcb_num_tc(&priv->channels.params)); 3529 return err; 3530} 3531 3532static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, 3533 struct tc_mqprio_qopt_offload *mqprio) 3534{ 3535 struct net_device *netdev = priv->netdev; 3536 struct mlx5e_ptp *ptp_channel; 3537 int agg_count = 0; 3538 int i; 3539 3540 ptp_channel = priv->channels.ptp; 3541 if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) { 3542 netdev_err(netdev, 3543 "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n"); 3544 return -EINVAL; 3545 } 3546 3547 if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 || 3548 mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC) 3549 return -EINVAL; 3550 3551 for (i = 0; i < mqprio->qopt.num_tc; i++) { 3552 if (!mqprio->qopt.count[i]) { 3553 netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i); 3554 return -EINVAL; 3555 } 3556 if (mqprio->min_rate[i]) { 3557 netdev_err(netdev, "Min tx rate is not supported\n"); 3558 return -EINVAL; 3559 } 3560 3561 if (mqprio->max_rate[i]) { 3562 int err; 3563 3564 err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]); 3565 if (err) 3566 return err; 3567 } 3568 3569 if (mqprio->qopt.offset[i] != agg_count) { 3570 netdev_err(netdev, "Discontinuous queues config is not supported\n"); 3571 return -EINVAL; 3572 } 3573 agg_count += mqprio->qopt.count[i]; 3574 } 3575 3576 if (priv->channels.params.num_channels != agg_count) { 3577 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n", 3578 agg_count, priv->channels.params.num_channels); 3579 return -EINVAL; 3580 } 3581 3582 return 0; 3583} 3584 3585static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) 3586{ 3587 int tc; 3588 3589 for (tc = 0; tc < num_tc; tc++) 3590 if (max_rate[tc]) 3591 return true; 3592 return false; 3593} 3594 3595static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, 3596 u8 num_tc, u64 max_rate[]) 3597{ 3598 struct mlx5e_mqprio_rl *rl; 3599 int err; 3600 3601 if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) 3602 return NULL; 3603 3604 rl = mlx5e_mqprio_rl_alloc(); 3605 if (!rl) 3606 return ERR_PTR(-ENOMEM); 3607 3608 err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); 3609 if (err) { 3610 mlx5e_mqprio_rl_free(rl); 3611 return ERR_PTR(err); 3612 } 3613 3614 return rl; 3615} 3616 3617static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, 3618 struct tc_mqprio_qopt_offload *mqprio) 3619{ 3620 mlx5e_fp_preactivate preactivate; 3621 struct mlx5e_params new_params; 3622 struct mlx5e_mqprio_rl *rl; 3623 bool nch_changed; 3624 int err; 3625 3626 err = mlx5e_mqprio_channel_validate(priv, mqprio); 3627 if (err) 3628 return err; 3629 3630 rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); 3631 if (IS_ERR(rl)) 3632 return PTR_ERR(rl); 3633 3634 new_params = priv->channels.params; 3635 mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); 3636 3637 nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1; 3638 preactivate = nch_changed ? mlx5e_num_channels_changed_ctx : 3639 mlx5e_update_netdev_queues_ctx; 3640 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true); 3641 if (err) { 3642 if (rl) { 3643 mlx5e_mqprio_rl_cleanup(rl); 3644 mlx5e_mqprio_rl_free(rl); 3645 } 3646 return err; 3647 } 3648 3649 if (priv->mqprio_rl) { 3650 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3651 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3652 } 3653 priv->mqprio_rl = rl; 3654 3655 return 0; 3656} 3657 3658static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, 3659 struct tc_mqprio_qopt_offload *mqprio) 3660{ 3661 /* MQPRIO is another toplevel qdisc that can't be attached 3662 * simultaneously with the offloaded HTB. 3663 */ 3664 if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq))) 3665 return -EINVAL; 3666 3667 switch (mqprio->mode) { 3668 case TC_MQPRIO_MODE_DCB: 3669 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt); 3670 case TC_MQPRIO_MODE_CHANNEL: 3671 return mlx5e_setup_tc_mqprio_channel(priv, mqprio); 3672 default: 3673 return -EOPNOTSUPP; 3674 } 3675} 3676 3677static LIST_HEAD(mlx5e_block_cb_list); 3678 3679static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 3680 void *type_data) 3681{ 3682 struct mlx5e_priv *priv = netdev_priv(dev); 3683 bool tc_unbind = false; 3684 int err; 3685 3686 if (type == TC_SETUP_BLOCK && 3687 ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND) 3688 tc_unbind = true; 3689 3690 if (!netif_device_present(dev) && !tc_unbind) 3691 return -ENODEV; 3692 3693 switch (type) { 3694 case TC_SETUP_BLOCK: { 3695 struct flow_block_offload *f = type_data; 3696 3697 f->unlocked_driver_cb = true; 3698 return flow_block_cb_setup_simple(type_data, 3699 &mlx5e_block_cb_list, 3700 mlx5e_setup_tc_block_cb, 3701 priv, priv, true); 3702 } 3703 case TC_SETUP_QDISC_MQPRIO: 3704 mutex_lock(&priv->state_lock); 3705 err = mlx5e_setup_tc_mqprio(priv, type_data); 3706 mutex_unlock(&priv->state_lock); 3707 return err; 3708 case TC_SETUP_QDISC_HTB: 3709 mutex_lock(&priv->state_lock); 3710 err = mlx5e_htb_setup_tc(priv, type_data); 3711 mutex_unlock(&priv->state_lock); 3712 return err; 3713 default: 3714 return -EOPNOTSUPP; 3715 } 3716} 3717 3718void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 3719{ 3720 int i; 3721 3722 for (i = 0; i < priv->stats_nch; i++) { 3723 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 3724 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; 3725 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 3726 int j; 3727 3728 s->rx_packets += rq_stats->packets + xskrq_stats->packets; 3729 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes; 3730 s->multicast += rq_stats->mcast_packets + xskrq_stats->mcast_packets; 3731 3732 for (j = 0; j < priv->max_opened_tc; j++) { 3733 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 3734 3735 s->tx_packets += sq_stats->packets; 3736 s->tx_bytes += sq_stats->bytes; 3737 s->tx_dropped += sq_stats->dropped; 3738 } 3739 } 3740 if (priv->tx_ptp_opened) { 3741 for (i = 0; i < priv->max_opened_tc; i++) { 3742 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i]; 3743 3744 s->tx_packets += sq_stats->packets; 3745 s->tx_bytes += sq_stats->bytes; 3746 s->tx_dropped += sq_stats->dropped; 3747 } 3748 } 3749 if (priv->rx_ptp_opened) { 3750 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 3751 3752 s->rx_packets += rq_stats->packets; 3753 s->rx_bytes += rq_stats->bytes; 3754 s->multicast += rq_stats->mcast_packets; 3755 } 3756} 3757 3758void 3759mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 3760{ 3761 struct mlx5e_priv *priv = netdev_priv(dev); 3762 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 3763 3764 if (!netif_device_present(dev)) 3765 return; 3766 3767 /* In switchdev mode, monitor counters doesn't monitor 3768 * rx/tx stats of 802_3. The update stats mechanism 3769 * should keep the 802_3 layout counters updated 3770 */ 3771 if (!mlx5e_monitor_counter_supported(priv) || 3772 mlx5e_is_uplink_rep(priv)) { 3773 /* update HW stats in background for next time */ 3774 mlx5e_queue_update_stats(priv); 3775 } 3776 3777 if (mlx5e_is_uplink_rep(priv)) { 3778 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 3779 3780 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 3781 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 3782 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 3783 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 3784 3785 /* vport multicast also counts packets that are dropped due to steering 3786 * or rx out of buffer 3787 */ 3788 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 3789 } else { 3790 mlx5e_fold_sw_stats64(priv, stats); 3791 } 3792 3793 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer; 3794 3795 stats->rx_length_errors = 3796 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 3797 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 3798 PPORT_802_3_GET(pstats, a_frame_too_long_errors) + 3799 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small); 3800 stats->rx_crc_errors = 3801 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 3802 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors); 3803 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards); 3804 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors + 3805 stats->rx_frame_errors; 3806 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; 3807} 3808 3809static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) 3810{ 3811 if (mlx5e_is_uplink_rep(priv)) 3812 return; /* no rx mode for uplink rep */ 3813 3814 queue_work(priv->wq, &priv->set_rx_mode_work); 3815} 3816 3817static void mlx5e_set_rx_mode(struct net_device *dev) 3818{ 3819 struct mlx5e_priv *priv = netdev_priv(dev); 3820 3821 mlx5e_nic_set_rx_mode(priv); 3822} 3823 3824static int mlx5e_set_mac(struct net_device *netdev, void *addr) 3825{ 3826 struct mlx5e_priv *priv = netdev_priv(netdev); 3827 struct sockaddr *saddr = addr; 3828 3829 if (!is_valid_ether_addr(saddr->sa_data)) 3830 return -EADDRNOTAVAIL; 3831 3832 netif_addr_lock_bh(netdev); 3833 eth_hw_addr_set(netdev, saddr->sa_data); 3834 netif_addr_unlock_bh(netdev); 3835 3836 mlx5e_nic_set_rx_mode(priv); 3837 3838 return 0; 3839} 3840 3841#define MLX5E_SET_FEATURE(features, feature, enable) \ 3842 do { \ 3843 if (enable) \ 3844 *features |= feature; \ 3845 else \ 3846 *features &= ~feature; \ 3847 } while (0) 3848 3849typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 3850 3851static int set_feature_lro(struct net_device *netdev, bool enable) 3852{ 3853 struct mlx5e_priv *priv = netdev_priv(netdev); 3854 struct mlx5_core_dev *mdev = priv->mdev; 3855 struct mlx5e_params *cur_params; 3856 struct mlx5e_params new_params; 3857 bool reset = true; 3858 int err = 0; 3859 3860 mutex_lock(&priv->state_lock); 3861 3862 cur_params = &priv->channels.params; 3863 new_params = *cur_params; 3864 3865 if (enable) 3866 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO; 3867 else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO) 3868 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 3869 else 3870 goto out; 3871 3872 if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO && 3873 new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) { 3874 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 3875 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) == 3876 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL)) 3877 reset = false; 3878 } 3879 } 3880 3881 err = mlx5e_safe_switch_params(priv, &new_params, 3882 mlx5e_modify_tirs_packet_merge_ctx, NULL, reset); 3883out: 3884 mutex_unlock(&priv->state_lock); 3885 return err; 3886} 3887 3888static int set_feature_hw_gro(struct net_device *netdev, bool enable) 3889{ 3890 struct mlx5e_priv *priv = netdev_priv(netdev); 3891 struct mlx5e_params new_params; 3892 bool reset = true; 3893 int err = 0; 3894 3895 mutex_lock(&priv->state_lock); 3896 new_params = priv->channels.params; 3897 3898 if (enable) { 3899 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO; 3900 new_params.packet_merge.shampo.match_criteria_type = 3901 MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED; 3902 new_params.packet_merge.shampo.alignment_granularity = 3903 MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE; 3904 } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { 3905 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 3906 } else { 3907 goto out; 3908 } 3909 3910 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 3911out: 3912 mutex_unlock(&priv->state_lock); 3913 return err; 3914} 3915 3916static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 3917{ 3918 struct mlx5e_priv *priv = netdev_priv(netdev); 3919 3920 if (enable) 3921 mlx5e_enable_cvlan_filter(priv->fs, 3922 !!(priv->netdev->flags & IFF_PROMISC)); 3923 else 3924 mlx5e_disable_cvlan_filter(priv->fs, 3925 !!(priv->netdev->flags & IFF_PROMISC)); 3926 3927 return 0; 3928} 3929 3930static int set_feature_hw_tc(struct net_device *netdev, bool enable) 3931{ 3932 struct mlx5e_priv *priv = netdev_priv(netdev); 3933 int err = 0; 3934 3935#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 3936 int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) : 3937 MLX5_TC_FLAG(NIC_OFFLOAD); 3938 if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) { 3939 netdev_err(netdev, 3940 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 3941 return -EINVAL; 3942 } 3943#endif 3944 3945 mutex_lock(&priv->state_lock); 3946 if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { 3947 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); 3948 err = -EINVAL; 3949 } 3950 mutex_unlock(&priv->state_lock); 3951 3952 return err; 3953} 3954 3955static int set_feature_rx_all(struct net_device *netdev, bool enable) 3956{ 3957 struct mlx5e_priv *priv = netdev_priv(netdev); 3958 struct mlx5_core_dev *mdev = priv->mdev; 3959 3960 return mlx5_set_port_fcs(mdev, !enable); 3961} 3962 3963static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable) 3964{ 3965 u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {}; 3966 bool supported, curr_state; 3967 int err; 3968 3969 if (!MLX5_CAP_GEN(mdev, ports_check)) 3970 return 0; 3971 3972 err = mlx5_query_ports_check(mdev, in, sizeof(in)); 3973 if (err) 3974 return err; 3975 3976 supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap); 3977 curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc); 3978 3979 if (!supported || enable == curr_state) 3980 return 0; 3981 3982 MLX5_SET(pcmr_reg, in, local_port, 1); 3983 MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable); 3984 3985 return mlx5_set_ports_check(mdev, in, sizeof(in)); 3986} 3987 3988static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx) 3989{ 3990 struct mlx5_core_dev *mdev = priv->mdev; 3991 bool enable = *(bool *)ctx; 3992 3993 return mlx5e_set_rx_port_ts(mdev, enable); 3994} 3995 3996static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 3997{ 3998 struct mlx5e_priv *priv = netdev_priv(netdev); 3999 struct mlx5e_channels *chs = &priv->channels; 4000 struct mlx5e_params new_params; 4001 int err; 4002 bool rx_ts_over_crc = !enable; 4003 4004 mutex_lock(&priv->state_lock); 4005 4006 new_params = chs->params; 4007 new_params.scatter_fcs_en = enable; 4008 err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_set_rx_port_ts_wrap, 4009 &rx_ts_over_crc, true); 4010 mutex_unlock(&priv->state_lock); 4011 return err; 4012} 4013 4014static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 4015{ 4016 struct mlx5e_priv *priv = netdev_priv(netdev); 4017 int err = 0; 4018 4019 mutex_lock(&priv->state_lock); 4020 4021 mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable); 4022 priv->channels.params.vlan_strip_disable = !enable; 4023 4024 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4025 goto unlock; 4026 4027 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 4028 if (err) { 4029 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable); 4030 priv->channels.params.vlan_strip_disable = enable; 4031 } 4032unlock: 4033 mutex_unlock(&priv->state_lock); 4034 4035 return err; 4036} 4037 4038int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 4039{ 4040 struct mlx5e_priv *priv = netdev_priv(dev); 4041 struct mlx5e_flow_steering *fs = priv->fs; 4042 4043 if (mlx5e_is_uplink_rep(priv)) 4044 return 0; /* no vlan table for uplink rep */ 4045 4046 return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid); 4047} 4048 4049int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 4050{ 4051 struct mlx5e_priv *priv = netdev_priv(dev); 4052 struct mlx5e_flow_steering *fs = priv->fs; 4053 4054 if (mlx5e_is_uplink_rep(priv)) 4055 return 0; /* no vlan table for uplink rep */ 4056 4057 return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid); 4058} 4059 4060#ifdef CONFIG_MLX5_EN_ARFS 4061static int set_feature_arfs(struct net_device *netdev, bool enable) 4062{ 4063 struct mlx5e_priv *priv = netdev_priv(netdev); 4064 int err; 4065 4066 if (enable) 4067 err = mlx5e_arfs_enable(priv->fs); 4068 else 4069 err = mlx5e_arfs_disable(priv->fs); 4070 4071 return err; 4072} 4073#endif 4074 4075static int mlx5e_handle_feature(struct net_device *netdev, 4076 netdev_features_t *features, 4077 netdev_features_t feature, 4078 mlx5e_feature_handler feature_handler) 4079{ 4080 netdev_features_t changes = *features ^ netdev->features; 4081 bool enable = !!(*features & feature); 4082 int err; 4083 4084 if (!(changes & feature)) 4085 return 0; 4086 4087 err = feature_handler(netdev, enable); 4088 if (err) { 4089 MLX5E_SET_FEATURE(features, feature, !enable); 4090 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 4091 enable ? "Enable" : "Disable", &feature, err); 4092 return err; 4093 } 4094 4095 return 0; 4096} 4097 4098void mlx5e_set_xdp_feature(struct net_device *netdev) 4099{ 4100 struct mlx5e_priv *priv = netdev_priv(netdev); 4101 struct mlx5e_params *params = &priv->channels.params; 4102 xdp_features_t val; 4103 4104 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) { 4105 xdp_clear_features_flag(netdev); 4106 return; 4107 } 4108 4109 val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4110 NETDEV_XDP_ACT_XSK_ZEROCOPY | 4111 NETDEV_XDP_ACT_RX_SG | 4112 NETDEV_XDP_ACT_NDO_XMIT | 4113 NETDEV_XDP_ACT_NDO_XMIT_SG; 4114 xdp_set_features_flag(netdev, val); 4115} 4116 4117int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 4118{ 4119 netdev_features_t oper_features = features; 4120 int err = 0; 4121 4122#define MLX5E_HANDLE_FEATURE(feature, handler) \ 4123 mlx5e_handle_feature(netdev, &oper_features, feature, handler) 4124 4125 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4126 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4127 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 4128 set_feature_cvlan_filter); 4129 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc); 4130 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 4131 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4132 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 4133#ifdef CONFIG_MLX5_EN_ARFS 4134 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 4135#endif 4136 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx); 4137 4138 if (err) { 4139 netdev->features = oper_features; 4140 return -EINVAL; 4141 } 4142 4143 /* update XDP supported features */ 4144 mlx5e_set_xdp_feature(netdev); 4145 4146 return 0; 4147} 4148 4149static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev, 4150 netdev_features_t features) 4151{ 4152 features &= ~NETIF_F_HW_TLS_RX; 4153 if (netdev->features & NETIF_F_HW_TLS_RX) 4154 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n"); 4155 4156 features &= ~NETIF_F_HW_TLS_TX; 4157 if (netdev->features & NETIF_F_HW_TLS_TX) 4158 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n"); 4159 4160 features &= ~NETIF_F_NTUPLE; 4161 if (netdev->features & NETIF_F_NTUPLE) 4162 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n"); 4163 4164 features &= ~NETIF_F_GRO_HW; 4165 if (netdev->features & NETIF_F_GRO_HW) 4166 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n"); 4167 4168 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4169 if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) 4170 netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n"); 4171 4172 return features; 4173} 4174 4175static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 4176 netdev_features_t features) 4177{ 4178 struct mlx5e_priv *priv = netdev_priv(netdev); 4179 struct mlx5e_vlan_table *vlan; 4180 struct mlx5e_params *params; 4181 4182 if (!netif_device_present(netdev)) 4183 return features; 4184 4185 vlan = mlx5e_fs_get_vlan(priv->fs); 4186 mutex_lock(&priv->state_lock); 4187 params = &priv->channels.params; 4188 if (!vlan || 4189 !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) { 4190 /* HW strips the outer C-tag header, this is a problem 4191 * for S-tag traffic. 4192 */ 4193 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4194 if (!params->vlan_strip_disable) 4195 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 4196 } 4197 4198 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 4199 if (features & NETIF_F_LRO) { 4200 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 4201 features &= ~NETIF_F_LRO; 4202 } 4203 if (features & NETIF_F_GRO_HW) { 4204 netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n"); 4205 features &= ~NETIF_F_GRO_HW; 4206 } 4207 } 4208 4209 if (params->xdp_prog) { 4210 if (features & NETIF_F_LRO) { 4211 netdev_warn(netdev, "LRO is incompatible with XDP\n"); 4212 features &= ~NETIF_F_LRO; 4213 } 4214 if (features & NETIF_F_GRO_HW) { 4215 netdev_warn(netdev, "HW GRO is incompatible with XDP\n"); 4216 features &= ~NETIF_F_GRO_HW; 4217 } 4218 } 4219 4220 if (priv->xsk.refcnt) { 4221 if (features & NETIF_F_LRO) { 4222 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", 4223 priv->xsk.refcnt); 4224 features &= ~NETIF_F_LRO; 4225 } 4226 if (features & NETIF_F_GRO_HW) { 4227 netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n", 4228 priv->xsk.refcnt); 4229 features &= ~NETIF_F_GRO_HW; 4230 } 4231 } 4232 4233 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 4234 features &= ~NETIF_F_RXHASH; 4235 if (netdev->features & NETIF_F_RXHASH) 4236 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 4237 4238 if (features & NETIF_F_GRO_HW) { 4239 netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n"); 4240 features &= ~NETIF_F_GRO_HW; 4241 } 4242 } 4243 4244 if (mlx5e_is_uplink_rep(priv)) { 4245 features = mlx5e_fix_uplink_rep_features(netdev, features); 4246 features |= NETIF_F_NETNS_LOCAL; 4247 } else { 4248 features &= ~NETIF_F_NETNS_LOCAL; 4249 } 4250 4251 mutex_unlock(&priv->state_lock); 4252 4253 return features; 4254} 4255 4256static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, 4257 struct mlx5e_channels *chs, 4258 struct mlx5e_params *new_params, 4259 struct mlx5_core_dev *mdev) 4260{ 4261 u16 ix; 4262 4263 for (ix = 0; ix < chs->params.num_channels; ix++) { 4264 struct xsk_buff_pool *xsk_pool = 4265 mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix); 4266 struct mlx5e_xsk_param xsk; 4267 int max_xdp_mtu; 4268 4269 if (!xsk_pool) 4270 continue; 4271 4272 mlx5e_build_xsk_param(xsk_pool, &xsk); 4273 max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk); 4274 4275 /* Validate XSK params and XDP MTU in advance */ 4276 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) || 4277 new_params->sw_mtu > max_xdp_mtu) { 4278 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); 4279 int max_mtu_frame, max_mtu_page, max_mtu; 4280 4281 /* Two criteria must be met: 4282 * 1. HW MTU + all headrooms <= XSK frame size. 4283 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. 4284 */ 4285 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); 4286 max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0)); 4287 max_mtu = min3(max_mtu_frame, max_mtu_page, max_xdp_mtu); 4288 4289 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u or its redirection XDP program. Try MTU <= %d\n", 4290 new_params->sw_mtu, ix, max_mtu); 4291 return false; 4292 } 4293 } 4294 4295 return true; 4296} 4297 4298static bool mlx5e_params_validate_xdp(struct net_device *netdev, 4299 struct mlx5_core_dev *mdev, 4300 struct mlx5e_params *params) 4301{ 4302 bool is_linear; 4303 4304 /* No XSK params: AF_XDP can't be enabled yet at the point of setting 4305 * the XDP program. 4306 */ 4307 is_linear = params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC ? 4308 mlx5e_rx_is_linear_skb(mdev, params, NULL) : 4309 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL); 4310 4311 if (!is_linear) { 4312 if (!params->xdp_prog->aux->xdp_has_frags) { 4313 netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n", 4314 params->sw_mtu, 4315 mlx5e_xdp_max_mtu(params, NULL)); 4316 return false; 4317 } 4318 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && 4319 !mlx5e_verify_params_rx_mpwqe_strides(mdev, params, NULL)) { 4320 netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n", 4321 params->sw_mtu, 4322 mlx5e_xdp_max_mtu(params, NULL)); 4323 return false; 4324 } 4325 } 4326 4327 return true; 4328} 4329 4330int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 4331 mlx5e_fp_preactivate preactivate) 4332{ 4333 struct mlx5e_priv *priv = netdev_priv(netdev); 4334 struct mlx5e_params new_params; 4335 struct mlx5e_params *params; 4336 bool reset = true; 4337 int err = 0; 4338 4339 mutex_lock(&priv->state_lock); 4340 4341 params = &priv->channels.params; 4342 4343 new_params = *params; 4344 new_params.sw_mtu = new_mtu; 4345 err = mlx5e_validate_params(priv->mdev, &new_params); 4346 if (err) 4347 goto out; 4348 4349 if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev, 4350 &new_params)) { 4351 err = -EINVAL; 4352 goto out; 4353 } 4354 4355 if (priv->xsk.refcnt && 4356 !mlx5e_xsk_validate_mtu(netdev, &priv->channels, 4357 &new_params, priv->mdev)) { 4358 err = -EINVAL; 4359 goto out; 4360 } 4361 4362 if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO) 4363 reset = false; 4364 4365 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && 4366 params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) { 4367 bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL); 4368 bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, 4369 &new_params, NULL); 4370 u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL); 4371 u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL); 4372 4373 /* Always reset in linear mode - hw_mtu is used in data path. 4374 * Check that the mode was non-linear and didn't change. 4375 * If XSK is active, XSK RQs are linear. 4376 * Reset if the RQ size changed, even if it's non-linear. 4377 */ 4378 if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt && 4379 sz_old == sz_new) 4380 reset = false; 4381 } 4382 4383 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, reset); 4384 4385out: 4386 netdev->mtu = params->sw_mtu; 4387 mutex_unlock(&priv->state_lock); 4388 return err; 4389} 4390 4391static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 4392{ 4393 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx); 4394} 4395 4396int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx) 4397{ 4398 bool set = *(bool *)ctx; 4399 4400 return mlx5e_ptp_rx_manage_fs(priv, set); 4401} 4402 4403static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter) 4404{ 4405 bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4406 int err; 4407 4408 if (!rx_filter) 4409 /* Reset CQE compression to Admin default */ 4410 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false); 4411 4412 if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4413 return 0; 4414 4415 /* Disable CQE compression */ 4416 netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); 4417 err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true); 4418 if (err) 4419 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 4420 4421 return err; 4422} 4423 4424static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx) 4425{ 4426 struct mlx5e_params new_params; 4427 4428 if (ptp_rx == priv->channels.params.ptp_rx) 4429 return 0; 4430 4431 new_params = priv->channels.params; 4432 new_params.ptp_rx = ptp_rx; 4433 return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx, 4434 &new_params.ptp_rx, true); 4435} 4436 4437int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) 4438{ 4439 struct hwtstamp_config config; 4440 bool rx_cqe_compress_def; 4441 bool ptp_rx; 4442 int err; 4443 4444 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 4445 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) 4446 return -EOPNOTSUPP; 4447 4448 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 4449 return -EFAULT; 4450 4451 /* TX HW timestamp */ 4452 switch (config.tx_type) { 4453 case HWTSTAMP_TX_OFF: 4454 case HWTSTAMP_TX_ON: 4455 break; 4456 default: 4457 return -ERANGE; 4458 } 4459 4460 mutex_lock(&priv->state_lock); 4461 rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4462 4463 /* RX HW timestamp */ 4464 switch (config.rx_filter) { 4465 case HWTSTAMP_FILTER_NONE: 4466 ptp_rx = false; 4467 break; 4468 case HWTSTAMP_FILTER_ALL: 4469 case HWTSTAMP_FILTER_SOME: 4470 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 4471 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 4472 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 4473 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 4474 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 4475 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 4476 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 4477 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 4478 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 4479 case HWTSTAMP_FILTER_PTP_V2_EVENT: 4480 case HWTSTAMP_FILTER_PTP_V2_SYNC: 4481 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 4482 case HWTSTAMP_FILTER_NTP_ALL: 4483 config.rx_filter = HWTSTAMP_FILTER_ALL; 4484 /* ptp_rx is set if both HW TS is set and CQE 4485 * compression is set 4486 */ 4487 ptp_rx = rx_cqe_compress_def; 4488 break; 4489 default: 4490 err = -ERANGE; 4491 goto err_unlock; 4492 } 4493 4494 if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) 4495 err = mlx5e_hwstamp_config_no_ptp_rx(priv, 4496 config.rx_filter != HWTSTAMP_FILTER_NONE); 4497 else 4498 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx); 4499 if (err) 4500 goto err_unlock; 4501 4502 memcpy(&priv->tstamp, &config, sizeof(config)); 4503 mutex_unlock(&priv->state_lock); 4504 4505 /* might need to fix some features */ 4506 netdev_update_features(priv->netdev); 4507 4508 return copy_to_user(ifr->ifr_data, &config, 4509 sizeof(config)) ? -EFAULT : 0; 4510err_unlock: 4511 mutex_unlock(&priv->state_lock); 4512 return err; 4513} 4514 4515int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr) 4516{ 4517 struct hwtstamp_config *cfg = &priv->tstamp; 4518 4519 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 4520 return -EOPNOTSUPP; 4521 4522 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0; 4523} 4524 4525static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 4526{ 4527 struct mlx5e_priv *priv = netdev_priv(dev); 4528 4529 switch (cmd) { 4530 case SIOCSHWTSTAMP: 4531 return mlx5e_hwstamp_set(priv, ifr); 4532 case SIOCGHWTSTAMP: 4533 return mlx5e_hwstamp_get(priv, ifr); 4534 default: 4535 return -EOPNOTSUPP; 4536 } 4537} 4538 4539#ifdef CONFIG_MLX5_ESWITCH 4540int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 4541{ 4542 struct mlx5e_priv *priv = netdev_priv(dev); 4543 struct mlx5_core_dev *mdev = priv->mdev; 4544 4545 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 4546} 4547 4548static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 4549 __be16 vlan_proto) 4550{ 4551 struct mlx5e_priv *priv = netdev_priv(dev); 4552 struct mlx5_core_dev *mdev = priv->mdev; 4553 4554 if (vlan_proto != htons(ETH_P_8021Q)) 4555 return -EPROTONOSUPPORT; 4556 4557 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 4558 vlan, qos); 4559} 4560 4561static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 4562{ 4563 struct mlx5e_priv *priv = netdev_priv(dev); 4564 struct mlx5_core_dev *mdev = priv->mdev; 4565 4566 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 4567} 4568 4569static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 4570{ 4571 struct mlx5e_priv *priv = netdev_priv(dev); 4572 struct mlx5_core_dev *mdev = priv->mdev; 4573 4574 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 4575} 4576 4577int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 4578 int max_tx_rate) 4579{ 4580 struct mlx5e_priv *priv = netdev_priv(dev); 4581 struct mlx5_core_dev *mdev = priv->mdev; 4582 4583 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 4584 max_tx_rate, min_tx_rate); 4585} 4586 4587static int mlx5_vport_link2ifla(u8 esw_link) 4588{ 4589 switch (esw_link) { 4590 case MLX5_VPORT_ADMIN_STATE_DOWN: 4591 return IFLA_VF_LINK_STATE_DISABLE; 4592 case MLX5_VPORT_ADMIN_STATE_UP: 4593 return IFLA_VF_LINK_STATE_ENABLE; 4594 } 4595 return IFLA_VF_LINK_STATE_AUTO; 4596} 4597 4598static int mlx5_ifla_link2vport(u8 ifla_link) 4599{ 4600 switch (ifla_link) { 4601 case IFLA_VF_LINK_STATE_DISABLE: 4602 return MLX5_VPORT_ADMIN_STATE_DOWN; 4603 case IFLA_VF_LINK_STATE_ENABLE: 4604 return MLX5_VPORT_ADMIN_STATE_UP; 4605 } 4606 return MLX5_VPORT_ADMIN_STATE_AUTO; 4607} 4608 4609static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 4610 int link_state) 4611{ 4612 struct mlx5e_priv *priv = netdev_priv(dev); 4613 struct mlx5_core_dev *mdev = priv->mdev; 4614 4615 if (mlx5e_is_uplink_rep(priv)) 4616 return -EOPNOTSUPP; 4617 4618 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 4619 mlx5_ifla_link2vport(link_state)); 4620} 4621 4622int mlx5e_get_vf_config(struct net_device *dev, 4623 int vf, struct ifla_vf_info *ivi) 4624{ 4625 struct mlx5e_priv *priv = netdev_priv(dev); 4626 struct mlx5_core_dev *mdev = priv->mdev; 4627 int err; 4628 4629 if (!netif_device_present(dev)) 4630 return -EOPNOTSUPP; 4631 4632 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 4633 if (err) 4634 return err; 4635 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 4636 return 0; 4637} 4638 4639int mlx5e_get_vf_stats(struct net_device *dev, 4640 int vf, struct ifla_vf_stats *vf_stats) 4641{ 4642 struct mlx5e_priv *priv = netdev_priv(dev); 4643 struct mlx5_core_dev *mdev = priv->mdev; 4644 4645 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 4646 vf_stats); 4647} 4648 4649static bool 4650mlx5e_has_offload_stats(const struct net_device *dev, int attr_id) 4651{ 4652 struct mlx5e_priv *priv = netdev_priv(dev); 4653 4654 if (!netif_device_present(dev)) 4655 return false; 4656 4657 if (!mlx5e_is_uplink_rep(priv)) 4658 return false; 4659 4660 return mlx5e_rep_has_offload_stats(dev, attr_id); 4661} 4662 4663static int 4664mlx5e_get_offload_stats(int attr_id, const struct net_device *dev, 4665 void *sp) 4666{ 4667 struct mlx5e_priv *priv = netdev_priv(dev); 4668 4669 if (!mlx5e_is_uplink_rep(priv)) 4670 return -EOPNOTSUPP; 4671 4672 return mlx5e_rep_get_offload_stats(attr_id, dev, sp); 4673} 4674#endif 4675 4676static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type) 4677{ 4678 switch (proto_type) { 4679 case IPPROTO_GRE: 4680 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre); 4681 case IPPROTO_IPIP: 4682 case IPPROTO_IPV6: 4683 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) || 4684 MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx)); 4685 default: 4686 return false; 4687 } 4688} 4689 4690static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev, 4691 struct sk_buff *skb) 4692{ 4693 switch (skb->inner_protocol) { 4694 case htons(ETH_P_IP): 4695 case htons(ETH_P_IPV6): 4696 case htons(ETH_P_TEB): 4697 return true; 4698 case htons(ETH_P_MPLS_UC): 4699 case htons(ETH_P_MPLS_MC): 4700 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre); 4701 } 4702 return false; 4703} 4704 4705static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 4706 struct sk_buff *skb, 4707 netdev_features_t features) 4708{ 4709 unsigned int offset = 0; 4710 struct udphdr *udph; 4711 u8 proto; 4712 u16 port; 4713 4714 switch (vlan_get_protocol(skb)) { 4715 case htons(ETH_P_IP): 4716 proto = ip_hdr(skb)->protocol; 4717 break; 4718 case htons(ETH_P_IPV6): 4719 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 4720 break; 4721 default: 4722 goto out; 4723 } 4724 4725 switch (proto) { 4726 case IPPROTO_GRE: 4727 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb)) 4728 return features; 4729 break; 4730 case IPPROTO_IPIP: 4731 case IPPROTO_IPV6: 4732 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP)) 4733 return features; 4734 break; 4735 case IPPROTO_UDP: 4736 udph = udp_hdr(skb); 4737 port = be16_to_cpu(udph->dest); 4738 4739 /* Verify if UDP port is being offloaded by HW */ 4740 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 4741 return features; 4742 4743#if IS_ENABLED(CONFIG_GENEVE) 4744 /* Support Geneve offload for default UDP port */ 4745 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 4746 return features; 4747#endif 4748 break; 4749#ifdef CONFIG_MLX5_EN_IPSEC 4750 case IPPROTO_ESP: 4751 return mlx5e_ipsec_feature_check(skb, features); 4752#endif 4753 } 4754 4755out: 4756 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */ 4757 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4758} 4759 4760netdev_features_t mlx5e_features_check(struct sk_buff *skb, 4761 struct net_device *netdev, 4762 netdev_features_t features) 4763{ 4764 struct mlx5e_priv *priv = netdev_priv(netdev); 4765 4766 features = vlan_features_check(skb, features); 4767 features = vxlan_features_check(skb, features); 4768 4769 /* Validate if the tunneled packet is being offloaded by HW */ 4770 if (skb->encapsulation && 4771 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 4772 return mlx5e_tunnel_features_check(priv, skb, features); 4773 4774 return features; 4775} 4776 4777static void mlx5e_tx_timeout_work(struct work_struct *work) 4778{ 4779 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 4780 tx_timeout_work); 4781 struct net_device *netdev = priv->netdev; 4782 int i; 4783 4784 /* Take rtnl_lock to ensure no change in netdev->real_num_tx_queues 4785 * through this flow. However, channel closing flows have to wait for 4786 * this work to finish while holding rtnl lock too. So either get the 4787 * lock or find that channels are being closed for other reason and 4788 * this work is not relevant anymore. 4789 */ 4790 while (!rtnl_trylock()) { 4791 if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) 4792 return; 4793 msleep(20); 4794 } 4795 4796 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4797 goto unlock; 4798 4799 for (i = 0; i < netdev->real_num_tx_queues; i++) { 4800 struct netdev_queue *dev_queue = 4801 netdev_get_tx_queue(netdev, i); 4802 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 4803 4804 if (!netif_xmit_stopped(dev_queue)) 4805 continue; 4806 4807 if (mlx5e_reporter_tx_timeout(sq)) 4808 /* break if tried to reopened channels */ 4809 break; 4810 } 4811 4812unlock: 4813 rtnl_unlock(); 4814} 4815 4816static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) 4817{ 4818 struct mlx5e_priv *priv = netdev_priv(dev); 4819 4820 netdev_err(dev, "TX timeout detected\n"); 4821 queue_work(priv->wq, &priv->tx_timeout_work); 4822} 4823 4824static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev, 4825 struct mlx5e_params *params) 4826{ 4827 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) { 4828 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n"); 4829 return -EINVAL; 4830 } 4831 4832 if (!mlx5e_params_validate_xdp(netdev, mdev, params)) 4833 return -EINVAL; 4834 4835 return 0; 4836} 4837 4838static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog) 4839{ 4840 struct bpf_prog *old_prog; 4841 4842 old_prog = rcu_replace_pointer(rq->xdp_prog, prog, 4843 lockdep_is_held(&rq->priv->state_lock)); 4844 if (old_prog) 4845 bpf_prog_put(old_prog); 4846} 4847 4848static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 4849{ 4850 struct mlx5e_priv *priv = netdev_priv(netdev); 4851 struct mlx5e_params new_params; 4852 struct bpf_prog *old_prog; 4853 int err = 0; 4854 bool reset; 4855 int i; 4856 4857 mutex_lock(&priv->state_lock); 4858 4859 new_params = priv->channels.params; 4860 new_params.xdp_prog = prog; 4861 4862 if (prog) { 4863 err = mlx5e_xdp_allowed(netdev, priv->mdev, &new_params); 4864 if (err) 4865 goto unlock; 4866 } 4867 4868 /* no need for full reset when exchanging programs */ 4869 reset = (!priv->channels.params.xdp_prog || !prog); 4870 4871 old_prog = priv->channels.params.xdp_prog; 4872 4873 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 4874 if (err) 4875 goto unlock; 4876 4877 if (old_prog) 4878 bpf_prog_put(old_prog); 4879 4880 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset) 4881 goto unlock; 4882 4883 /* exchanging programs w/o reset, we update ref counts on behalf 4884 * of the channels RQs here. 4885 */ 4886 bpf_prog_add(prog, priv->channels.num); 4887 for (i = 0; i < priv->channels.num; i++) { 4888 struct mlx5e_channel *c = priv->channels.c[i]; 4889 4890 mlx5e_rq_replace_xdp_prog(&c->rq, prog); 4891 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) { 4892 bpf_prog_inc(prog); 4893 mlx5e_rq_replace_xdp_prog(&c->xskrq, prog); 4894 } 4895 } 4896 4897unlock: 4898 mutex_unlock(&priv->state_lock); 4899 4900 /* Need to fix some features. */ 4901 if (!err) 4902 netdev_update_features(netdev); 4903 4904 return err; 4905} 4906 4907static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4908{ 4909 switch (xdp->command) { 4910 case XDP_SETUP_PROG: 4911 return mlx5e_xdp_set(dev, xdp->prog); 4912 case XDP_SETUP_XSK_POOL: 4913 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool, 4914 xdp->xsk.queue_id); 4915 default: 4916 return -EINVAL; 4917 } 4918} 4919 4920#ifdef CONFIG_MLX5_ESWITCH 4921static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4922 struct net_device *dev, u32 filter_mask, 4923 int nlflags) 4924{ 4925 struct mlx5e_priv *priv = netdev_priv(dev); 4926 struct mlx5_core_dev *mdev = priv->mdev; 4927 u8 mode, setting; 4928 int err; 4929 4930 err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting); 4931 if (err) 4932 return err; 4933 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 4934 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4935 mode, 4936 0, 0, nlflags, filter_mask, NULL); 4937} 4938 4939static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4940 u16 flags, struct netlink_ext_ack *extack) 4941{ 4942 struct mlx5e_priv *priv = netdev_priv(dev); 4943 struct mlx5_core_dev *mdev = priv->mdev; 4944 struct nlattr *attr, *br_spec; 4945 u16 mode = BRIDGE_MODE_UNDEF; 4946 u8 setting; 4947 int rem; 4948 4949 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4950 if (!br_spec) 4951 return -EINVAL; 4952 4953 nla_for_each_nested(attr, br_spec, rem) { 4954 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4955 continue; 4956 4957 mode = nla_get_u16(attr); 4958 if (mode > BRIDGE_MODE_VEPA) 4959 return -EINVAL; 4960 4961 break; 4962 } 4963 4964 if (mode == BRIDGE_MODE_UNDEF) 4965 return -EINVAL; 4966 4967 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 4968 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 4969} 4970#endif 4971 4972const struct net_device_ops mlx5e_netdev_ops = { 4973 .ndo_open = mlx5e_open, 4974 .ndo_stop = mlx5e_close, 4975 .ndo_start_xmit = mlx5e_xmit, 4976 .ndo_setup_tc = mlx5e_setup_tc, 4977 .ndo_select_queue = mlx5e_select_queue, 4978 .ndo_get_stats64 = mlx5e_get_stats, 4979 .ndo_set_rx_mode = mlx5e_set_rx_mode, 4980 .ndo_set_mac_address = mlx5e_set_mac, 4981 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 4982 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 4983 .ndo_set_features = mlx5e_set_features, 4984 .ndo_fix_features = mlx5e_fix_features, 4985 .ndo_change_mtu = mlx5e_change_nic_mtu, 4986 .ndo_eth_ioctl = mlx5e_ioctl, 4987 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 4988 .ndo_features_check = mlx5e_features_check, 4989 .ndo_tx_timeout = mlx5e_tx_timeout, 4990 .ndo_bpf = mlx5e_xdp, 4991 .ndo_xdp_xmit = mlx5e_xdp_xmit, 4992 .ndo_xsk_wakeup = mlx5e_xsk_wakeup, 4993#ifdef CONFIG_MLX5_EN_ARFS 4994 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 4995#endif 4996#ifdef CONFIG_MLX5_ESWITCH 4997 .ndo_bridge_setlink = mlx5e_bridge_setlink, 4998 .ndo_bridge_getlink = mlx5e_bridge_getlink, 4999 5000 /* SRIOV E-Switch NDOs */ 5001 .ndo_set_vf_mac = mlx5e_set_vf_mac, 5002 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 5003 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 5004 .ndo_set_vf_trust = mlx5e_set_vf_trust, 5005 .ndo_set_vf_rate = mlx5e_set_vf_rate, 5006 .ndo_get_vf_config = mlx5e_get_vf_config, 5007 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 5008 .ndo_get_vf_stats = mlx5e_get_vf_stats, 5009 .ndo_has_offload_stats = mlx5e_has_offload_stats, 5010 .ndo_get_offload_stats = mlx5e_get_offload_stats, 5011#endif 5012}; 5013 5014static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) 5015{ 5016 int i; 5017 5018 /* The supported periods are organized in ascending order */ 5019 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++) 5020 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout) 5021 break; 5022 5023 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]); 5024} 5025 5026void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu) 5027{ 5028 struct mlx5e_params *params = &priv->channels.params; 5029 struct mlx5_core_dev *mdev = priv->mdev; 5030 u8 rx_cq_period_mode; 5031 5032 params->sw_mtu = mtu; 5033 params->hard_mtu = MLX5E_ETH_HARD_MTU; 5034 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2, 5035 priv->max_nch); 5036 mlx5e_params_mqprio_reset(params); 5037 5038 /* SQ */ 5039 params->log_sq_size = is_kdump_kernel() ? 5040 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 5041 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 5042 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5043 5044 /* XDP SQ */ 5045 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5046 5047 /* set CQE compression */ 5048 params->rx_cqe_compress_def = false; 5049 if (MLX5_CAP_GEN(mdev, cqe_compression) && 5050 MLX5_CAP_GEN(mdev, vport_group_manager)) 5051 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 5052 5053 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 5054 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 5055 5056 /* RQ */ 5057 mlx5e_build_rq_params(mdev, params); 5058 5059 params->terminate_lkey_be = mlx5_core_get_terminate_scatter_list_mkey(mdev); 5060 5061 params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 5062 5063 /* CQ moderation params */ 5064 rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 5065 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 5066 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 5067 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 5068 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 5069 mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode); 5070 mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); 5071 5072 /* TX inline */ 5073 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 5074 5075 /* AF_XDP */ 5076 params->xsk = xsk; 5077 5078 /* Do not update netdev->features directly in here 5079 * on mlx5e_attach_netdev() we will call mlx5e_update_features() 5080 * To update netdev->features please modify mlx5e_fix_features() 5081 */ 5082} 5083 5084static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 5085{ 5086 struct mlx5e_priv *priv = netdev_priv(netdev); 5087 u8 addr[ETH_ALEN]; 5088 5089 mlx5_query_mac_address(priv->mdev, addr); 5090 if (is_zero_ether_addr(addr) && 5091 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) { 5092 eth_hw_addr_random(netdev); 5093 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 5094 return; 5095 } 5096 5097 eth_hw_addr_set(netdev, addr); 5098} 5099 5100static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table, 5101 unsigned int entry, struct udp_tunnel_info *ti) 5102{ 5103 struct mlx5e_priv *priv = netdev_priv(netdev); 5104 5105 return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port)); 5106} 5107 5108static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table, 5109 unsigned int entry, struct udp_tunnel_info *ti) 5110{ 5111 struct mlx5e_priv *priv = netdev_priv(netdev); 5112 5113 return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port)); 5114} 5115 5116void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv) 5117{ 5118 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 5119 return; 5120 5121 priv->nic_info.set_port = mlx5e_vxlan_set_port; 5122 priv->nic_info.unset_port = mlx5e_vxlan_unset_port; 5123 priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 5124 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN; 5125 priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; 5126 /* Don't count the space hard-coded to the IANA port */ 5127 priv->nic_info.tables[0].n_entries = 5128 mlx5_vxlan_max_udp_ports(priv->mdev) - 1; 5129 5130 priv->netdev->udp_tunnel_nic_info = &priv->nic_info; 5131} 5132 5133static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev) 5134{ 5135 int tt; 5136 5137 for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) { 5138 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt))) 5139 return true; 5140 } 5141 return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)); 5142} 5143 5144static void mlx5e_build_nic_netdev(struct net_device *netdev) 5145{ 5146 struct mlx5e_priv *priv = netdev_priv(netdev); 5147 struct mlx5_core_dev *mdev = priv->mdev; 5148 bool fcs_supported; 5149 bool fcs_enabled; 5150 5151 SET_NETDEV_DEV(netdev, mdev->device); 5152 5153 netdev->netdev_ops = &mlx5e_netdev_ops; 5154 netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops; 5155 netdev->xsk_tx_metadata_ops = &mlx5e_xsk_tx_metadata_ops; 5156 5157 mlx5e_dcbnl_build_netdev(netdev); 5158 5159 netdev->watchdog_timeo = 15 * HZ; 5160 5161 netdev->ethtool_ops = &mlx5e_ethtool_ops; 5162 5163 netdev->vlan_features |= NETIF_F_SG; 5164 netdev->vlan_features |= NETIF_F_HW_CSUM; 5165 netdev->vlan_features |= NETIF_F_HW_MACSEC; 5166 netdev->vlan_features |= NETIF_F_GRO; 5167 netdev->vlan_features |= NETIF_F_TSO; 5168 netdev->vlan_features |= NETIF_F_TSO6; 5169 netdev->vlan_features |= NETIF_F_RXCSUM; 5170 netdev->vlan_features |= NETIF_F_RXHASH; 5171 netdev->vlan_features |= NETIF_F_GSO_PARTIAL; 5172 5173 netdev->mpls_features |= NETIF_F_SG; 5174 netdev->mpls_features |= NETIF_F_HW_CSUM; 5175 netdev->mpls_features |= NETIF_F_TSO; 5176 netdev->mpls_features |= NETIF_F_TSO6; 5177 5178 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 5179 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 5180 5181 /* Tunneled LRO is not supported in the driver, and the same RQs are 5182 * shared between inner and outer TIRs, so the driver can't disable LRO 5183 * for inner TIRs while having it enabled for outer TIRs. Due to this, 5184 * block LRO altogether if the firmware declares tunneled LRO support. 5185 */ 5186 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 5187 !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && 5188 !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && 5189 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5190 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5191 netdev->vlan_features |= NETIF_F_LRO; 5192 5193 netdev->hw_features = netdev->vlan_features; 5194 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 5195 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 5196 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5197 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 5198 5199 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) { 5200 netdev->hw_enc_features |= NETIF_F_HW_CSUM; 5201 netdev->hw_enc_features |= NETIF_F_TSO; 5202 netdev->hw_enc_features |= NETIF_F_TSO6; 5203 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 5204 } 5205 5206 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 5207 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 5208 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5209 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 5210 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5211 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 5212 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL | 5213 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5214 } 5215 5216 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) { 5217 netdev->hw_features |= NETIF_F_GSO_GRE | 5218 NETIF_F_GSO_GRE_CSUM; 5219 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 5220 NETIF_F_GSO_GRE_CSUM; 5221 netdev->gso_partial_features |= NETIF_F_GSO_GRE | 5222 NETIF_F_GSO_GRE_CSUM; 5223 } 5224 5225 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) { 5226 netdev->hw_features |= NETIF_F_GSO_IPXIP4 | 5227 NETIF_F_GSO_IPXIP6; 5228 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 | 5229 NETIF_F_GSO_IPXIP6; 5230 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 | 5231 NETIF_F_GSO_IPXIP6; 5232 } 5233 5234 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 5235 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 5236 5237 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 5238 5239 if (fcs_supported) 5240 netdev->hw_features |= NETIF_F_RXALL; 5241 5242 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 5243 netdev->hw_features |= NETIF_F_RXFCS; 5244 5245 if (mlx5_qos_is_supported(mdev)) 5246 netdev->hw_features |= NETIF_F_HW_TC; 5247 5248 netdev->features = netdev->hw_features; 5249 5250 /* Defaults */ 5251 if (fcs_enabled) 5252 netdev->features &= ~NETIF_F_RXALL; 5253 netdev->features &= ~NETIF_F_LRO; 5254 netdev->features &= ~NETIF_F_GRO_HW; 5255 netdev->features &= ~NETIF_F_RXFCS; 5256 5257#define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 5258 if (FT_CAP(flow_modify_en) && 5259 FT_CAP(modify_root) && 5260 FT_CAP(identified_miss_table_mode) && 5261 FT_CAP(flow_table_modify)) { 5262#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 5263 netdev->hw_features |= NETIF_F_HW_TC; 5264#endif 5265#ifdef CONFIG_MLX5_EN_ARFS 5266 netdev->hw_features |= NETIF_F_NTUPLE; 5267#endif 5268 } 5269 5270 netdev->features |= NETIF_F_HIGHDMA; 5271 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 5272 5273 netdev->priv_flags |= IFF_UNICAST_FLT; 5274 5275 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 5276 mlx5e_set_xdp_feature(netdev); 5277 mlx5e_set_netdev_dev_addr(netdev); 5278 mlx5e_macsec_build_netdev(priv); 5279 mlx5e_ipsec_build_netdev(priv); 5280 mlx5e_ktls_build_netdev(priv); 5281} 5282 5283void mlx5e_create_q_counters(struct mlx5e_priv *priv) 5284{ 5285 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 5286 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 5287 struct mlx5_core_dev *mdev = priv->mdev; 5288 struct mlx5_core_dev *pos; 5289 int err, i; 5290 5291 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 5292 5293 mlx5_sd_for_each_dev(i, mdev, pos) { 5294 err = mlx5_cmd_exec_inout(pos, alloc_q_counter, in, out); 5295 if (!err) 5296 priv->q_counter[i] = 5297 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5298 } 5299 5300 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5301 if (!err) 5302 priv->drop_rq_q_counter = 5303 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5304} 5305 5306void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 5307{ 5308 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 5309 struct mlx5_core_dev *pos; 5310 int i; 5311 5312 MLX5_SET(dealloc_q_counter_in, in, opcode, 5313 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 5314 mlx5_sd_for_each_dev(i, priv->mdev, pos) { 5315 if (priv->q_counter[i]) { 5316 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5317 priv->q_counter[i]); 5318 mlx5_cmd_exec_in(pos, dealloc_q_counter, in); 5319 } 5320 } 5321 5322 if (priv->drop_rq_q_counter) { 5323 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5324 priv->drop_rq_q_counter); 5325 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 5326 } 5327} 5328 5329static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 5330 struct net_device *netdev) 5331{ 5332 const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED; 5333 struct mlx5e_priv *priv = netdev_priv(netdev); 5334 struct mlx5e_flow_steering *fs; 5335 int err; 5336 5337 mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu); 5338 mlx5e_vxlan_set_netdev_info(priv); 5339 5340 mlx5e_timestamp_init(priv); 5341 5342 priv->dfs_root = debugfs_create_dir("nic", 5343 mlx5_debugfs_get_dev_root(mdev)); 5344 5345 fs = mlx5e_fs_init(priv->profile, mdev, 5346 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 5347 priv->dfs_root); 5348 if (!fs) { 5349 err = -ENOMEM; 5350 mlx5_core_err(mdev, "FS initialization failed, %d\n", err); 5351 debugfs_remove_recursive(priv->dfs_root); 5352 return err; 5353 } 5354 priv->fs = fs; 5355 5356 err = mlx5e_ktls_init(priv); 5357 if (err) 5358 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 5359 5360 mlx5e_health_create_reporters(priv); 5361 5362 /* If netdev is already registered (e.g. move from uplink to nic profile), 5363 * RTNL lock must be held before triggering netdev notifiers. 5364 */ 5365 if (take_rtnl) 5366 rtnl_lock(); 5367 5368 /* update XDP supported features */ 5369 mlx5e_set_xdp_feature(netdev); 5370 5371 if (take_rtnl) 5372 rtnl_unlock(); 5373 5374 return 0; 5375} 5376 5377static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 5378{ 5379 mlx5e_health_destroy_reporters(priv); 5380 mlx5e_ktls_cleanup(priv); 5381 mlx5e_fs_cleanup(priv->fs); 5382 debugfs_remove_recursive(priv->dfs_root); 5383 priv->fs = NULL; 5384} 5385 5386static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 5387{ 5388 struct mlx5_core_dev *mdev = priv->mdev; 5389 enum mlx5e_rx_res_features features; 5390 int err; 5391 5392 mlx5e_create_q_counters(priv); 5393 5394 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 5395 if (err) { 5396 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 5397 goto err_destroy_q_counters; 5398 } 5399 5400 features = MLX5E_RX_RES_FEATURE_PTP; 5401 if (mlx5_tunnel_inner_ft_supported(mdev)) 5402 features |= MLX5E_RX_RES_FEATURE_INNER_FT; 5403 if (mlx5_get_sd(priv->mdev)) 5404 features |= MLX5E_RX_RES_FEATURE_MULTI_VHCA; 5405 5406 priv->rx_res = mlx5e_rx_res_create(priv->mdev, features, priv->max_nch, priv->drop_rq.rqn, 5407 &priv->channels.params.packet_merge, 5408 priv->channels.params.num_channels); 5409 if (IS_ERR(priv->rx_res)) { 5410 err = PTR_ERR(priv->rx_res); 5411 priv->rx_res = NULL; 5412 mlx5_core_err(mdev, "create rx resources failed, %d\n", err); 5413 goto err_close_drop_rq; 5414 } 5415 5416 err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, 5417 priv->netdev); 5418 if (err) { 5419 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 5420 goto err_destroy_rx_res; 5421 } 5422 5423 err = mlx5e_tc_nic_init(priv); 5424 if (err) 5425 goto err_destroy_flow_steering; 5426 5427 err = mlx5e_accel_init_rx(priv); 5428 if (err) 5429 goto err_tc_nic_cleanup; 5430 5431#ifdef CONFIG_MLX5_EN_ARFS 5432 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev); 5433#endif 5434 5435 return 0; 5436 5437err_tc_nic_cleanup: 5438 mlx5e_tc_nic_cleanup(priv); 5439err_destroy_flow_steering: 5440 mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), 5441 priv->profile); 5442err_destroy_rx_res: 5443 mlx5e_rx_res_destroy(priv->rx_res); 5444 priv->rx_res = NULL; 5445err_close_drop_rq: 5446 mlx5e_close_drop_rq(&priv->drop_rq); 5447err_destroy_q_counters: 5448 mlx5e_destroy_q_counters(priv); 5449 return err; 5450} 5451 5452static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 5453{ 5454 mlx5e_accel_cleanup_rx(priv); 5455 mlx5e_tc_nic_cleanup(priv); 5456 mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), 5457 priv->profile); 5458 mlx5e_rx_res_destroy(priv->rx_res); 5459 priv->rx_res = NULL; 5460 mlx5e_close_drop_rq(&priv->drop_rq); 5461 mlx5e_destroy_q_counters(priv); 5462} 5463 5464static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) 5465{ 5466 struct mlx5e_params *params; 5467 struct mlx5e_mqprio_rl *rl; 5468 5469 params = &priv->channels.params; 5470 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) 5471 return; 5472 5473 rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, 5474 params->mqprio.channel.max_rate); 5475 if (IS_ERR(rl)) 5476 rl = NULL; 5477 priv->mqprio_rl = rl; 5478 mlx5e_mqprio_rl_update_params(params, rl); 5479} 5480 5481static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 5482{ 5483 int err; 5484 5485 err = mlx5e_accel_init_tx(priv); 5486 if (err) 5487 return err; 5488 5489 mlx5e_set_mqprio_rl(priv); 5490 mlx5e_dcbnl_initialize(priv); 5491 return 0; 5492} 5493 5494static void mlx5e_nic_enable(struct mlx5e_priv *priv) 5495{ 5496 struct net_device *netdev = priv->netdev; 5497 struct mlx5_core_dev *mdev = priv->mdev; 5498 int err; 5499 5500 mlx5e_fs_init_l2_addr(priv->fs, netdev); 5501 mlx5e_ipsec_init(priv); 5502 5503 err = mlx5e_macsec_init(priv); 5504 if (err) 5505 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err); 5506 5507 /* Marking the link as currently not needed by the Driver */ 5508 if (!netif_running(netdev)) 5509 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); 5510 5511 mlx5e_set_netdev_mtu_boundaries(priv); 5512 mlx5e_set_dev_port_mtu(priv); 5513 5514 mlx5_lag_add_netdev(mdev, netdev); 5515 5516 mlx5e_enable_async_events(priv); 5517 mlx5e_enable_blocking_events(priv); 5518 if (mlx5e_monitor_counter_supported(priv)) 5519 mlx5e_monitor_counter_init(priv); 5520 5521 mlx5e_hv_vhca_stats_create(priv); 5522 if (netdev->reg_state != NETREG_REGISTERED) 5523 return; 5524 mlx5e_dcbnl_init_app(priv); 5525 5526 mlx5e_nic_set_rx_mode(priv); 5527 5528 rtnl_lock(); 5529 if (netif_running(netdev)) 5530 mlx5e_open(netdev); 5531 udp_tunnel_nic_reset_ntf(priv->netdev); 5532 netif_device_attach(netdev); 5533 rtnl_unlock(); 5534} 5535 5536static void mlx5e_nic_disable(struct mlx5e_priv *priv) 5537{ 5538 struct mlx5_core_dev *mdev = priv->mdev; 5539 5540 if (priv->netdev->reg_state == NETREG_REGISTERED) 5541 mlx5e_dcbnl_delete_app(priv); 5542 5543 rtnl_lock(); 5544 if (netif_running(priv->netdev)) 5545 mlx5e_close(priv->netdev); 5546 netif_device_detach(priv->netdev); 5547 rtnl_unlock(); 5548 5549 mlx5e_nic_set_rx_mode(priv); 5550 5551 mlx5e_hv_vhca_stats_destroy(priv); 5552 if (mlx5e_monitor_counter_supported(priv)) 5553 mlx5e_monitor_counter_cleanup(priv); 5554 5555 mlx5e_disable_blocking_events(priv); 5556 if (priv->en_trap) { 5557 mlx5e_deactivate_trap(priv); 5558 mlx5e_close_trap(priv->en_trap); 5559 priv->en_trap = NULL; 5560 } 5561 mlx5e_disable_async_events(priv); 5562 mlx5_lag_remove_netdev(mdev, priv->netdev); 5563 mlx5_vxlan_reset_to_default(mdev->vxlan); 5564 mlx5e_macsec_cleanup(priv); 5565 mlx5e_ipsec_cleanup(priv); 5566} 5567 5568int mlx5e_update_nic_rx(struct mlx5e_priv *priv) 5569{ 5570 return mlx5e_refresh_tirs(priv, false, false); 5571} 5572 5573static const struct mlx5e_profile mlx5e_nic_profile = { 5574 .init = mlx5e_nic_init, 5575 .cleanup = mlx5e_nic_cleanup, 5576 .init_rx = mlx5e_init_nic_rx, 5577 .cleanup_rx = mlx5e_cleanup_nic_rx, 5578 .init_tx = mlx5e_init_nic_tx, 5579 .cleanup_tx = mlx5e_cleanup_nic_tx, 5580 .enable = mlx5e_nic_enable, 5581 .disable = mlx5e_nic_disable, 5582 .update_rx = mlx5e_update_nic_rx, 5583 .update_stats = mlx5e_stats_update_ndo_stats, 5584 .update_carrier = mlx5e_update_carrier, 5585 .rx_handlers = &mlx5e_rx_handlers_nic, 5586 .max_tc = MLX5_MAX_NUM_TC, 5587 .stats_grps = mlx5e_nic_stats_grps, 5588 .stats_grps_num = mlx5e_nic_stats_grps_num, 5589 .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | 5590 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) | 5591 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) | 5592 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) | 5593 BIT(MLX5E_PROFILE_FEATURE_FS_TC), 5594}; 5595 5596static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev, 5597 const struct mlx5e_profile *profile) 5598{ 5599 int nch; 5600 5601 nch = mlx5e_get_max_num_channels(mdev); 5602 5603 if (profile->max_nch_limit) 5604 nch = min_t(int, nch, profile->max_nch_limit(mdev)); 5605 return nch; 5606} 5607 5608static unsigned int 5609mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev, 5610 const struct mlx5e_profile *profile) 5611 5612{ 5613 unsigned int max_nch, tmp; 5614 5615 /* core resources */ 5616 max_nch = mlx5e_profile_max_num_channels(mdev, profile); 5617 5618 /* netdev rx queues */ 5619 max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues); 5620 5621 /* netdev tx queues */ 5622 tmp = netdev->num_tx_queues; 5623 if (mlx5_qos_is_supported(mdev)) 5624 tmp -= mlx5e_qos_max_leaf_nodes(mdev); 5625 if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn)) 5626 tmp -= profile->max_tc; 5627 tmp = tmp / profile->max_tc; 5628 max_nch = min_t(unsigned int, max_nch, tmp); 5629 5630 return max_nch; 5631} 5632 5633int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev) 5634{ 5635 /* Indirect TIRS: 2 sets of TTCs (inner + outer steering) 5636 * and 1 set of direct TIRS 5637 */ 5638 return 2 * MLX5E_NUM_INDIR_TIRS 5639 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile); 5640} 5641 5642void mlx5e_set_rx_mode_work(struct work_struct *work) 5643{ 5644 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 5645 set_rx_mode_work); 5646 5647 return mlx5e_fs_set_rx_mode_work(priv->fs, priv->netdev); 5648} 5649 5650/* mlx5e generic netdev management API (move to en_common.c) */ 5651int mlx5e_priv_init(struct mlx5e_priv *priv, 5652 const struct mlx5e_profile *profile, 5653 struct net_device *netdev, 5654 struct mlx5_core_dev *mdev) 5655{ 5656 int nch, num_txqs, node; 5657 int err; 5658 5659 num_txqs = netdev->num_tx_queues; 5660 nch = mlx5e_calc_max_nch(mdev, netdev, profile); 5661 node = dev_to_node(mlx5_core_dma_dev(mdev)); 5662 5663 /* priv init */ 5664 priv->mdev = mdev; 5665 priv->netdev = netdev; 5666 priv->max_nch = nch; 5667 priv->max_opened_tc = 1; 5668 5669 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL)) 5670 return -ENOMEM; 5671 5672 mutex_init(&priv->state_lock); 5673 5674 err = mlx5e_selq_init(&priv->selq, &priv->state_lock); 5675 if (err) 5676 goto err_free_cpumask; 5677 5678 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 5679 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 5680 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 5681 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 5682 5683 priv->wq = create_singlethread_workqueue("mlx5e"); 5684 if (!priv->wq) 5685 goto err_free_selq; 5686 5687 priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node); 5688 if (!priv->txq2sq) 5689 goto err_destroy_workqueue; 5690 5691 priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node); 5692 if (!priv->tx_rates) 5693 goto err_free_txq2sq; 5694 5695 priv->channel_stats = 5696 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node); 5697 if (!priv->channel_stats) 5698 goto err_free_tx_rates; 5699 5700 return 0; 5701 5702err_free_tx_rates: 5703 kfree(priv->tx_rates); 5704err_free_txq2sq: 5705 kfree(priv->txq2sq); 5706err_destroy_workqueue: 5707 destroy_workqueue(priv->wq); 5708err_free_selq: 5709 mlx5e_selq_cleanup(&priv->selq); 5710err_free_cpumask: 5711 free_cpumask_var(priv->scratchpad.cpumask); 5712 return -ENOMEM; 5713} 5714 5715void mlx5e_priv_cleanup(struct mlx5e_priv *priv) 5716{ 5717 int i; 5718 5719 /* bail if change profile failed and also rollback failed */ 5720 if (!priv->mdev) 5721 return; 5722 5723 for (i = 0; i < priv->stats_nch; i++) 5724 kvfree(priv->channel_stats[i]); 5725 kfree(priv->channel_stats); 5726 kfree(priv->tx_rates); 5727 kfree(priv->txq2sq); 5728 destroy_workqueue(priv->wq); 5729 mlx5e_selq_cleanup(&priv->selq); 5730 free_cpumask_var(priv->scratchpad.cpumask); 5731 5732 for (i = 0; i < priv->htb_max_qos_sqs; i++) 5733 kfree(priv->htb_qos_sq_stats[i]); 5734 kvfree(priv->htb_qos_sq_stats); 5735 5736 memset(priv, 0, sizeof(*priv)); 5737} 5738 5739static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev, 5740 const struct mlx5e_profile *profile) 5741{ 5742 unsigned int nch, ptp_txqs, qos_txqs; 5743 5744 nch = mlx5e_profile_max_num_channels(mdev, profile); 5745 5746 ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) && 5747 mlx5e_profile_feature_cap(profile, PTP_TX) ? 5748 profile->max_tc : 0; 5749 5750 qos_txqs = mlx5_qos_is_supported(mdev) && 5751 mlx5e_profile_feature_cap(profile, QOS_HTB) ? 5752 mlx5e_qos_max_leaf_nodes(mdev) : 0; 5753 5754 return nch * profile->max_tc + ptp_txqs + qos_txqs; 5755} 5756 5757static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev, 5758 const struct mlx5e_profile *profile) 5759{ 5760 return mlx5e_profile_max_num_channels(mdev, profile); 5761} 5762 5763struct net_device * 5764mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile) 5765{ 5766 struct net_device *netdev; 5767 unsigned int txqs, rxqs; 5768 int err; 5769 5770 txqs = mlx5e_get_max_num_txqs(mdev, profile); 5771 rxqs = mlx5e_get_max_num_rxqs(mdev, profile); 5772 5773 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs); 5774 if (!netdev) { 5775 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 5776 return NULL; 5777 } 5778 5779 err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev); 5780 if (err) { 5781 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 5782 goto err_free_netdev; 5783 } 5784 5785 netif_carrier_off(netdev); 5786 netif_tx_disable(netdev); 5787 dev_net_set(netdev, mlx5_core_net(mdev)); 5788 5789 return netdev; 5790 5791err_free_netdev: 5792 free_netdev(netdev); 5793 5794 return NULL; 5795} 5796 5797static void mlx5e_update_features(struct net_device *netdev) 5798{ 5799 if (netdev->reg_state != NETREG_REGISTERED) 5800 return; /* features will be updated on netdev registration */ 5801 5802 rtnl_lock(); 5803 netdev_update_features(netdev); 5804 rtnl_unlock(); 5805} 5806 5807static void mlx5e_reset_channels(struct net_device *netdev) 5808{ 5809 netdev_reset_tc(netdev); 5810} 5811 5812int mlx5e_attach_netdev(struct mlx5e_priv *priv) 5813{ 5814 const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED; 5815 const struct mlx5e_profile *profile = priv->profile; 5816 int max_nch; 5817 int err; 5818 5819 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 5820 if (priv->fs) 5821 mlx5e_fs_set_state_destroy(priv->fs, 5822 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 5823 5824 /* Validate the max_wqe_size_sq capability. */ 5825 if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) { 5826 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %u\n", 5827 mlx5e_get_max_sq_wqebbs(priv->mdev), (unsigned int)MLX5E_MAX_TX_WQEBBS); 5828 return -EIO; 5829 } 5830 5831 /* max number of channels may have changed */ 5832 max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); 5833 if (priv->channels.params.num_channels > max_nch) { 5834 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 5835 /* Reducing the number of channels - RXFH has to be reset, and 5836 * mlx5e_num_channels_changed below will build the RQT. 5837 */ 5838 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED; 5839 priv->channels.params.num_channels = max_nch; 5840 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) { 5841 mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n"); 5842 mlx5e_params_mqprio_reset(&priv->channels.params); 5843 } 5844 } 5845 if (max_nch != priv->max_nch) { 5846 mlx5_core_warn(priv->mdev, 5847 "MLX5E: Updating max number of channels from %u to %u\n", 5848 priv->max_nch, max_nch); 5849 priv->max_nch = max_nch; 5850 } 5851 5852 /* 1. Set the real number of queues in the kernel the first time. 5853 * 2. Set our default XPS cpumask. 5854 * 3. Build the RQT. 5855 * 5856 * rtnl_lock is required by netif_set_real_num_*_queues in case the 5857 * netdev has been registered by this point (if this function was called 5858 * in the reload or resume flow). 5859 */ 5860 if (take_rtnl) 5861 rtnl_lock(); 5862 err = mlx5e_num_channels_changed(priv); 5863 if (take_rtnl) 5864 rtnl_unlock(); 5865 if (err) 5866 goto out; 5867 5868 err = profile->init_tx(priv); 5869 if (err) 5870 goto out; 5871 5872 err = profile->init_rx(priv); 5873 if (err) 5874 goto err_cleanup_tx; 5875 5876 if (profile->enable) 5877 profile->enable(priv); 5878 5879 mlx5e_update_features(priv->netdev); 5880 5881 return 0; 5882 5883err_cleanup_tx: 5884 profile->cleanup_tx(priv); 5885 5886out: 5887 mlx5e_reset_channels(priv->netdev); 5888 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5889 if (priv->fs) 5890 mlx5e_fs_set_state_destroy(priv->fs, 5891 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 5892 cancel_work_sync(&priv->update_stats_work); 5893 return err; 5894} 5895 5896void mlx5e_detach_netdev(struct mlx5e_priv *priv) 5897{ 5898 const struct mlx5e_profile *profile = priv->profile; 5899 5900 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5901 if (priv->fs) 5902 mlx5e_fs_set_state_destroy(priv->fs, 5903 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 5904 5905 if (profile->disable) 5906 profile->disable(priv); 5907 flush_workqueue(priv->wq); 5908 5909 profile->cleanup_rx(priv); 5910 profile->cleanup_tx(priv); 5911 mlx5e_reset_channels(priv->netdev); 5912 cancel_work_sync(&priv->update_stats_work); 5913} 5914 5915static int 5916mlx5e_netdev_init_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 5917 const struct mlx5e_profile *new_profile, void *new_ppriv) 5918{ 5919 struct mlx5e_priv *priv = netdev_priv(netdev); 5920 int err; 5921 5922 err = mlx5e_priv_init(priv, new_profile, netdev, mdev); 5923 if (err) { 5924 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 5925 return err; 5926 } 5927 netif_carrier_off(netdev); 5928 priv->profile = new_profile; 5929 priv->ppriv = new_ppriv; 5930 err = new_profile->init(priv->mdev, priv->netdev); 5931 if (err) 5932 goto priv_cleanup; 5933 5934 return 0; 5935 5936priv_cleanup: 5937 mlx5e_priv_cleanup(priv); 5938 return err; 5939} 5940 5941static int 5942mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 5943 const struct mlx5e_profile *new_profile, void *new_ppriv) 5944{ 5945 struct mlx5e_priv *priv = netdev_priv(netdev); 5946 int err; 5947 5948 err = mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 5949 if (err) 5950 return err; 5951 5952 err = mlx5e_attach_netdev(priv); 5953 if (err) 5954 goto profile_cleanup; 5955 return err; 5956 5957profile_cleanup: 5958 new_profile->cleanup(priv); 5959 mlx5e_priv_cleanup(priv); 5960 return err; 5961} 5962 5963int mlx5e_netdev_change_profile(struct mlx5e_priv *priv, 5964 const struct mlx5e_profile *new_profile, void *new_ppriv) 5965{ 5966 const struct mlx5e_profile *orig_profile = priv->profile; 5967 struct net_device *netdev = priv->netdev; 5968 struct mlx5_core_dev *mdev = priv->mdev; 5969 void *orig_ppriv = priv->ppriv; 5970 int err, rollback_err; 5971 5972 /* cleanup old profile */ 5973 mlx5e_detach_netdev(priv); 5974 priv->profile->cleanup(priv); 5975 mlx5e_priv_cleanup(priv); 5976 5977 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 5978 mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 5979 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5980 return -EIO; 5981 } 5982 5983 err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv); 5984 if (err) { /* roll back to original profile */ 5985 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err); 5986 goto rollback; 5987 } 5988 5989 return 0; 5990 5991rollback: 5992 rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv); 5993 if (rollback_err) 5994 netdev_err(netdev, "%s: failed to rollback to orig profile, %d\n", 5995 __func__, rollback_err); 5996 return err; 5997} 5998 5999void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv) 6000{ 6001 mlx5e_netdev_change_profile(priv, &mlx5e_nic_profile, NULL); 6002} 6003 6004void mlx5e_destroy_netdev(struct mlx5e_priv *priv) 6005{ 6006 struct net_device *netdev = priv->netdev; 6007 6008 mlx5e_priv_cleanup(priv); 6009 free_netdev(netdev); 6010} 6011 6012static int _mlx5e_resume(struct auxiliary_device *adev) 6013{ 6014 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6015 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6016 struct mlx5e_priv *priv = mlx5e_dev->priv; 6017 struct net_device *netdev = priv->netdev; 6018 struct mlx5_core_dev *mdev = edev->mdev; 6019 struct mlx5_core_dev *pos, *to; 6020 int err, i; 6021 6022 if (netif_device_present(netdev)) 6023 return 0; 6024 6025 mlx5_sd_for_each_dev(i, mdev, pos) { 6026 err = mlx5e_create_mdev_resources(pos, true); 6027 if (err) 6028 goto err_destroy_mdev_res; 6029 } 6030 6031 err = mlx5e_attach_netdev(priv); 6032 if (err) 6033 goto err_destroy_mdev_res; 6034 6035 return 0; 6036 6037err_destroy_mdev_res: 6038 to = pos; 6039 mlx5_sd_for_each_dev_to(i, mdev, to, pos) 6040 mlx5e_destroy_mdev_resources(pos); 6041 return err; 6042} 6043 6044static int mlx5e_resume(struct auxiliary_device *adev) 6045{ 6046 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6047 struct mlx5_core_dev *mdev = edev->mdev; 6048 struct auxiliary_device *actual_adev; 6049 int err; 6050 6051 err = mlx5_sd_init(mdev); 6052 if (err) 6053 return err; 6054 6055 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6056 if (actual_adev) 6057 return _mlx5e_resume(actual_adev); 6058 return 0; 6059} 6060 6061static int _mlx5e_suspend(struct auxiliary_device *adev) 6062{ 6063 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6064 struct mlx5e_priv *priv = mlx5e_dev->priv; 6065 struct net_device *netdev = priv->netdev; 6066 struct mlx5_core_dev *mdev = priv->mdev; 6067 struct mlx5_core_dev *pos; 6068 int i; 6069 6070 if (!netif_device_present(netdev)) { 6071 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) 6072 mlx5_sd_for_each_dev(i, mdev, pos) 6073 mlx5e_destroy_mdev_resources(pos); 6074 return -ENODEV; 6075 } 6076 6077 mlx5e_detach_netdev(priv); 6078 mlx5_sd_for_each_dev(i, mdev, pos) 6079 mlx5e_destroy_mdev_resources(pos); 6080 6081 return 0; 6082} 6083 6084static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) 6085{ 6086 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6087 struct mlx5_core_dev *mdev = edev->mdev; 6088 struct auxiliary_device *actual_adev; 6089 int err = 0; 6090 6091 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6092 if (actual_adev) 6093 err = _mlx5e_suspend(actual_adev); 6094 6095 mlx5_sd_cleanup(mdev); 6096 return err; 6097} 6098 6099static int _mlx5e_probe(struct auxiliary_device *adev) 6100{ 6101 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6102 const struct mlx5e_profile *profile = &mlx5e_nic_profile; 6103 struct mlx5_core_dev *mdev = edev->mdev; 6104 struct mlx5e_dev *mlx5e_dev; 6105 struct net_device *netdev; 6106 struct mlx5e_priv *priv; 6107 int err; 6108 6109 mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev); 6110 if (IS_ERR(mlx5e_dev)) 6111 return PTR_ERR(mlx5e_dev); 6112 auxiliary_set_drvdata(adev, mlx5e_dev); 6113 6114 err = mlx5e_devlink_port_register(mlx5e_dev, mdev); 6115 if (err) { 6116 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); 6117 goto err_devlink_unregister; 6118 } 6119 6120 netdev = mlx5e_create_netdev(mdev, profile); 6121 if (!netdev) { 6122 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 6123 err = -ENOMEM; 6124 goto err_devlink_port_unregister; 6125 } 6126 SET_NETDEV_DEVLINK_PORT(netdev, &mlx5e_dev->dl_port); 6127 6128 mlx5e_build_nic_netdev(netdev); 6129 6130 priv = netdev_priv(netdev); 6131 mlx5e_dev->priv = priv; 6132 6133 priv->profile = profile; 6134 priv->ppriv = NULL; 6135 6136 err = profile->init(mdev, netdev); 6137 if (err) { 6138 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err); 6139 goto err_destroy_netdev; 6140 } 6141 6142 err = _mlx5e_resume(adev); 6143 if (err) { 6144 mlx5_core_err(mdev, "_mlx5e_resume failed, %d\n", err); 6145 goto err_profile_cleanup; 6146 } 6147 6148 err = register_netdev(netdev); 6149 if (err) { 6150 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 6151 goto err_resume; 6152 } 6153 6154 mlx5e_dcbnl_init_app(priv); 6155 mlx5_core_uplink_netdev_set(mdev, netdev); 6156 mlx5e_params_print_info(mdev, &priv->channels.params); 6157 return 0; 6158 6159err_resume: 6160 _mlx5e_suspend(adev); 6161err_profile_cleanup: 6162 profile->cleanup(priv); 6163err_destroy_netdev: 6164 mlx5e_destroy_netdev(priv); 6165err_devlink_port_unregister: 6166 mlx5e_devlink_port_unregister(mlx5e_dev); 6167err_devlink_unregister: 6168 mlx5e_destroy_devlink(mlx5e_dev); 6169 return err; 6170} 6171 6172static int mlx5e_probe(struct auxiliary_device *adev, 6173 const struct auxiliary_device_id *id) 6174{ 6175 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6176 struct mlx5_core_dev *mdev = edev->mdev; 6177 struct auxiliary_device *actual_adev; 6178 int err; 6179 6180 err = mlx5_sd_init(mdev); 6181 if (err) 6182 return err; 6183 6184 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6185 if (actual_adev) 6186 return _mlx5e_probe(actual_adev); 6187 return 0; 6188} 6189 6190static void _mlx5e_remove(struct auxiliary_device *adev) 6191{ 6192 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6193 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6194 struct mlx5e_priv *priv = mlx5e_dev->priv; 6195 struct mlx5_core_dev *mdev = edev->mdev; 6196 6197 mlx5_core_uplink_netdev_set(mdev, NULL); 6198 mlx5e_dcbnl_delete_app(priv); 6199 unregister_netdev(priv->netdev); 6200 _mlx5e_suspend(adev); 6201 priv->profile->cleanup(priv); 6202 mlx5e_destroy_netdev(priv); 6203 mlx5e_devlink_port_unregister(mlx5e_dev); 6204 mlx5e_destroy_devlink(mlx5e_dev); 6205} 6206 6207static void mlx5e_remove(struct auxiliary_device *adev) 6208{ 6209 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6210 struct mlx5_core_dev *mdev = edev->mdev; 6211 struct auxiliary_device *actual_adev; 6212 6213 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6214 if (actual_adev) 6215 _mlx5e_remove(actual_adev); 6216 6217 mlx5_sd_cleanup(mdev); 6218} 6219 6220static const struct auxiliary_device_id mlx5e_id_table[] = { 6221 { .name = MLX5_ADEV_NAME ".eth", }, 6222 {}, 6223}; 6224 6225MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table); 6226 6227static struct auxiliary_driver mlx5e_driver = { 6228 .name = "eth", 6229 .probe = mlx5e_probe, 6230 .remove = mlx5e_remove, 6231 .suspend = mlx5e_suspend, 6232 .resume = mlx5e_resume, 6233 .id_table = mlx5e_id_table, 6234}; 6235 6236int mlx5e_init(void) 6237{ 6238 int ret; 6239 6240 mlx5e_build_ptys2ethtool_map(); 6241 ret = auxiliary_driver_register(&mlx5e_driver); 6242 if (ret) 6243 return ret; 6244 6245 ret = mlx5e_rep_init(); 6246 if (ret) 6247 auxiliary_driver_unregister(&mlx5e_driver); 6248 return ret; 6249} 6250 6251void mlx5e_cleanup(void) 6252{ 6253 mlx5e_rep_cleanup(); 6254 auxiliary_driver_unregister(&mlx5e_driver); 6255} 6256