1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020 Alexander V. Chernikov 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29#include "opt_inet.h" 30#include "opt_inet6.h" 31#include "opt_route.h" 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/lock.h> 36#include <sys/rwlock.h> 37#include <sys/malloc.h> 38#include <sys/socket.h> 39#include <sys/sysctl.h> 40#include <sys/kernel.h> 41#include <sys/epoch.h> 42 43#include <net/if.h> 44#include <net/if_var.h> 45#include <net/if_private.h> 46#include <net/if_dl.h> 47#include <net/route.h> 48#include <net/route/route_ctl.h> 49#include <net/route/route_var.h> 50#include <net/route/nhop_utils.h> 51#include <net/route/nhop.h> 52#include <net/route/nhop_var.h> 53#include <net/vnet.h> 54 55#define DEBUG_MOD_NAME nhop_ctl 56#define DEBUG_MAX_LEVEL LOG_DEBUG 57#include <net/route/route_debug.h> 58_DECLARE_DEBUG(LOG_INFO); 59 60/* 61 * This file contains core functionality for the nexthop ("nhop") route subsystem. 62 * The business logic needed to create nexhop objects is implemented here. 63 * 64 * Nexthops in the original sense are the objects containing all the necessary 65 * information to forward the packet to the selected destination. 66 * In particular, nexthop is defined by a combination of 67 * ifp, ifa, aifp, mtu, gw addr(if set), nh_type, nh_upper_family, mask of rt_flags and 68 * NHF_DEFAULT 69 * 70 * Additionally, each nexthop gets assigned its unique index (nexthop index). 71 * It serves two purposes: first one is to ease the ability of userland programs to 72 * reference nexthops by their index. The second one allows lookup algorithms to 73 * to store index instead of pointer (2 bytes vs 8) as a lookup result. 74 * All nexthops are stored in the resizable hash table. 75 * 76 * Basically, this file revolves around supporting 3 functions: 77 * 1) nhop_create_from_info / nhop_create_from_nhop, which contains all 78 * business logic on filling the nexthop fields based on the provided request. 79 * 2) nhop_get(), which gets a usable referenced nexthops. 80 * 81 * Conventions: 82 * 1) non-exported functions start with verb 83 * 2) exported function starts with the subsystem prefix: "nhop" 84 */ 85 86static int dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w); 87 88static int finalize_nhop(struct nh_control *ctl, struct nhop_object *nh, bool link); 89static struct ifnet *get_aifp(const struct nhop_object *nh); 90static void fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp); 91 92static void destroy_nhop_epoch(epoch_context_t ctx); 93static void destroy_nhop(struct nhop_object *nh); 94 95_Static_assert(__offsetof(struct nhop_object, nh_ifp) == 32, 96 "nhop_object: wrong nh_ifp offset"); 97_Static_assert(sizeof(struct nhop_object) <= 128, 98 "nhop_object: size exceeds 128 bytes"); 99 100static uma_zone_t nhops_zone; /* Global zone for each and every nexthop */ 101 102#define NHOP_OBJECT_ALIGNED_SIZE roundup2(sizeof(struct nhop_object), \ 103 2 * CACHE_LINE_SIZE) 104#define NHOP_PRIV_ALIGNED_SIZE roundup2(sizeof(struct nhop_priv), \ 105 2 * CACHE_LINE_SIZE) 106void 107nhops_init(void) 108{ 109 110 nhops_zone = uma_zcreate("routing nhops", 111 NHOP_OBJECT_ALIGNED_SIZE + NHOP_PRIV_ALIGNED_SIZE, 112 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 113} 114 115/* 116 * Fetches the interface of source address used by the route. 117 * In all cases except interface-address-route it would be the 118 * same as the transmit interfaces. 119 * However, for the interface address this function will return 120 * this interface ifp instead of loopback. This is needed to support 121 * link-local IPv6 loopback communications. 122 * 123 * Returns found ifp. 124 */ 125static struct ifnet * 126get_aifp(const struct nhop_object *nh) 127{ 128 struct ifnet *aifp = NULL; 129 130 /* 131 * Adjust the "outgoing" interface. If we're going to loop 132 * the packet back to ourselves, the ifp would be the loopback 133 * interface. However, we'd rather know the interface associated 134 * to the destination address (which should probably be one of 135 * our own addresses). 136 */ 137 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) && 138 nh->gw_sa.sa_family == AF_LINK) { 139 aifp = ifnet_byindex(nh->gwl_sa.sdl_index); 140 if (aifp == NULL) { 141 FIB_NH_LOG(LOG_WARNING, nh, "unable to get aifp for %s index %d", 142 if_name(nh->nh_ifp), nh->gwl_sa.sdl_index); 143 } 144 } 145 146 if (aifp == NULL) 147 aifp = nh->nh_ifp; 148 149 return (aifp); 150} 151 152int 153cmp_priv(const struct nhop_priv *_one, const struct nhop_priv *_two) 154{ 155 156 if (memcmp(_one->nh, _two->nh, NHOP_END_CMP) != 0) 157 return (0); 158 159 if (memcmp(_one, _two, NH_PRIV_END_CMP) != 0) 160 return (0); 161 162 return (1); 163} 164 165/* 166 * Conditionally sets @nh mtu data based on the @info data. 167 */ 168static void 169set_nhop_mtu_from_info(struct nhop_object *nh, const struct rt_addrinfo *info) 170{ 171 if (info->rti_mflags & RTV_MTU) 172 nhop_set_mtu(nh, info->rti_rmx->rmx_mtu, true); 173} 174 175/* 176 * Fills in shorted link-level sockadd version suitable to be stored inside the 177 * nexthop gateway buffer. 178 */ 179static void 180fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp) 181{ 182 183 bzero(sdl, sizeof(struct sockaddr_dl_short)); 184 sdl->sdl_family = AF_LINK; 185 sdl->sdl_len = sizeof(struct sockaddr_dl_short); 186 sdl->sdl_index = ifp->if_index; 187 sdl->sdl_type = ifp->if_type; 188} 189 190static int 191set_nhop_gw_from_info(struct nhop_object *nh, struct rt_addrinfo *info) 192{ 193 struct sockaddr *gw; 194 195 gw = info->rti_info[RTAX_GATEWAY]; 196 MPASS(gw != NULL); 197 bool is_gw = info->rti_flags & RTF_GATEWAY; 198 199 if ((gw->sa_family == AF_LINK) && !is_gw) { 200 201 /* 202 * Interface route with interface specified by the interface 203 * index in sockadd_dl structure. It is used in the IPv6 loopback 204 * output code, where we need to preserve the original interface 205 * to maintain proper scoping. 206 * Despite the fact that nexthop code stores original interface 207 * in the separate field (nh_aifp, see below), write AF_LINK 208 * compatible sa with shorter total length. 209 */ 210 struct sockaddr_dl *sdl = (struct sockaddr_dl *)gw; 211 struct ifnet *ifp = ifnet_byindex(sdl->sdl_index); 212 if (ifp == NULL) { 213 FIB_NH_LOG(LOG_DEBUG, nh, "error: invalid ifindex %d", 214 sdl->sdl_index); 215 return (EINVAL); 216 } 217 nhop_set_direct_gw(nh, ifp); 218 } else { 219 220 /* 221 * Multiple options here: 222 * 223 * 1) RTF_GATEWAY with IPv4/IPv6 gateway data 224 * 2) Interface route with IPv4/IPv6 address of the 225 * matching interface. Some routing daemons do that 226 * instead of specifying ifindex in AF_LINK. 227 * 228 * In both cases, save the original nexthop to make the callers 229 * happy. 230 */ 231 if (!nhop_set_gw(nh, gw, is_gw)) 232 return (EINVAL); 233 } 234 return (0); 235} 236 237static void 238set_nhop_expire_from_info(struct nhop_object *nh, const struct rt_addrinfo *info) 239{ 240 uint32_t nh_expire = 0; 241 242 /* Kernel -> userland timebase conversion. */ 243 if ((info->rti_mflags & RTV_EXPIRE) && (info->rti_rmx->rmx_expire > 0)) 244 nh_expire = info->rti_rmx->rmx_expire - time_second + time_uptime; 245 nhop_set_expire(nh, nh_expire); 246} 247 248/* 249 * Creates a new nexthop based on the information in @info. 250 * 251 * Returns: 252 * 0 on success, filling @nh_ret with the desired nexthop object ptr 253 * errno otherwise 254 */ 255int 256nhop_create_from_info(struct rib_head *rnh, struct rt_addrinfo *info, 257 struct nhop_object **nh_ret) 258{ 259 int error; 260 261 NET_EPOCH_ASSERT(); 262 263 MPASS(info->rti_ifa != NULL); 264 MPASS(info->rti_ifp != NULL); 265 266 if (info->rti_info[RTAX_GATEWAY] == NULL) { 267 FIB_RH_LOG(LOG_DEBUG, rnh, "error: empty gateway"); 268 return (EINVAL); 269 } 270 271 struct nhop_object *nh = nhop_alloc(rnh->rib_fibnum, rnh->rib_family); 272 if (nh == NULL) 273 return (ENOMEM); 274 275 if ((error = set_nhop_gw_from_info(nh, info)) != 0) { 276 nhop_free(nh); 277 return (error); 278 } 279 nhop_set_transmit_ifp(nh, info->rti_ifp); 280 281 nhop_set_blackhole(nh, info->rti_flags & (RTF_BLACKHOLE | RTF_REJECT)); 282 283 error = rnh->rnh_set_nh_pfxflags(rnh->rib_fibnum, info->rti_info[RTAX_DST], 284 info->rti_info[RTAX_NETMASK], nh); 285 286 nhop_set_redirect(nh, info->rti_flags & RTF_DYNAMIC); 287 nhop_set_pinned(nh, info->rti_flags & RTF_PINNED); 288 set_nhop_expire_from_info(nh, info); 289 nhop_set_rtflags(nh, info->rti_flags); 290 291 set_nhop_mtu_from_info(nh, info); 292 nhop_set_src(nh, info->rti_ifa); 293 294 /* 295 * The remaining fields are either set from nh_preadd hook 296 * or are computed from the provided data 297 */ 298 *nh_ret = nhop_get_nhop(nh, &error); 299 300 return (error); 301} 302 303/* 304 * Gets linked nhop using the provided @nh nexhop data. 305 * If linked nhop is found, returns it, freeing the provided one. 306 * If there is no such nexthop, attaches the remaining data to the 307 * provided nexthop and links it. 308 * 309 * Returns 0 on success, storing referenced nexthop in @pnh. 310 * Otherwise, errno is returned. 311 */ 312struct nhop_object * 313nhop_get_nhop(struct nhop_object *nh, int *perror) 314{ 315 struct rib_head *rnh = nhop_get_rh(nh); 316 317 if (__predict_false(rnh == NULL)) { 318 *perror = EAFNOSUPPORT; 319 nhop_free(nh); 320 return (NULL); 321 } 322 323 return (nhop_get_nhop_internal(rnh, nh, perror)); 324} 325 326struct nhop_object * 327nhop_get_nhop_internal(struct rib_head *rnh, struct nhop_object *nh, int *perror) 328{ 329 struct nhop_priv *tmp_priv; 330 int error; 331 332 nh->nh_aifp = get_aifp(nh); 333 334 /* Give the protocols chance to augment nexthop properties */ 335 error = rnh->rnh_augment_nh(rnh->rib_fibnum, nh); 336 if (error != 0) { 337 nhop_free(nh); 338 *perror = error; 339 return (NULL); 340 } 341 342 tmp_priv = find_nhop(rnh->nh_control, nh->nh_priv); 343 if (tmp_priv != NULL) { 344 nhop_free(nh); 345 *perror = 0; 346 return (tmp_priv->nh); 347 } 348 349 /* 350 * Existing nexthop not found, need to create new one. 351 * Note: multiple simultaneous requests 352 * can result in multiple equal nexhops existing in the 353 * nexthop table. This is not a not a problem until the 354 * relative number of such nexthops is significant, which 355 * is extremely unlikely. 356 */ 357 *perror = finalize_nhop(rnh->nh_control, nh, true); 358 return (*perror == 0 ? nh : NULL); 359} 360 361/* 362 * Gets referenced but unlinked nhop. 363 * Alocates/references the remaining bits of the nexthop data, so 364 * it can be safely linked later or used as a clone source. 365 * 366 * Returns 0 on success. 367 */ 368int 369nhop_get_unlinked(struct nhop_object *nh) 370{ 371 struct rib_head *rnh = nhop_get_rh(nh); 372 373 if (__predict_false(rnh == NULL)) { 374 nhop_free(nh); 375 return (EAFNOSUPPORT); 376 } 377 378 nh->nh_aifp = get_aifp(nh); 379 380 return (finalize_nhop(rnh->nh_control, nh, false)); 381} 382 383 384/* 385 * Update @nh with data supplied in @info. 386 * This is a helper function to support route changes. 387 * 388 * It limits the changes that can be done to the route to the following: 389 * 1) all combination of gateway changes 390 * 2) route flags (FLAG[123],STATIC) 391 * 3) route MTU 392 * 393 * Returns: 394 * 0 on success, errno otherwise 395 */ 396static int 397alter_nhop_from_info(struct nhop_object *nh, struct rt_addrinfo *info) 398{ 399 struct sockaddr *info_gw; 400 int error; 401 402 /* Update MTU if set in the request*/ 403 set_nhop_mtu_from_info(nh, info); 404 405 /* Only RTF_FLAG[123] and RTF_STATIC */ 406 uint32_t rt_flags = nhop_get_rtflags(nh) & ~RT_CHANGE_RTFLAGS_MASK; 407 rt_flags |= info->rti_flags & RT_CHANGE_RTFLAGS_MASK; 408 nhop_set_rtflags(nh, rt_flags); 409 410 /* Consider gateway change */ 411 info_gw = info->rti_info[RTAX_GATEWAY]; 412 if (info_gw != NULL) { 413 error = set_nhop_gw_from_info(nh, info); 414 if (error != 0) 415 return (error); 416 } 417 418 if (info->rti_ifa != NULL) 419 nhop_set_src(nh, info->rti_ifa); 420 if (info->rti_ifp != NULL) 421 nhop_set_transmit_ifp(nh, info->rti_ifp); 422 423 return (0); 424} 425 426/* 427 * Creates new nexthop based on @nh_orig and augmentation data from @info. 428 * Helper function used in the route changes, please see 429 * alter_nhop_from_info() comments for more details. 430 * 431 * Returns: 432 * 0 on success, filling @nh_ret with the desired nexthop object 433 * errno otherwise 434 */ 435int 436nhop_create_from_nhop(struct rib_head *rnh, const struct nhop_object *nh_orig, 437 struct rt_addrinfo *info, struct nhop_object **pnh) 438{ 439 struct nhop_object *nh; 440 int error; 441 442 NET_EPOCH_ASSERT(); 443 444 nh = nhop_alloc(rnh->rib_fibnum, rnh->rib_family); 445 if (nh == NULL) 446 return (ENOMEM); 447 448 nhop_copy(nh, nh_orig); 449 450 error = alter_nhop_from_info(nh, info); 451 if (error != 0) { 452 nhop_free(nh); 453 return (error); 454 } 455 456 *pnh = nhop_get_nhop(nh, &error); 457 458 return (error); 459} 460 461static bool 462reference_nhop_deps(struct nhop_object *nh) 463{ 464 if (!ifa_try_ref(nh->nh_ifa)) 465 return (false); 466 nh->nh_aifp = get_aifp(nh); 467 if (!if_try_ref(nh->nh_aifp)) { 468 ifa_free(nh->nh_ifa); 469 return (false); 470 } 471 FIB_NH_LOG(LOG_DEBUG2, nh, "nh_aifp: %s nh_ifp %s", 472 if_name(nh->nh_aifp), if_name(nh->nh_ifp)); 473 if (!if_try_ref(nh->nh_ifp)) { 474 ifa_free(nh->nh_ifa); 475 if_rele(nh->nh_aifp); 476 return (false); 477 } 478 479 return (true); 480} 481 482/* 483 * Alocates/references the remaining bits of nexthop data and links 484 * it to the hash table. 485 * Returns 0 if successful, 486 * errno otherwise. @nh_priv is freed in case of error. 487 */ 488static int 489finalize_nhop(struct nh_control *ctl, struct nhop_object *nh, bool link) 490{ 491 492 /* Allocate per-cpu packet counter */ 493 nh->nh_pksent = counter_u64_alloc(M_NOWAIT); 494 if (nh->nh_pksent == NULL) { 495 nhop_free(nh); 496 RTSTAT_INC(rts_nh_alloc_failure); 497 FIB_NH_LOG(LOG_WARNING, nh, "counter_u64_alloc() failed"); 498 return (ENOMEM); 499 } 500 501 if (!reference_nhop_deps(nh)) { 502 counter_u64_free(nh->nh_pksent); 503 nhop_free(nh); 504 RTSTAT_INC(rts_nh_alloc_failure); 505 FIB_NH_LOG(LOG_WARNING, nh, "interface reference failed"); 506 return (EAGAIN); 507 } 508 509 /* Save vnet to ease destruction */ 510 nh->nh_priv->nh_vnet = curvnet; 511 512 /* Please see nhop_free() comments on the initial value */ 513 refcount_init(&nh->nh_priv->nh_linked, 2); 514 515 MPASS(nh->nh_priv->nh_fibnum == ctl->ctl_rh->rib_fibnum); 516 517 if (!link) { 518 refcount_release(&nh->nh_priv->nh_linked); 519 NHOPS_WLOCK(ctl); 520 nh->nh_priv->nh_finalized = 1; 521 NHOPS_WUNLOCK(ctl); 522 } else if (link_nhop(ctl, nh->nh_priv) == 0) { 523 /* 524 * Adding nexthop to the datastructures 525 * failed. Call destructor w/o waiting for 526 * the epoch end, as nexthop is not used 527 * and return. 528 */ 529 char nhbuf[NHOP_PRINT_BUFSIZE]; 530 FIB_NH_LOG(LOG_WARNING, nh, "failed to link %s", 531 nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); 532 destroy_nhop(nh); 533 534 return (ENOBUFS); 535 } 536 537 IF_DEBUG_LEVEL(LOG_DEBUG) { 538 char nhbuf[NHOP_PRINT_BUFSIZE] __unused; 539 FIB_NH_LOG(LOG_DEBUG, nh, "finalized: %s", 540 nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); 541 } 542 543 return (0); 544} 545 546static void 547destroy_nhop(struct nhop_object *nh) 548{ 549 if_rele(nh->nh_ifp); 550 if_rele(nh->nh_aifp); 551 ifa_free(nh->nh_ifa); 552 counter_u64_free(nh->nh_pksent); 553 554 uma_zfree(nhops_zone, nh); 555} 556 557/* 558 * Epoch callback indicating nhop is safe to destroy 559 */ 560static void 561destroy_nhop_epoch(epoch_context_t ctx) 562{ 563 struct nhop_priv *nh_priv; 564 565 nh_priv = __containerof(ctx, struct nhop_priv, nh_epoch_ctx); 566 567 destroy_nhop(nh_priv->nh); 568} 569 570void 571nhop_ref_object(struct nhop_object *nh) 572{ 573 u_int old __diagused; 574 575 old = refcount_acquire(&nh->nh_priv->nh_refcnt); 576 KASSERT(old > 0, ("%s: nhop object %p has 0 refs", __func__, nh)); 577} 578 579int 580nhop_try_ref_object(struct nhop_object *nh) 581{ 582 583 return (refcount_acquire_if_not_zero(&nh->nh_priv->nh_refcnt)); 584} 585 586void 587nhop_free(struct nhop_object *nh) 588{ 589 struct nh_control *ctl; 590 struct nhop_priv *nh_priv = nh->nh_priv; 591 struct epoch_tracker et; 592 593 if (!refcount_release(&nh_priv->nh_refcnt)) 594 return; 595 596 /* allows to use nhop_free() during nhop init */ 597 if (__predict_false(nh_priv->nh_finalized == 0)) { 598 uma_zfree(nhops_zone, nh); 599 return; 600 } 601 602 IF_DEBUG_LEVEL(LOG_DEBUG) { 603 char nhbuf[NHOP_PRINT_BUFSIZE] __unused; 604 FIB_NH_LOG(LOG_DEBUG, nh, "deleting %s", 605 nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); 606 } 607 608 /* 609 * There are only 2 places, where nh_linked can be decreased: 610 * rib destroy (nhops_destroy_rib) and this function. 611 * nh_link can never be increased. 612 * 613 * Hence, use initial value of 2 to make use of 614 * refcount_release_if_not_last(). 615 * 616 * There can be two scenarious when calling this function: 617 * 618 * 1) nh_linked value is 2. This means that either 619 * nhops_destroy_rib() has not been called OR it is running, 620 * but we are guaranteed that nh_control won't be freed in 621 * this epoch. Hence, nexthop can be safely unlinked. 622 * 623 * 2) nh_linked value is 1. In that case, nhops_destroy_rib() 624 * has been called and nhop unlink can be skipped. 625 */ 626 627 NET_EPOCH_ENTER(et); 628 if (refcount_release_if_not_last(&nh_priv->nh_linked)) { 629 ctl = nh_priv->nh_control; 630 if (unlink_nhop(ctl, nh_priv) == NULL) { 631 /* Do not try to reclaim */ 632 char nhbuf[NHOP_PRINT_BUFSIZE]; 633 FIB_NH_LOG(LOG_WARNING, nh, "failed to unlink %s", 634 nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); 635 NET_EPOCH_EXIT(et); 636 return; 637 } 638 } 639 NET_EPOCH_EXIT(et); 640 641 NET_EPOCH_CALL(destroy_nhop_epoch, &nh_priv->nh_epoch_ctx); 642} 643 644void 645nhop_ref_any(struct nhop_object *nh) 646{ 647#ifdef ROUTE_MPATH 648 if (!NH_IS_NHGRP(nh)) 649 nhop_ref_object(nh); 650 else 651 nhgrp_ref_object((struct nhgrp_object *)nh); 652#else 653 nhop_ref_object(nh); 654#endif 655} 656 657void 658nhop_free_any(struct nhop_object *nh) 659{ 660 661#ifdef ROUTE_MPATH 662 if (!NH_IS_NHGRP(nh)) 663 nhop_free(nh); 664 else 665 nhgrp_free((struct nhgrp_object *)nh); 666#else 667 nhop_free(nh); 668#endif 669} 670 671/* Nhop-related methods */ 672 673/* 674 * Allocates an empty unlinked nhop object. 675 * Returns object pointer or NULL on failure 676 */ 677struct nhop_object * 678nhop_alloc(uint32_t fibnum, int family) 679{ 680 struct nhop_object *nh; 681 struct nhop_priv *nh_priv; 682 683 nh = (struct nhop_object *)uma_zalloc(nhops_zone, M_NOWAIT | M_ZERO); 684 if (__predict_false(nh == NULL)) 685 return (NULL); 686 687 nh_priv = (struct nhop_priv *)((char *)nh + NHOP_OBJECT_ALIGNED_SIZE); 688 nh->nh_priv = nh_priv; 689 nh_priv->nh = nh; 690 691 nh_priv->nh_upper_family = family; 692 nh_priv->nh_fibnum = fibnum; 693 694 /* Setup refcount early to allow nhop_free() to work */ 695 refcount_init(&nh_priv->nh_refcnt, 1); 696 697 return (nh); 698} 699 700void 701nhop_copy(struct nhop_object *nh, const struct nhop_object *nh_orig) 702{ 703 struct nhop_priv *nh_priv = nh->nh_priv; 704 705 nh->nh_flags = nh_orig->nh_flags; 706 nh->nh_mtu = nh_orig->nh_mtu; 707 memcpy(&nh->gw_sa, &nh_orig->gw_sa, nh_orig->gw_sa.sa_len); 708 nh->nh_ifp = nh_orig->nh_ifp; 709 nh->nh_ifa = nh_orig->nh_ifa; 710 nh->nh_aifp = nh_orig->nh_aifp; 711 712 nh_priv->nh_upper_family = nh_orig->nh_priv->nh_upper_family; 713 nh_priv->nh_neigh_family = nh_orig->nh_priv->nh_neigh_family; 714 nh_priv->nh_type = nh_orig->nh_priv->nh_type; 715 nh_priv->rt_flags = nh_orig->nh_priv->rt_flags; 716 nh_priv->nh_fibnum = nh_orig->nh_priv->nh_fibnum; 717 nh_priv->nh_origin = nh_orig->nh_priv->nh_origin; 718} 719 720void 721nhop_set_direct_gw(struct nhop_object *nh, struct ifnet *ifp) 722{ 723 nh->nh_flags &= ~NHF_GATEWAY; 724 nh->nh_priv->rt_flags &= ~RTF_GATEWAY; 725 nh->nh_priv->nh_neigh_family = nh->nh_priv->nh_upper_family; 726 727 fill_sdl_from_ifp(&nh->gwl_sa, ifp); 728 memset(&nh->gw_buf[nh->gw_sa.sa_len], 0, sizeof(nh->gw_buf) - nh->gw_sa.sa_len); 729} 730 731bool 732nhop_check_gateway(int upper_family, int neigh_family) 733{ 734 if (upper_family == neigh_family) 735 return (true); 736 else if (neigh_family == AF_UNSPEC || neigh_family == AF_LINK) 737 return (true); 738#if defined(INET) && defined(INET6) 739 else if (upper_family == AF_INET && neigh_family == AF_INET6 && 740 rib_can_4o6_nhop()) 741 return (true); 742#endif 743 else 744 return (false); 745} 746 747/* 748 * Sets gateway for the nexthop. 749 * It can be "normal" gateway with is_gw set or a special form of 750 * adding interface route, refering to it by specifying local interface 751 * address. In that case is_gw is set to false. 752 */ 753bool 754nhop_set_gw(struct nhop_object *nh, const struct sockaddr *gw, bool is_gw) 755{ 756 if (gw->sa_len > sizeof(nh->gw_buf)) { 757 FIB_NH_LOG(LOG_DEBUG, nh, "nhop SA size too big: AF %d len %u", 758 gw->sa_family, gw->sa_len); 759 return (false); 760 } 761 762 if (!nhop_check_gateway(nh->nh_priv->nh_upper_family, gw->sa_family)) { 763 FIB_NH_LOG(LOG_DEBUG, nh, 764 "error: invalid dst/gateway family combination (%d, %d)", 765 nh->nh_priv->nh_upper_family, gw->sa_family); 766 return (false); 767 } 768 769 memcpy(&nh->gw_sa, gw, gw->sa_len); 770 memset(&nh->gw_buf[gw->sa_len], 0, sizeof(nh->gw_buf) - gw->sa_len); 771 772 if (is_gw) { 773 nh->nh_flags |= NHF_GATEWAY; 774 nh->nh_priv->rt_flags |= RTF_GATEWAY; 775 nh->nh_priv->nh_neigh_family = gw->sa_family; 776 } else { 777 nh->nh_flags &= ~NHF_GATEWAY; 778 nh->nh_priv->rt_flags &= ~RTF_GATEWAY; 779 nh->nh_priv->nh_neigh_family = nh->nh_priv->nh_upper_family; 780 } 781 782 return (true); 783} 784 785bool 786nhop_set_upper_family(struct nhop_object *nh, int family) 787{ 788 if (!nhop_check_gateway(nh->nh_priv->nh_upper_family, family)) { 789 FIB_NH_LOG(LOG_DEBUG, nh, 790 "error: invalid upper/neigh family combination (%d, %d)", 791 nh->nh_priv->nh_upper_family, family); 792 return (false); 793 } 794 795 nh->nh_priv->nh_upper_family = family; 796 return (true); 797} 798 799void 800nhop_set_broadcast(struct nhop_object *nh, bool is_broadcast) 801{ 802 if (is_broadcast) { 803 nh->nh_flags |= NHF_BROADCAST; 804 nh->nh_priv->rt_flags |= RTF_BROADCAST; 805 } else { 806 nh->nh_flags &= ~NHF_BROADCAST; 807 nh->nh_priv->rt_flags &= ~RTF_BROADCAST; 808 } 809} 810 811void 812nhop_set_blackhole(struct nhop_object *nh, int blackhole_rt_flag) 813{ 814 nh->nh_flags &= ~(NHF_BLACKHOLE | NHF_REJECT); 815 nh->nh_priv->rt_flags &= ~(RTF_BLACKHOLE | RTF_REJECT); 816 switch (blackhole_rt_flag) { 817 case RTF_BLACKHOLE: 818 nh->nh_flags |= NHF_BLACKHOLE; 819 nh->nh_priv->rt_flags |= RTF_BLACKHOLE; 820 break; 821 case RTF_REJECT: 822 nh->nh_flags |= NHF_REJECT; 823 nh->nh_priv->rt_flags |= RTF_REJECT; 824 break; 825 default: 826 /* Not a blackhole nexthop */ 827 return; 828 } 829 830 nh->nh_ifp = V_loif; 831 nh->nh_flags &= ~NHF_GATEWAY; 832 nh->nh_priv->rt_flags &= ~RTF_GATEWAY; 833 nh->nh_priv->nh_neigh_family = nh->nh_priv->nh_upper_family; 834 835 bzero(&nh->gw_sa, sizeof(nh->gw_sa)); 836 837 switch (nh->nh_priv->nh_upper_family) { 838#ifdef INET 839 case AF_INET: 840 nh->gw4_sa.sin_family = AF_INET; 841 nh->gw4_sa.sin_len = sizeof(struct sockaddr_in); 842 nh->gw4_sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK); 843 break; 844#endif 845#ifdef INET6 846 case AF_INET6: 847 nh->gw6_sa.sin6_family = AF_INET6; 848 nh->gw6_sa.sin6_len = sizeof(struct sockaddr_in6); 849 nh->gw6_sa.sin6_addr = in6addr_loopback; 850 break; 851#endif 852 } 853} 854 855void 856nhop_set_redirect(struct nhop_object *nh, bool is_redirect) 857{ 858 if (is_redirect) { 859 nh->nh_priv->rt_flags |= RTF_DYNAMIC; 860 nh->nh_flags |= NHF_REDIRECT; 861 } else { 862 nh->nh_priv->rt_flags &= ~RTF_DYNAMIC; 863 nh->nh_flags &= ~NHF_REDIRECT; 864 } 865} 866 867void 868nhop_set_pinned(struct nhop_object *nh, bool is_pinned) 869{ 870 if (is_pinned) 871 nh->nh_priv->rt_flags |= RTF_PINNED; 872 else 873 nh->nh_priv->rt_flags &= ~RTF_PINNED; 874} 875 876uint32_t 877nhop_get_idx(const struct nhop_object *nh) 878{ 879 880 return (nh->nh_priv->nh_idx); 881} 882 883uint32_t 884nhop_get_uidx(const struct nhop_object *nh) 885{ 886 return (nh->nh_priv->nh_uidx); 887} 888 889void 890nhop_set_uidx(struct nhop_object *nh, uint32_t uidx) 891{ 892 nh->nh_priv->nh_uidx = uidx; 893} 894 895enum nhop_type 896nhop_get_type(const struct nhop_object *nh) 897{ 898 899 return (nh->nh_priv->nh_type); 900} 901 902void 903nhop_set_type(struct nhop_object *nh, enum nhop_type nh_type) 904{ 905 906 nh->nh_priv->nh_type = nh_type; 907} 908 909int 910nhop_get_rtflags(const struct nhop_object *nh) 911{ 912 913 return (nh->nh_priv->rt_flags); 914} 915 916/* 917 * Sets generic rtflags that are not covered by other functions. 918 */ 919void 920nhop_set_rtflags(struct nhop_object *nh, int rt_flags) 921{ 922 nh->nh_priv->rt_flags &= ~RT_SET_RTFLAGS_MASK; 923 nh->nh_priv->rt_flags |= (rt_flags & RT_SET_RTFLAGS_MASK); 924} 925 926/* 927 * Sets flags that are specific to the prefix (NHF_HOST or NHF_DEFAULT). 928 */ 929void 930nhop_set_pxtype_flag(struct nhop_object *nh, int nh_flag) 931{ 932 if (nh_flag == NHF_HOST) { 933 nh->nh_flags |= NHF_HOST; 934 nh->nh_flags &= ~NHF_DEFAULT; 935 nh->nh_priv->rt_flags |= RTF_HOST; 936 } else if (nh_flag == NHF_DEFAULT) { 937 nh->nh_flags |= NHF_DEFAULT; 938 nh->nh_flags &= ~NHF_HOST; 939 nh->nh_priv->rt_flags &= ~RTF_HOST; 940 } else { 941 nh->nh_flags &= ~(NHF_HOST | NHF_DEFAULT); 942 nh->nh_priv->rt_flags &= ~RTF_HOST; 943 } 944} 945 946/* 947 * Sets nhop MTU. Sets RTF_FIXEDMTU if mtu is explicitly 948 * specified by userland. 949 */ 950void 951nhop_set_mtu(struct nhop_object *nh, uint32_t mtu, bool from_user) 952{ 953 if (from_user) { 954 if (mtu != 0) 955 nh->nh_priv->rt_flags |= RTF_FIXEDMTU; 956 else 957 nh->nh_priv->rt_flags &= ~RTF_FIXEDMTU; 958 } 959 nh->nh_mtu = mtu; 960} 961 962void 963nhop_set_src(struct nhop_object *nh, struct ifaddr *ifa) 964{ 965 nh->nh_ifa = ifa; 966} 967 968void 969nhop_set_transmit_ifp(struct nhop_object *nh, struct ifnet *ifp) 970{ 971 nh->nh_ifp = ifp; 972} 973 974 975struct vnet * 976nhop_get_vnet(const struct nhop_object *nh) 977{ 978 979 return (nh->nh_priv->nh_vnet); 980} 981 982struct nhop_object * 983nhop_select_func(struct nhop_object *nh, uint32_t flowid) 984{ 985 986 return (nhop_select(nh, flowid)); 987} 988 989/* 990 * Returns address family of the traffic uses the nexthop. 991 */ 992int 993nhop_get_upper_family(const struct nhop_object *nh) 994{ 995 return (nh->nh_priv->nh_upper_family); 996} 997 998/* 999 * Returns address family of the LLE or gateway that is used 1000 * to forward the traffic to. 1001 */ 1002int 1003nhop_get_neigh_family(const struct nhop_object *nh) 1004{ 1005 return (nh->nh_priv->nh_neigh_family); 1006} 1007 1008uint32_t 1009nhop_get_fibnum(const struct nhop_object *nh) 1010{ 1011 return (nh->nh_priv->nh_fibnum); 1012} 1013 1014void 1015nhop_set_fibnum(struct nhop_object *nh, uint32_t fibnum) 1016{ 1017 nh->nh_priv->nh_fibnum = fibnum; 1018} 1019 1020uint32_t 1021nhop_get_expire(const struct nhop_object *nh) 1022{ 1023 return (nh->nh_priv->nh_expire); 1024} 1025 1026void 1027nhop_set_expire(struct nhop_object *nh, uint32_t expire) 1028{ 1029 MPASS(!NH_IS_LINKED(nh)); 1030 nh->nh_priv->nh_expire = expire; 1031} 1032 1033struct rib_head * 1034nhop_get_rh(const struct nhop_object *nh) 1035{ 1036 uint32_t fibnum = nhop_get_fibnum(nh); 1037 int family = nhop_get_neigh_family(nh); 1038 1039 return (rt_tables_get_rnh(fibnum, family)); 1040} 1041 1042uint8_t 1043nhop_get_origin(const struct nhop_object *nh) 1044{ 1045 return (nh->nh_priv->nh_origin); 1046} 1047 1048void 1049nhop_set_origin(struct nhop_object *nh, uint8_t origin) 1050{ 1051 nh->nh_priv->nh_origin = origin; 1052} 1053 1054void 1055nhops_update_ifmtu(struct rib_head *rh, struct ifnet *ifp, uint32_t mtu) 1056{ 1057 struct nh_control *ctl; 1058 struct nhop_priv *nh_priv; 1059 struct nhop_object *nh; 1060 1061 ctl = rh->nh_control; 1062 1063 NHOPS_WLOCK(ctl); 1064 CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) { 1065 nh = nh_priv->nh; 1066 if (nh->nh_ifp == ifp) { 1067 if ((nh_priv->rt_flags & RTF_FIXEDMTU) == 0 || 1068 nh->nh_mtu > mtu) { 1069 /* Update MTU directly */ 1070 nh->nh_mtu = mtu; 1071 } 1072 } 1073 } CHT_SLIST_FOREACH_END; 1074 NHOPS_WUNLOCK(ctl); 1075 1076} 1077 1078struct nhop_object * 1079nhops_iter_start(struct nhop_iter *iter) 1080{ 1081 if (iter->rh == NULL) 1082 iter->rh = rt_tables_get_rnh_safe(iter->fibnum, iter->family); 1083 if (iter->rh != NULL) { 1084 struct nh_control *ctl = iter->rh->nh_control; 1085 1086 NHOPS_RLOCK(ctl); 1087 1088 iter->_i = 0; 1089 iter->_next = CHT_FIRST(&ctl->nh_head, iter->_i); 1090 1091 return (nhops_iter_next(iter)); 1092 } else 1093 return (NULL); 1094} 1095 1096struct nhop_object * 1097nhops_iter_next(struct nhop_iter *iter) 1098{ 1099 struct nhop_priv *nh_priv = iter->_next; 1100 1101 if (nh_priv != NULL) { 1102 iter->_next = nh_priv->nh_next; 1103 return (nh_priv->nh); 1104 } 1105 1106 struct nh_control *ctl = iter->rh->nh_control; 1107 while (++iter->_i < ctl->nh_head.hash_size) { 1108 nh_priv = CHT_FIRST(&ctl->nh_head, iter->_i); 1109 if (nh_priv != NULL) { 1110 iter->_next = nh_priv->nh_next; 1111 return (nh_priv->nh); 1112 } 1113 } 1114 1115 return (NULL); 1116} 1117 1118void 1119nhops_iter_stop(struct nhop_iter *iter) 1120{ 1121 if (iter->rh != NULL) { 1122 struct nh_control *ctl = iter->rh->nh_control; 1123 1124 NHOPS_RUNLOCK(ctl); 1125 } 1126} 1127 1128/* 1129 * Prints nexthop @nh data in the provided @buf. 1130 * Example: nh#33/inet/em0/192.168.0.1 1131 */ 1132char * 1133nhop_print_buf(const struct nhop_object *nh, char *buf, size_t bufsize) 1134{ 1135#if defined(INET) || defined(INET6) 1136 char abuf[INET6_ADDRSTRLEN]; 1137#endif 1138 struct nhop_priv *nh_priv = nh->nh_priv; 1139 const char *upper_str = rib_print_family(nh->nh_priv->nh_upper_family); 1140 1141 switch (nh->gw_sa.sa_family) { 1142#ifdef INET 1143 case AF_INET: 1144 inet_ntop(AF_INET, &nh->gw4_sa.sin_addr, abuf, sizeof(abuf)); 1145 snprintf(buf, bufsize, "nh#%d/%s/%s/%s", nh_priv->nh_idx, upper_str, 1146 if_name(nh->nh_ifp), abuf); 1147 break; 1148#endif 1149#ifdef INET6 1150 case AF_INET6: 1151 inet_ntop(AF_INET6, &nh->gw6_sa.sin6_addr, abuf, sizeof(abuf)); 1152 snprintf(buf, bufsize, "nh#%d/%s/%s/%s", nh_priv->nh_idx, upper_str, 1153 if_name(nh->nh_ifp), abuf); 1154 break; 1155#endif 1156 case AF_LINK: 1157 snprintf(buf, bufsize, "nh#%d/%s/%s/resolve", nh_priv->nh_idx, upper_str, 1158 if_name(nh->nh_ifp)); 1159 break; 1160 default: 1161 snprintf(buf, bufsize, "nh#%d/%s/%s/????", nh_priv->nh_idx, upper_str, 1162 if_name(nh->nh_ifp)); 1163 break; 1164 } 1165 1166 return (buf); 1167} 1168 1169char * 1170nhop_print_buf_any(const struct nhop_object *nh, char *buf, size_t bufsize) 1171{ 1172#ifdef ROUTE_MPATH 1173 if (NH_IS_NHGRP(nh)) 1174 return (nhgrp_print_buf((const struct nhgrp_object *)nh, buf, bufsize)); 1175 else 1176#endif 1177 return (nhop_print_buf(nh, buf, bufsize)); 1178} 1179 1180/* 1181 * Dumps a single entry to sysctl buffer. 1182 * 1183 * Layout: 1184 * rt_msghdr - generic RTM header to allow users to skip non-understood messages 1185 * nhop_external - nexhop description structure (with length) 1186 * nhop_addrs - structure encapsulating GW/SRC sockaddrs 1187 */ 1188static int 1189dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w) 1190{ 1191 struct { 1192 struct rt_msghdr rtm; 1193 struct nhop_external nhe; 1194 struct nhop_addrs na; 1195 } arpc; 1196 struct nhop_external *pnhe; 1197 struct sockaddr *gw_sa, *src_sa; 1198 struct sockaddr_storage ss; 1199 size_t addrs_len; 1200 int error; 1201 1202 memset(&arpc, 0, sizeof(arpc)); 1203 1204 arpc.rtm.rtm_msglen = sizeof(arpc); 1205 arpc.rtm.rtm_version = RTM_VERSION; 1206 arpc.rtm.rtm_type = RTM_GET; 1207 //arpc.rtm.rtm_flags = RTF_UP; 1208 arpc.rtm.rtm_flags = nh->nh_priv->rt_flags; 1209 1210 /* nhop_external */ 1211 pnhe = &arpc.nhe; 1212 pnhe->nh_len = sizeof(struct nhop_external); 1213 pnhe->nh_idx = nh->nh_priv->nh_idx; 1214 pnhe->nh_fib = rh->rib_fibnum; 1215 pnhe->ifindex = nh->nh_ifp->if_index; 1216 pnhe->aifindex = nh->nh_aifp->if_index; 1217 pnhe->nh_family = nh->nh_priv->nh_upper_family; 1218 pnhe->nh_type = nh->nh_priv->nh_type; 1219 pnhe->nh_mtu = nh->nh_mtu; 1220 pnhe->nh_flags = nh->nh_flags; 1221 1222 memcpy(pnhe->nh_prepend, nh->nh_prepend, sizeof(nh->nh_prepend)); 1223 pnhe->prepend_len = nh->nh_prepend_len; 1224 pnhe->nh_refcount = nh->nh_priv->nh_refcnt; 1225 pnhe->nh_pksent = counter_u64_fetch(nh->nh_pksent); 1226 1227 /* sockaddr container */ 1228 addrs_len = sizeof(struct nhop_addrs); 1229 arpc.na.gw_sa_off = addrs_len; 1230 gw_sa = (struct sockaddr *)&nh->gw4_sa; 1231 addrs_len += gw_sa->sa_len; 1232 1233 src_sa = nh->nh_ifa->ifa_addr; 1234 if (src_sa->sa_family == AF_LINK) { 1235 /* Shorten structure */ 1236 memset(&ss, 0, sizeof(struct sockaddr_storage)); 1237 fill_sdl_from_ifp((struct sockaddr_dl_short *)&ss, 1238 nh->nh_ifa->ifa_ifp); 1239 src_sa = (struct sockaddr *)&ss; 1240 } 1241 arpc.na.src_sa_off = addrs_len; 1242 addrs_len += src_sa->sa_len; 1243 1244 /* Write total container length */ 1245 arpc.na.na_len = addrs_len; 1246 1247 arpc.rtm.rtm_msglen += arpc.na.na_len - sizeof(struct nhop_addrs); 1248 1249 error = SYSCTL_OUT(w, &arpc, sizeof(arpc)); 1250 if (error == 0) 1251 error = SYSCTL_OUT(w, gw_sa, gw_sa->sa_len); 1252 if (error == 0) 1253 error = SYSCTL_OUT(w, src_sa, src_sa->sa_len); 1254 1255 return (error); 1256} 1257 1258uint32_t 1259nhops_get_count(struct rib_head *rh) 1260{ 1261 struct nh_control *ctl; 1262 uint32_t count; 1263 1264 ctl = rh->nh_control; 1265 1266 NHOPS_RLOCK(ctl); 1267 count = ctl->nh_head.items_count; 1268 NHOPS_RUNLOCK(ctl); 1269 1270 return (count); 1271} 1272 1273int 1274nhops_dump_sysctl(struct rib_head *rh, struct sysctl_req *w) 1275{ 1276 struct nh_control *ctl; 1277 struct nhop_priv *nh_priv; 1278 int error; 1279 1280 ctl = rh->nh_control; 1281 1282 NHOPS_RLOCK(ctl); 1283 FIB_RH_LOG(LOG_DEBUG, rh, "dump %u items", ctl->nh_head.items_count); 1284 CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) { 1285 error = dump_nhop_entry(rh, nh_priv->nh, w); 1286 if (error != 0) { 1287 NHOPS_RUNLOCK(ctl); 1288 return (error); 1289 } 1290 } CHT_SLIST_FOREACH_END; 1291 NHOPS_RUNLOCK(ctl); 1292 1293 return (0); 1294} 1295