1/* 2 * Copyright (c) 2011 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <sys/cdefs.h> 30#include <sys/param.h> 31#include <sys/mbuf.h> 32#include <sys/errno.h> 33#include <sys/random.h> 34#include <sys/kernel_types.h> 35#include <sys/sysctl.h> 36 37#include <kern/zalloc.h> 38 39#include <net/if.h> 40#include <net/net_osdep.h> 41#include <net/classq/classq.h> 42#if CLASSQ_RED 43#include <net/classq/classq_red.h> 44#endif /* CLASSQ_RED */ 45#if CLASSQ_RIO 46#include <net/classq/classq_rio.h> 47#endif /* CLASSQ_RIO */ 48#if CLASSQ_BLUE 49#include <net/classq/classq_blue.h> 50#endif /* CLASSQ_BLUE */ 51#include <net/classq/classq_sfb.h> 52#include <net/pktsched/pktsched.h> 53 54#include <libkern/libkern.h> 55 56#if PF_ALTQ 57#include <net/altq/altq.h> 58#endif /* PF_ALTQ */ 59 60static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t, 61 u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *, 62 boolean_t); 63static struct mbuf *ifclassq_poll_common(struct ifclassq *, 64 mbuf_svc_class_t, boolean_t); 65static struct mbuf *ifclassq_tbr_dequeue_common(struct ifclassq *, int, 66 mbuf_svc_class_t, boolean_t); 67 68void 69classq_init(void) 70{ 71 _CASSERT(MBUF_TC_BE == 0); 72 _CASSERT(MBUF_SC_BE == 0); 73 _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES); 74 75#if CLASSQ_RED 76 red_init(); 77#endif /* CLASSQ_RED */ 78#if CLASSQ_RIO 79 rio_init(); 80#endif /* CLASSQ_RIO */ 81#if CLASSQ_BLUE 82 blue_init(); 83#endif /* CLASSQ_BLUE */ 84 sfb_init(); 85} 86 87int 88ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse) 89{ 90#pragma unused(reuse) 91 struct ifclassq *ifq = &ifp->if_snd; 92 int err = 0; 93 94 IFCQ_LOCK(ifq); 95 VERIFY(IFCQ_IS_EMPTY(ifq)); 96 ifq->ifcq_ifp = ifp; 97 IFCQ_LEN(ifq) = 0; 98 bzero(&ifq->ifcq_xmitcnt, sizeof (ifq->ifcq_xmitcnt)); 99 bzero(&ifq->ifcq_dropcnt, sizeof (ifq->ifcq_dropcnt)); 100 101 VERIFY(!IFCQ_TBR_IS_ENABLED(ifq)); 102 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); 103 VERIFY(ifq->ifcq_flags == 0); 104 VERIFY(ifq->ifcq_sflags == 0); 105 VERIFY(ifq->ifcq_disc == NULL); 106 VERIFY(ifq->ifcq_enqueue == NULL); 107 VERIFY(ifq->ifcq_dequeue == NULL); 108 VERIFY(ifq->ifcq_dequeue_sc == NULL); 109 VERIFY(ifq->ifcq_request == NULL); 110 111 if (ifp->if_eflags & IFEF_TXSTART) { 112 u_int32_t maxlen = 0; 113 114 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0) 115 maxlen = if_sndq_maxlen; 116 IFCQ_SET_MAXLEN(ifq, maxlen); 117 118 ifq->ifcq_sflags = sflags; 119 err = ifclassq_pktsched_setup(ifq); 120 if (err == 0) 121 ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED); 122 } 123 124#if PF_ALTQ 125 ifq->ifcq_drain = 0; 126 IFCQ_ALTQ(ifq)->altq_ifcq = ifq; 127 VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE); 128 VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0); 129 VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL); 130 VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL); 131 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL); 132 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL); 133 VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL); 134 135 if ((ifp->if_eflags & IFEF_TXSTART) && 136 ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED) 137 ALTQ_SET_READY(IFCQ_ALTQ(ifq)); 138 else 139 ALTQ_CLEAR_READY(IFCQ_ALTQ(ifq)); 140#endif /* PF_ALTQ */ 141 IFCQ_UNLOCK(ifq); 142 143 return (err); 144} 145 146void 147ifclassq_teardown(struct ifnet *ifp) 148{ 149 struct ifclassq *ifq = &ifp->if_snd; 150 151 IFCQ_LOCK(ifq); 152#if PF_ALTQ 153 if (ALTQ_IS_READY(IFCQ_ALTQ(ifq))) { 154 if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) 155 altq_disable(IFCQ_ALTQ(ifq)); 156 if (ALTQ_IS_ATTACHED(IFCQ_ALTQ(ifq))) 157 altq_detach(IFCQ_ALTQ(ifq)); 158 IFCQ_ALTQ(ifq)->altq_flags = 0; 159 } 160 ifq->ifcq_drain = 0; 161 IFCQ_ALTQ(ifq)->altq_ifcq = NULL; 162 VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE); 163 VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0); 164 VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL); 165 VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL); 166 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL); 167 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL); 168 VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL); 169#endif /* PF_ALTQ */ 170 171 if (IFCQ_IS_READY(ifq)) { 172 if (IFCQ_TBR_IS_ENABLED(ifq)) { 173 struct tb_profile tb = { 0, 0, 0 }; 174 (void) ifclassq_tbr_set(ifq, &tb, FALSE); 175 } 176 (void) pktsched_teardown(ifq); 177 ifq->ifcq_flags = 0; 178 } 179 ifq->ifcq_sflags = 0; 180 181 VERIFY(IFCQ_IS_EMPTY(ifq)); 182 VERIFY(!IFCQ_TBR_IS_ENABLED(ifq)); 183 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); 184 VERIFY(ifq->ifcq_flags == 0); 185 VERIFY(ifq->ifcq_sflags == 0); 186 VERIFY(ifq->ifcq_disc == NULL); 187 VERIFY(ifq->ifcq_enqueue == NULL); 188 VERIFY(ifq->ifcq_dequeue == NULL); 189 VERIFY(ifq->ifcq_dequeue_sc == NULL); 190 VERIFY(ifq->ifcq_request == NULL); 191 IFCQ_LEN(ifq) = 0; 192 IFCQ_MAXLEN(ifq) = 0; 193 bzero(&ifq->ifcq_xmitcnt, sizeof (ifq->ifcq_xmitcnt)); 194 bzero(&ifq->ifcq_dropcnt, sizeof (ifq->ifcq_dropcnt)); 195 196 IFCQ_UNLOCK(ifq); 197} 198 199int 200ifclassq_pktsched_setup(struct ifclassq *ifq) 201{ 202 struct ifnet *ifp = ifq->ifcq_ifp; 203 int err = 0; 204 205 IFCQ_LOCK_ASSERT_HELD(ifq); 206 VERIFY(ifp->if_eflags & IFEF_TXSTART); 207 208 switch (ifp->if_output_sched_model) { 209 case IFNET_SCHED_MODEL_DRIVER_MANAGED: 210 err = pktsched_setup(ifq, PKTSCHEDT_TCQ, ifq->ifcq_sflags); 211 break; 212 213 case IFNET_SCHED_MODEL_NORMAL: 214 err = pktsched_setup(ifq, PKTSCHEDT_QFQ, ifq->ifcq_sflags); 215 break; 216 217 default: 218 VERIFY(0); 219 /* NOTREACHED */ 220 } 221 222 return (err); 223} 224 225void 226ifclassq_set_maxlen(struct ifclassq *ifq, u_int32_t maxqlen) 227{ 228 IFCQ_LOCK(ifq); 229 if (maxqlen == 0) 230 maxqlen = if_sndq_maxlen; 231 IFCQ_SET_MAXLEN(ifq, maxqlen); 232 IFCQ_UNLOCK(ifq); 233} 234 235u_int32_t 236ifclassq_get_maxlen(struct ifclassq *ifq) 237{ 238 return (IFCQ_MAXLEN(ifq)); 239} 240 241u_int32_t 242ifclassq_get_len(struct ifclassq *ifq) 243{ 244 return (IFCQ_LEN(ifq)); 245} 246 247errno_t 248ifclassq_enqueue(struct ifclassq *ifq, struct mbuf *m) 249{ 250 errno_t err; 251 252 IFCQ_LOCK_SPIN(ifq); 253 254#if PF_ALTQ 255 if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) { 256 ALTQ_ENQUEUE(IFCQ_ALTQ(ifq), m, err); 257 } else { 258 u_int32_t qlen = IFCQ_LEN(ifq); 259 IFCQ_ENQUEUE(ifq, m, err); 260 if (IFCQ_LEN(ifq) > qlen) 261 ifq->ifcq_drain += (IFCQ_LEN(ifq) - qlen); 262 } 263#else /* !PF_ALTQ */ 264 IFCQ_ENQUEUE(ifq, m, err); 265#endif /* PF_ALTQ */ 266 267 IFCQ_UNLOCK(ifq); 268 269 return (err); 270} 271 272errno_t 273ifclassq_dequeue(struct ifclassq *ifq, u_int32_t limit, struct mbuf **head, 274 struct mbuf **tail, u_int32_t *cnt, u_int32_t *len) 275{ 276 return (ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, limit, head, tail, 277 cnt, len, FALSE)); 278} 279 280errno_t 281ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc, 282 u_int32_t limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, 283 u_int32_t *len) 284{ 285 return (ifclassq_dequeue_common(ifq, sc, limit, head, tail, 286 cnt, len, TRUE)); 287} 288 289static errno_t 290ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, 291 u_int32_t limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, 292 u_int32_t *len, boolean_t drvmgt) 293{ 294 struct ifnet *ifp = ifq->ifcq_ifp; 295 u_int32_t i = 0, l = 0; 296 struct mbuf **first, *last; 297#if PF_ALTQ 298 struct ifaltq *altq = IFCQ_ALTQ(ifq); 299 boolean_t draining; 300#endif /* PF_ALTQ */ 301 302 VERIFY(!drvmgt || MBUF_VALID_SC(sc)); 303 304 *head = NULL; 305 first = &(*head); 306 last = NULL; 307 308 ifq = &ifp->if_snd; 309 IFCQ_LOCK_SPIN(ifq); 310 311 while (i < limit) { 312 u_int64_t pktlen; 313#if PF_ALTQ 314 u_int32_t qlen; 315 316 qlen = IFCQ_LEN(ifq); 317 draining = IFCQ_IS_DRAINING(ifq); 318 319 if (drvmgt) { 320 if (IFCQ_TBR_IS_ENABLED(ifq)) 321 IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head); 322 else if (draining) 323 IFCQ_DEQUEUE_SC(ifq, sc, *head); 324 else if (ALTQ_IS_ENABLED(altq)) 325 ALTQ_DEQUEUE_SC(altq, sc, *head); 326 else 327 *head = NULL; 328 } else { 329 if (IFCQ_TBR_IS_ENABLED(ifq)) 330 IFCQ_TBR_DEQUEUE(ifq, *head); 331 else if (draining) 332 IFCQ_DEQUEUE(ifq, *head); 333 else if (ALTQ_IS_ENABLED(altq)) 334 ALTQ_DEQUEUE(altq, *head); 335 else 336 *head = NULL; 337 } 338 339 if (draining && *head != NULL) { 340 VERIFY(ifq->ifcq_drain >= (qlen - IFCQ_LEN(ifq))); 341 ifq->ifcq_drain -= (qlen - IFCQ_LEN(ifq)); 342 } 343#else /* ! PF_ALTQ */ 344 if (drvmgt) { 345 if (IFCQ_TBR_IS_ENABLED(ifq)) 346 IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head); 347 else 348 IFCQ_DEQUEUE_SC(ifq, sc, *head); 349 } else { 350 if (IFCQ_TBR_IS_ENABLED(ifq)) 351 IFCQ_TBR_DEQUEUE(ifq, *head); 352 else 353 IFCQ_DEQUEUE(ifq, *head); 354 } 355#endif /* !PF_ALTQ */ 356 357 if (*head == NULL) 358 break; 359 360 (*head)->m_nextpkt = NULL; 361 last = *head; 362 363 l += (*head)->m_pkthdr.len; 364 pktlen = (*head)->m_pkthdr.len; 365 366 (*head)->m_pkthdr.pf_mtag.pftag_pktseq = 367 atomic_add_64_ov(&(ifp->if_bw.cur_seq), pktlen); 368 369 head = &(*head)->m_nextpkt; 370 i++; 371 } 372 373 IFCQ_UNLOCK(ifq); 374 375 if (tail != NULL) 376 *tail = last; 377 if (cnt != NULL) 378 *cnt = i; 379 if (len != NULL) 380 *len = l; 381 382 return ((*first != NULL) ? 0 : EAGAIN); 383} 384 385struct mbuf * 386ifclassq_poll(struct ifclassq *ifq) 387{ 388 return (ifclassq_poll_common(ifq, MBUF_SC_UNSPEC, FALSE)); 389} 390 391struct mbuf * 392ifclassq_poll_sc(struct ifclassq *ifq, mbuf_svc_class_t sc) 393{ 394 return (ifclassq_poll_common(ifq, sc, TRUE)); 395} 396 397static struct mbuf * 398ifclassq_poll_common(struct ifclassq *ifq, mbuf_svc_class_t sc, 399 boolean_t drvmgt) 400{ 401#if PF_ALTQ 402 struct ifaltq *altq = IFCQ_ALTQ(ifq); 403#endif /* PF_ALTQ */ 404 struct mbuf *m; 405 406 VERIFY(!drvmgt || MBUF_VALID_SC(sc)); 407 408#if PF_ALTQ 409 if (drvmgt) { 410 if (IFCQ_TBR_IS_ENABLED(ifq)) 411 IFCQ_TBR_POLL_SC(ifq, sc, m); 412 else if (IFCQ_IS_DRAINING(ifq)) 413 IFCQ_POLL_SC(ifq, sc, m); 414 else if (ALTQ_IS_ENABLED(altq)) 415 ALTQ_POLL_SC(altq, sc, m); 416 else 417 m = NULL; 418 } else { 419 if (IFCQ_TBR_IS_ENABLED(ifq)) 420 IFCQ_TBR_POLL(ifq, m); 421 else if (IFCQ_IS_DRAINING(ifq)) 422 IFCQ_POLL(ifq, m); 423 else if (ALTQ_IS_ENABLED(altq)) 424 ALTQ_POLL(altq, m); 425 else 426 m = NULL; 427 } 428#else /* ! PF_ALTQ */ 429 if (drvmgt) { 430 if (IFCQ_TBR_IS_ENABLED(ifq)) 431 IFCQ_TBR_POLL_SC(ifq, sc, m); 432 else 433 IFCQ_POLL_SC(ifq, sc, m); 434 } else { 435 if (IFCQ_TBR_IS_ENABLED(ifq)) 436 IFCQ_TBR_POLL(ifq, m); 437 else 438 IFCQ_POLL(ifq, m); 439 } 440#endif /* !PF_ALTQ */ 441 442 return (m); 443} 444 445void 446ifclassq_update(struct ifclassq *ifq, cqev_t ev) 447{ 448 IFCQ_LOCK_ASSERT_HELD(ifq); 449 VERIFY(IFCQ_IS_READY(ifq)); 450 451#if PF_ALTQ 452 if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) 453 ALTQ_UPDATE(IFCQ_ALTQ(ifq), ev); 454#endif /* PF_ALTQ */ 455 IFCQ_UPDATE(ifq, ev); 456} 457 458int 459ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline, 460 ifclassq_enq_func enqueue, ifclassq_deq_func dequeue, 461 ifclassq_deq_sc_func dequeue_sc, ifclassq_req_func request) 462{ 463 IFCQ_LOCK_ASSERT_HELD(ifq); 464 465 VERIFY(ifq->ifcq_disc == NULL); 466 VERIFY(enqueue != NULL); 467 VERIFY(!(dequeue != NULL && dequeue_sc != NULL)); 468 VERIFY(request != NULL); 469 470 ifq->ifcq_type = type; 471 ifq->ifcq_disc = discipline; 472 ifq->ifcq_enqueue = enqueue; 473 ifq->ifcq_dequeue = dequeue; 474 ifq->ifcq_dequeue_sc = dequeue_sc; 475 ifq->ifcq_request = request; 476 477 return (0); 478} 479 480int 481ifclassq_detach(struct ifclassq *ifq) 482{ 483 IFCQ_LOCK_ASSERT_HELD(ifq); 484 485 VERIFY(ifq->ifcq_disc == NULL); 486 487 ifq->ifcq_type = PKTSCHEDT_NONE; 488 ifq->ifcq_disc = NULL; 489 ifq->ifcq_enqueue = NULL; 490 ifq->ifcq_dequeue = NULL; 491 ifq->ifcq_dequeue_sc = NULL; 492 ifq->ifcq_request = NULL; 493 494 return (0); 495} 496 497int 498ifclassq_getqstats(struct ifclassq *ifq, u_int32_t qid, void *ubuf, 499 u_int32_t *nbytes) 500{ 501 struct if_ifclassq_stats *ifqs; 502 int err; 503 504 if (*nbytes < sizeof (*ifqs)) 505 return (EINVAL); 506 507 ifqs = _MALLOC(sizeof (*ifqs), M_TEMP, M_WAITOK | M_ZERO); 508 if (ifqs == NULL) 509 return (ENOMEM); 510 511 IFCQ_LOCK(ifq); 512 if (!IFCQ_IS_READY(ifq)) { 513 IFCQ_UNLOCK(ifq); 514 _FREE(ifqs, M_TEMP); 515 return (ENXIO); 516 } 517 518 ifqs->ifqs_len = IFCQ_LEN(ifq); 519 ifqs->ifqs_maxlen = IFCQ_MAXLEN(ifq); 520 *(&ifqs->ifqs_xmitcnt) = *(&ifq->ifcq_xmitcnt); 521 *(&ifqs->ifqs_dropcnt) = *(&ifq->ifcq_dropcnt); 522 ifqs->ifqs_scheduler = ifq->ifcq_type; 523 524 err = pktsched_getqstats(ifq, qid, ifqs); 525 IFCQ_UNLOCK(ifq); 526 527 if (err == 0 && (err = copyout((caddr_t)ifqs, 528 (user_addr_t)(uintptr_t)ubuf, sizeof (*ifqs))) == 0) 529 *nbytes = sizeof (*ifqs); 530 531 _FREE(ifqs, M_TEMP); 532 533 return (err); 534} 535 536const char * 537ifclassq_ev2str(cqev_t ev) 538{ 539 const char *c; 540 541 switch (ev) { 542 case CLASSQ_EV_LINK_SPEED: 543 c = "LINK_SPEED"; 544 break; 545 546 case CLASSQ_EV_LINK_MTU: 547 c = "LINK_MTU"; 548 break; 549 550 case CLASSQ_EV_LINK_UP: 551 c = "LINK_UP"; 552 break; 553 554 case CLASSQ_EV_LINK_DOWN: 555 c = "LINK_DOWN"; 556 break; 557 558 default: 559 c = "UNKNOWN"; 560 break; 561 } 562 563 return (c); 564} 565 566/* 567 * internal representation of token bucket parameters 568 * rate: byte_per_unittime << 32 569 * (((bits_per_sec) / 8) << 32) / machclk_freq 570 * depth: byte << 32 571 * 572 */ 573#define TBR_SHIFT 32 574#define TBR_SCALE(x) ((int64_t)(x) << TBR_SHIFT) 575#define TBR_UNSCALE(x) ((x) >> TBR_SHIFT) 576 577struct mbuf * 578ifclassq_tbr_dequeue(struct ifclassq *ifq, int op) 579{ 580 return (ifclassq_tbr_dequeue_common(ifq, op, MBUF_SC_UNSPEC, FALSE)); 581} 582 583struct mbuf * 584ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, int op, mbuf_svc_class_t sc) 585{ 586 return (ifclassq_tbr_dequeue_common(ifq, op, sc, TRUE)); 587} 588 589static struct mbuf * 590ifclassq_tbr_dequeue_common(struct ifclassq *ifq, int op, 591 mbuf_svc_class_t sc, boolean_t drvmgt) 592{ 593 struct tb_regulator *tbr; 594 struct mbuf *m; 595 int64_t interval; 596 u_int64_t now; 597 598 IFCQ_LOCK_ASSERT_HELD(ifq); 599 600 VERIFY(!drvmgt || MBUF_VALID_SC(sc)); 601 VERIFY(IFCQ_TBR_IS_ENABLED(ifq)); 602 603 tbr = &ifq->ifcq_tbr; 604 if (op == CLASSQDQ_REMOVE && tbr->tbr_lastop == CLASSQDQ_POLL) { 605 /* if this is a remove after poll, bypass tbr check */ 606 } else { 607 /* update token only when it is negative */ 608 if (tbr->tbr_token <= 0) { 609 now = read_machclk(); 610 interval = now - tbr->tbr_last; 611 if (interval >= tbr->tbr_filluptime) { 612 tbr->tbr_token = tbr->tbr_depth; 613 } else { 614 tbr->tbr_token += interval * tbr->tbr_rate; 615 if (tbr->tbr_token > tbr->tbr_depth) 616 tbr->tbr_token = tbr->tbr_depth; 617 } 618 tbr->tbr_last = now; 619 } 620 /* if token is still negative, don't allow dequeue */ 621 if (tbr->tbr_token <= 0) 622 return (NULL); 623 } 624 625 /* 626 * ifclassq takes precedence over ALTQ queue; 627 * ifcq_drain count is adjusted by the caller. 628 */ 629#if PF_ALTQ 630 if (IFCQ_IS_DRAINING(ifq)) { 631#endif /* PF_ALTQ */ 632 if (op == CLASSQDQ_POLL) { 633 if (drvmgt) 634 IFCQ_POLL_SC(ifq, sc, m); 635 else 636 IFCQ_POLL(ifq, m); 637 } else { 638 if (drvmgt) 639 IFCQ_DEQUEUE_SC(ifq, sc, m); 640 else 641 IFCQ_DEQUEUE(ifq, m); 642 } 643#if PF_ALTQ 644 } else { 645 struct ifaltq *altq = IFCQ_ALTQ(ifq); 646 if (ALTQ_IS_ENABLED(altq)) { 647 if (drvmgt) 648 m = (*altq->altq_dequeue_sc)(altq, sc, op); 649 else 650 m = (*altq->altq_dequeue)(altq, op); 651 } else { 652 m = NULL; 653 } 654 } 655#endif /* PF_ALTQ */ 656 657 if (m != NULL && op == CLASSQDQ_REMOVE) 658 tbr->tbr_token -= TBR_SCALE(m_pktlen(m)); 659 tbr->tbr_lastop = op; 660 661 return (m); 662} 663 664/* 665 * set a token bucket regulator. 666 * if the specified rate is zero, the token bucket regulator is deleted. 667 */ 668int 669ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile, 670 boolean_t update) 671{ 672 struct tb_regulator *tbr; 673 struct ifnet *ifp = ifq->ifcq_ifp; 674 u_int64_t rate, old_rate; 675 676 IFCQ_LOCK_ASSERT_HELD(ifq); 677 VERIFY(IFCQ_IS_READY(ifq)); 678 679 VERIFY(machclk_freq != 0); 680 681 tbr = &ifq->ifcq_tbr; 682 old_rate = tbr->tbr_rate_raw; 683 684 rate = profile->rate; 685 if (profile->percent > 0) { 686 u_int64_t eff_rate; 687 688 if (profile->percent > 100) 689 return (EINVAL); 690 if ((eff_rate = ifp->if_output_bw.eff_bw) == 0) 691 return (ENODEV); 692 rate = (eff_rate * profile->percent) / 100; 693 } 694 695 if (rate == 0) { 696 if (!IFCQ_TBR_IS_ENABLED(ifq)) 697 return (ENOENT); 698 699 if (pktsched_verbose) 700 printf("%s: TBR disabled\n", if_name(ifp)); 701 702 /* disable this TBR */ 703 ifq->ifcq_flags &= ~IFCQF_TBR; 704 bzero(tbr, sizeof (*tbr)); 705 ifnet_set_start_cycle(ifp, NULL); 706 if (update) 707 ifclassq_update(ifq, CLASSQ_EV_LINK_SPEED); 708 return (0); 709 } 710 711 if (pktsched_verbose) { 712 printf("%s: TBR %s (rate %llu bps depth %u)\n", if_name(ifp), 713 (ifq->ifcq_flags & IFCQF_TBR) ? "reconfigured" : 714 "enabled", rate, profile->depth); 715 } 716 717 /* set the new TBR */ 718 bzero(tbr, sizeof (*tbr)); 719 tbr->tbr_rate_raw = rate; 720 tbr->tbr_percent = profile->percent; 721 ifq->ifcq_flags |= IFCQF_TBR; 722 723 /* 724 * Note that the TBR fill up time (hence the ifnet restart time) 725 * is directly related to the specified TBR depth. The ideal 726 * depth value should be computed such that the interval time 727 * between each successive wakeup is adequately spaced apart, 728 * in order to reduce scheduling overheads. A target interval 729 * of 10 ms seems to provide good performance balance. This can be 730 * overridden by specifying the depth profile. Values smaller than 731 * the ideal depth will reduce delay at the expense of CPU cycles. 732 */ 733 tbr->tbr_rate = TBR_SCALE(rate / 8) / machclk_freq; 734 if (tbr->tbr_rate > 0) { 735 u_int32_t mtu = ifp->if_mtu; 736 int64_t ival, idepth = 0; 737 int i; 738 739 if (mtu < IF_MINMTU) 740 mtu = IF_MINMTU; 741 742 ival = pktsched_nsecs_to_abstime(10 * NSEC_PER_MSEC); /* 10ms */ 743 744 for (i = 1; ; i++) { 745 idepth = TBR_SCALE(i * mtu); 746 if ((idepth / tbr->tbr_rate) > ival) 747 break; 748 } 749 VERIFY(idepth > 0); 750 751 tbr->tbr_depth = TBR_SCALE(profile->depth); 752 if (tbr->tbr_depth == 0) { 753 tbr->tbr_filluptime = idepth / tbr->tbr_rate; 754 /* a little fudge factor to get closer to rate */ 755 tbr->tbr_depth = idepth + (idepth >> 3); 756 } else { 757 tbr->tbr_filluptime = tbr->tbr_depth / tbr->tbr_rate; 758 } 759 } else { 760 tbr->tbr_depth = TBR_SCALE(profile->depth); 761 tbr->tbr_filluptime = 0xffffffffffffffffLL; 762 } 763 tbr->tbr_token = tbr->tbr_depth; 764 tbr->tbr_last = read_machclk(); 765 tbr->tbr_lastop = CLASSQDQ_REMOVE; 766 767 if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) { 768 struct timespec ts = 769 { 0, pktsched_abs_to_nsecs(tbr->tbr_filluptime) }; 770 if (pktsched_verbose) { 771 printf("%s: TBR calculated tokens %lld " 772 "filluptime %llu ns\n", if_name(ifp), 773 TBR_UNSCALE(tbr->tbr_token), 774 pktsched_abs_to_nsecs(tbr->tbr_filluptime)); 775 } 776 ifnet_set_start_cycle(ifp, &ts); 777 } else { 778 if (pktsched_verbose) { 779 if (tbr->tbr_rate == 0) { 780 printf("%s: TBR calculated tokens %lld " 781 "infinite filluptime\n", if_name(ifp), 782 TBR_UNSCALE(tbr->tbr_token)); 783 } else if (!(ifp->if_flags & IFF_UP)) { 784 printf("%s: TBR suspended (link is down)\n", 785 if_name(ifp)); 786 } 787 } 788 ifnet_set_start_cycle(ifp, NULL); 789 } 790 if (update && tbr->tbr_rate_raw != old_rate) 791 ifclassq_update(ifq, CLASSQ_EV_LINK_SPEED); 792 793 return (0); 794} 795