1/* 2 * Copyright (c) 2004-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#define __KPI__ 30//#include <sys/kpi_interface.h> 31 32#include <sys/param.h> 33#include <sys/mbuf.h> 34#include <sys/mcache.h> 35#include <sys/socket.h> 36#include <kern/debug.h> 37#include <libkern/OSAtomic.h> 38#include <kern/kalloc.h> 39#include <string.h> 40#include <netinet/in.h> 41#include <netinet/ip_var.h> 42 43#include "net/net_str_id.h" 44 45/* mbuf flags visible to KPI clients; do not add private flags here */ 46static const mbuf_flags_t mbuf_flags_mask = (MBUF_EXT | MBUF_PKTHDR | MBUF_EOR | 47 MBUF_LOOP | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG | 48 MBUF_LASTFRAG | MBUF_PROMISC | MBUF_HASFCS); 49 50/* Unalterable mbuf flags */ 51static const mbuf_flags_t mbuf_cflags_mask = (MBUF_EXT); 52 53void* mbuf_data(mbuf_t mbuf) 54{ 55 return mbuf->m_data; 56} 57 58void* mbuf_datastart(mbuf_t mbuf) 59{ 60 if (mbuf->m_flags & M_EXT) 61 return mbuf->m_ext.ext_buf; 62 if (mbuf->m_flags & M_PKTHDR) 63 return mbuf->m_pktdat; 64 return mbuf->m_dat; 65} 66 67errno_t mbuf_setdata(mbuf_t mbuf, void* data, size_t len) 68{ 69 size_t start = (size_t)((char*)mbuf_datastart(mbuf)); 70 size_t maxlen = mbuf_maxlen(mbuf); 71 72 if ((size_t)data < start || ((size_t)data) + len > start + maxlen) 73 return EINVAL; 74 mbuf->m_data = data; 75 mbuf->m_len = len; 76 77 return 0; 78} 79 80errno_t mbuf_align_32(mbuf_t mbuf, size_t len) 81{ 82 if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf)) 83 return ENOTSUP; 84 mbuf->m_data = mbuf_datastart(mbuf); 85 mbuf->m_data += ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1)); 86 87 return 0; 88} 89 90/* This function is used to provide mcl_to_paddr via symbol indirection, 91 * please avoid any change in behavior or remove the indirection in 92 * config/Unsupported* 93 */ 94addr64_t mbuf_data_to_physical(void* ptr) 95{ 96 return ((addr64_t)mcl_to_paddr(ptr)); 97} 98 99errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) 100{ 101 /* Must set *mbuf to NULL in failure case */ 102 *mbuf = m_get(how, type); 103 104 return (*mbuf == NULL) ? ENOMEM : 0; 105} 106 107errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) 108{ 109 /* Must set *mbuf to NULL in failure case */ 110 *mbuf = m_gethdr(how, type); 111 112 return (*mbuf == NULL) ? ENOMEM : 0; 113} 114 115errno_t 116mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf, 117 caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t), 118 size_t extsize, caddr_t extarg) 119{ 120 if (mbuf == NULL || extbuf == NULL || extfree == NULL || extsize == 0) 121 return (EINVAL); 122 123 if ((*mbuf = m_clattach(*mbuf, type, extbuf, 124 extfree, extsize, extarg, how)) == NULL) 125 return (ENOMEM); 126 127 return (0); 128} 129 130errno_t 131mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr) 132{ 133 if (size == NULL || *size == 0 || addr == NULL) 134 return (EINVAL); 135 136 *addr = NULL; 137 138 /* Jumbo cluster pool not available? */ 139 if (*size > MBIGCLBYTES && njcl == 0) 140 return (ENOTSUP); 141 142 if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL) 143 *size = MCLBYTES; 144 else if (*size > MCLBYTES && *size <= MBIGCLBYTES && 145 (*addr = m_bigalloc(how)) != NULL) 146 *size = MBIGCLBYTES; 147 else if (*size > MBIGCLBYTES && *size <= M16KCLBYTES && 148 (*addr = m_16kalloc(how)) != NULL) 149 *size = M16KCLBYTES; 150 else 151 *size = 0; 152 153 if (*addr == NULL) 154 return (ENOMEM); 155 156 return (0); 157} 158 159void 160mbuf_freecluster(caddr_t addr, size_t size) 161{ 162 if (size != MCLBYTES && size != MBIGCLBYTES && size != M16KCLBYTES) 163 panic("%s: invalid size (%ld) for cluster %p", __func__, 164 size, (void *)addr); 165 166 if (size == MCLBYTES) 167 m_mclfree(addr); 168 else if (size == MBIGCLBYTES) 169 m_bigfree(addr, MBIGCLBYTES, NULL); 170 else if (njcl > 0) 171 m_16kfree(addr, M16KCLBYTES, NULL); 172 else 173 panic("%s: freeing jumbo cluster to an empty pool", __func__); 174} 175 176errno_t 177mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf) 178{ 179 /* Must set *mbuf to NULL in failure case */ 180 errno_t error = 0; 181 int created = 0; 182 183 if (mbuf == NULL) 184 return EINVAL; 185 if (*mbuf == NULL) { 186 *mbuf = m_get(how, type); 187 if (*mbuf == NULL) 188 return ENOMEM; 189 created = 1; 190 } 191 /* 192 * At the time this code was written, m_{mclget,mbigget,m16kget} 193 * would always return the same value that was passed in to it. 194 */ 195 if (size == MCLBYTES) { 196 *mbuf = m_mclget(*mbuf, how); 197 } else if (size == MBIGCLBYTES) { 198 *mbuf = m_mbigget(*mbuf, how); 199 } else if (size == M16KCLBYTES) { 200 if (njcl > 0) { 201 *mbuf = m_m16kget(*mbuf, how); 202 } else { 203 /* Jumbo cluster pool not available? */ 204 error = ENOTSUP; 205 goto out; 206 } 207 } else { 208 error = EINVAL; 209 goto out; 210 } 211 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0) 212 error = ENOMEM; 213out: 214 if (created && error != 0) { 215 mbuf_free(*mbuf); 216 *mbuf = NULL; 217 } 218 return error; 219} 220 221errno_t mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) 222{ 223 /* Must set *mbuf to NULL in failure case */ 224 errno_t error = 0; 225 int created = 0; 226 if (mbuf == NULL) return EINVAL; 227 if (*mbuf == NULL) { 228 error = mbuf_get(how, type, mbuf); 229 if (error) 230 return error; 231 created = 1; 232 } 233 234 /* 235 * At the time this code was written, m_mclget would always 236 * return the same value that was passed in to it. 237 */ 238 *mbuf = m_mclget(*mbuf, how); 239 240 if (created && ((*mbuf)->m_flags & M_EXT) == 0) { 241 mbuf_free(*mbuf); 242 *mbuf = NULL; 243 } 244 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0) 245 error = ENOMEM; 246 return error; 247} 248 249 250errno_t mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf) 251{ 252 /* Must set *mbuf to NULL in failure case */ 253 errno_t error = 0; 254 255 *mbuf = m_getpacket_how(how); 256 257 if (*mbuf == NULL) { 258 if (how == MBUF_WAITOK) 259 error = ENOMEM; 260 else 261 error = EWOULDBLOCK; 262 } 263 264 return error; 265} 266 267/* This function is used to provide m_free via symbol indirection, please avoid 268 * any change in behavior or remove the indirection in config/Unsupported* 269 */ 270mbuf_t mbuf_free(mbuf_t mbuf) 271{ 272 return m_free(mbuf); 273} 274 275/* This function is used to provide m_freem via symbol indirection, please avoid 276 * any change in behavior or remove the indirection in config/Unsupported* 277 */ 278void mbuf_freem(mbuf_t mbuf) 279{ 280 m_freem(mbuf); 281} 282 283int mbuf_freem_list(mbuf_t mbuf) 284{ 285 return m_freem_list(mbuf); 286} 287 288size_t mbuf_leadingspace(const mbuf_t mbuf) 289{ 290 return m_leadingspace(mbuf); 291} 292 293/* This function is used to provide m_trailingspace via symbol indirection, 294 * please avoid any change in behavior or remove the indirection in 295 * config/Unsupported* 296 */ 297size_t mbuf_trailingspace(const mbuf_t mbuf) 298{ 299 return m_trailingspace(mbuf); 300} 301 302/* Manipulation */ 303errno_t mbuf_copym(const mbuf_t src, size_t offset, size_t len, 304 mbuf_how_t how, mbuf_t *new_mbuf) 305{ 306 /* Must set *mbuf to NULL in failure case */ 307 *new_mbuf = m_copym(src, offset, len, how); 308 309 return (*new_mbuf == NULL) ? ENOMEM : 0; 310} 311 312errno_t mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf) 313{ 314 /* Must set *new_mbuf to NULL in failure case */ 315 *new_mbuf = m_dup(src, how); 316 317 return (*new_mbuf == NULL) ? ENOMEM : 0; 318} 319 320errno_t mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how) 321{ 322 /* Must set *orig to NULL in failure case */ 323 *orig = m_prepend_2(*orig, len, how); 324 325 return (*orig == NULL) ? ENOMEM : 0; 326} 327 328errno_t mbuf_split(mbuf_t src, size_t offset, 329 mbuf_how_t how, mbuf_t *new_mbuf) 330{ 331 /* Must set *new_mbuf to NULL in failure case */ 332 *new_mbuf = m_split(src, offset, how); 333 334 return (*new_mbuf == NULL) ? ENOMEM : 0; 335} 336 337errno_t mbuf_pullup(mbuf_t *mbuf, size_t len) 338{ 339 /* Must set *mbuf to NULL in failure case */ 340 *mbuf = m_pullup(*mbuf, len); 341 342 return (*mbuf == NULL) ? ENOMEM : 0; 343} 344 345errno_t mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location) 346{ 347 /* Must set *location to NULL in failure case */ 348 int new_offset; 349 *location = m_pulldown(src, *offset, len, &new_offset); 350 *offset = new_offset; 351 352 return (*location == NULL) ? ENOMEM : 0; 353} 354 355/* This function is used to provide m_adj via symbol indirection, please avoid 356 * any change in behavior or remove the indirection in config/Unsupported* 357 */ 358void mbuf_adj(mbuf_t mbuf, int len) 359{ 360 m_adj(mbuf, len); 361} 362 363errno_t mbuf_adjustlen(mbuf_t m, int amount) 364{ 365 /* Verify m_len will be valid after adding amount */ 366 if (amount > 0) { 367 int used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) + 368 m->m_len; 369 370 if ((size_t)(amount + used) > mbuf_maxlen(m)) 371 return EINVAL; 372 } 373 else if (-amount > m->m_len) { 374 return EINVAL; 375 } 376 377 m->m_len += amount; 378 return 0; 379} 380 381mbuf_t 382mbuf_concatenate(mbuf_t dst, mbuf_t src) 383{ 384 if (dst == NULL) 385 return (NULL); 386 387 m_cat(dst, src); 388 389 /* return dst as is in the current implementation */ 390 return (dst); 391} 392errno_t mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void* out_data) 393{ 394 /* Copied m_copydata, added error handling (don't just panic) */ 395 int count; 396 mbuf_t m = m0; 397 398 while (off > 0) { 399 if (m == 0) 400 return EINVAL; 401 if (off < (size_t)m->m_len) 402 break; 403 off -= m->m_len; 404 m = m->m_next; 405 } 406 while (len > 0) { 407 if (m == 0) 408 return EINVAL; 409 count = m->m_len - off > len ? len : m->m_len - off; 410 bcopy(mtod(m, caddr_t) + off, out_data, count); 411 len -= count; 412 out_data = ((char*)out_data) + count; 413 off = 0; 414 m = m->m_next; 415 } 416 417 return 0; 418} 419 420int mbuf_mclhasreference(mbuf_t mbuf) 421{ 422 if ((mbuf->m_flags & M_EXT)) 423 return m_mclhasreference(mbuf); 424 else 425 return 0; 426} 427 428 429/* mbuf header */ 430mbuf_t mbuf_next(const mbuf_t mbuf) 431{ 432 return mbuf->m_next; 433} 434 435errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next) 436{ 437 if (next && ((next)->m_nextpkt != NULL || 438 (next)->m_type == MT_FREE)) return EINVAL; 439 mbuf->m_next = next; 440 441 return 0; 442} 443 444mbuf_t mbuf_nextpkt(const mbuf_t mbuf) 445{ 446 return mbuf->m_nextpkt; 447} 448 449void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt) 450{ 451 mbuf->m_nextpkt = nextpkt; 452} 453 454size_t mbuf_len(const mbuf_t mbuf) 455{ 456 return mbuf->m_len; 457} 458 459void mbuf_setlen(mbuf_t mbuf, size_t len) 460{ 461 mbuf->m_len = len; 462} 463 464size_t mbuf_maxlen(const mbuf_t mbuf) 465{ 466 if (mbuf->m_flags & M_EXT) 467 return mbuf->m_ext.ext_size; 468 return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf)); 469} 470 471mbuf_type_t mbuf_type(const mbuf_t mbuf) 472{ 473 return mbuf->m_type; 474} 475 476errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type) 477{ 478 if (new_type == MBUF_TYPE_FREE) return EINVAL; 479 480 m_mchtype(mbuf, new_type); 481 482 return 0; 483} 484 485mbuf_flags_t 486mbuf_flags(const mbuf_t mbuf) 487{ 488 return (mbuf->m_flags & mbuf_flags_mask); 489} 490 491errno_t 492mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags) 493{ 494 errno_t ret = 0; 495 mbuf_flags_t oflags = mbuf->m_flags; 496 497 /* 498 * 1. Return error if public but un-alterable flags are changed 499 * in flags argument. 500 * 2. Return error if bits other than public flags are set in passed 501 * flags argument. 502 * Please note that private flag bits must be passed as reset by kexts, 503 * as they must use mbuf_flags KPI to get current set of mbuf flags 504 * and mbuf_flags KPI does not expose private flags. 505 */ 506 if ((flags ^ oflags) & mbuf_cflags_mask) { 507 ret = EINVAL; 508 } else if (flags & ~mbuf_flags_mask) { 509 ret = EINVAL; 510 } else { 511 mbuf->m_flags = flags | (mbuf->m_flags & ~mbuf_flags_mask); 512 /* 513 * If M_PKTHDR bit has changed, we have work to do; 514 * m_reinit() will take care of setting/clearing the 515 * bit, as well as the rest of bookkeeping. 516 */ 517 if ((oflags ^ mbuf->m_flags) & M_PKTHDR) { 518 mbuf->m_flags ^= M_PKTHDR; /* restore */ 519 ret = m_reinit(mbuf, 520 (mbuf->m_flags & M_PKTHDR) ? 0 : 1); 521 } 522 } 523 524 return (ret); 525} 526 527errno_t 528mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask) 529{ 530 errno_t ret = 0; 531 532 if (mask & (~mbuf_flags_mask | mbuf_cflags_mask)) { 533 ret = EINVAL; 534 } else { 535 mbuf_flags_t oflags = mbuf->m_flags; 536 mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask); 537 /* 538 * If M_PKTHDR bit has changed, we have work to do; 539 * m_reinit() will take care of setting/clearing the 540 * bit, as well as the rest of bookkeeping. 541 */ 542 if ((oflags ^ mbuf->m_flags) & M_PKTHDR) { 543 mbuf->m_flags ^= M_PKTHDR; /* restore */ 544 ret = m_reinit(mbuf, 545 (mbuf->m_flags & M_PKTHDR) ? 0 : 1); 546 } 547 } 548 549 return (ret); 550} 551 552errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src) 553{ 554 if (((src)->m_flags & M_PKTHDR) == 0) 555 return EINVAL; 556 557 m_copy_pkthdr(dest, src); 558 559 return 0; 560} 561 562size_t mbuf_pkthdr_len(const mbuf_t mbuf) 563{ 564 return mbuf->m_pkthdr.len; 565} 566 567__private_extern__ size_t mbuf_pkthdr_maxlen(mbuf_t m) 568{ 569 size_t maxlen = 0; 570 mbuf_t n = m; 571 572 while (n) { 573 maxlen += mbuf_maxlen(n); 574 n = mbuf_next(n); 575 } 576 return (maxlen); 577} 578 579void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len) 580{ 581 mbuf->m_pkthdr.len = len; 582} 583 584void mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount) 585{ 586 mbuf->m_pkthdr.len += amount; 587} 588 589ifnet_t mbuf_pkthdr_rcvif(const mbuf_t mbuf) 590{ 591 // If we reference count ifnets, we should take a reference here before returning 592 return mbuf->m_pkthdr.rcvif; 593} 594 595errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet) 596{ 597 /* May want to walk ifnet list to determine if interface is valid */ 598 mbuf->m_pkthdr.rcvif = (struct ifnet*)ifnet; 599 return 0; 600} 601 602void* mbuf_pkthdr_header(const mbuf_t mbuf) 603{ 604 return mbuf->m_pkthdr.pkt_hdr; 605} 606 607void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header) 608{ 609 mbuf->m_pkthdr.pkt_hdr = (void*)header; 610} 611 612void 613mbuf_inbound_modified(mbuf_t mbuf) 614{ 615 /* Invalidate hardware generated checksum flags */ 616 mbuf->m_pkthdr.csum_flags = 0; 617} 618 619void 620mbuf_outbound_finalize(struct mbuf *m, u_int32_t pf, size_t o) 621{ 622 /* Generate the packet in software, client needs it */ 623 switch (pf) { 624 case PF_INET: 625 (void) in_finalize_cksum(m, o, m->m_pkthdr.csum_flags); 626 break; 627 628 case PF_INET6: 629#if INET6 630 /* 631 * Checksum offload should not have been enabled when 632 * extension headers exist; indicate that the callee 633 * should skip such case by setting optlen to -1. 634 */ 635 (void) in6_finalize_cksum(m, o, -1, -1, m->m_pkthdr.csum_flags); 636#endif /* INET6 */ 637 break; 638 639 default: 640 break; 641 } 642} 643 644errno_t 645mbuf_set_vlan_tag( 646 mbuf_t mbuf, 647 u_int16_t vlan) 648{ 649 mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID; 650 mbuf->m_pkthdr.vlan_tag = vlan; 651 652 return 0; 653} 654 655errno_t 656mbuf_get_vlan_tag( 657 mbuf_t mbuf, 658 u_int16_t *vlan) 659{ 660 if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) 661 return ENXIO; // No vlan tag set 662 663 *vlan = mbuf->m_pkthdr.vlan_tag; 664 665 return 0; 666} 667 668errno_t 669mbuf_clear_vlan_tag( 670 mbuf_t mbuf) 671{ 672 mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID; 673 mbuf->m_pkthdr.vlan_tag = 0; 674 675 return 0; 676} 677 678static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags = 679 MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP | 680 MBUF_CSUM_PARTIAL | MBUF_CSUM_REQ_TCPIPV6 | MBUF_CSUM_REQ_UDPIPV6; 681 682errno_t 683mbuf_set_csum_requested( 684 mbuf_t mbuf, 685 mbuf_csum_request_flags_t request, 686 u_int32_t value) 687{ 688 request &= mbuf_valid_csum_request_flags; 689 mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request; 690 mbuf->m_pkthdr.csum_data = value; 691 692 return 0; 693} 694 695static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags = 696 MBUF_TSO_IPV4 | MBUF_TSO_IPV6; 697 698errno_t 699mbuf_get_tso_requested( 700 mbuf_t mbuf, 701 mbuf_tso_request_flags_t *request, 702 u_int32_t *value) 703{ 704 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || 705 request == NULL || value == NULL) 706 return EINVAL; 707 708 *request = mbuf->m_pkthdr.csum_flags; 709 *request &= mbuf_valid_tso_request_flags; 710 if (*request && value != NULL) 711 *value = mbuf->m_pkthdr.tso_segsz; 712 713 return 0; 714} 715 716errno_t 717mbuf_get_csum_requested( 718 mbuf_t mbuf, 719 mbuf_csum_request_flags_t *request, 720 u_int32_t *value) 721{ 722 *request = mbuf->m_pkthdr.csum_flags; 723 *request &= mbuf_valid_csum_request_flags; 724 if (value != NULL) { 725 *value = mbuf->m_pkthdr.csum_data; 726 } 727 728 return 0; 729} 730 731errno_t 732mbuf_clear_csum_requested( 733 mbuf_t mbuf) 734{ 735 mbuf->m_pkthdr.csum_flags &= 0xffff0000; 736 mbuf->m_pkthdr.csum_data = 0; 737 738 return 0; 739} 740 741static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags = 742 MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA | 743 MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_PARTIAL; 744 745errno_t 746mbuf_set_csum_performed( 747 mbuf_t mbuf, 748 mbuf_csum_performed_flags_t performed, 749 u_int32_t value) 750{ 751 performed &= mbuf_valid_csum_performed_flags; 752 mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed; 753 mbuf->m_pkthdr.csum_data = value; 754 755 return 0; 756} 757 758errno_t 759mbuf_get_csum_performed( 760 mbuf_t mbuf, 761 mbuf_csum_performed_flags_t *performed, 762 u_int32_t *value) 763{ 764 *performed = mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags; 765 *value = mbuf->m_pkthdr.csum_data; 766 767 return 0; 768} 769 770errno_t 771mbuf_clear_csum_performed( 772 mbuf_t mbuf) 773{ 774 mbuf->m_pkthdr.csum_flags &= 0xffff0000; 775 mbuf->m_pkthdr.csum_data = 0; 776 777 return 0; 778} 779 780errno_t 781mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, 782 u_int16_t *csum) 783{ 784 if (mbuf == NULL || length == 0 || csum == NULL || 785 (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) 786 return (EINVAL); 787 788 *csum = inet_cksum(mbuf, protocol, offset, length); 789 return (0); 790} 791 792#if INET6 793errno_t 794mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, 795 u_int16_t *csum) 796{ 797 if (mbuf == NULL || length == 0 || csum == NULL || 798 (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) 799 return (EINVAL); 800 801 *csum = inet6_cksum(mbuf, protocol, offset, length); 802 return (0); 803} 804#else /* INET6 */ 805errno_t 806mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol, 807 __unused u_int32_t offset, __unused u_int32_t length, 808 __unused u_int16_t *csum) 809{ 810 panic("mbuf_inet6_cksum() doesn't exist on this platform\n"); 811 return (0); 812} 813 814u_int16_t 815inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt, 816 __unused unsigned int off, __unused unsigned int len) 817{ 818 panic("inet6_cksum() doesn't exist on this platform\n"); 819 return (0); 820} 821 822void nd6_lookup_ipv6(void); 823void 824nd6_lookup_ipv6(void) 825{ 826 panic("nd6_lookup_ipv6() doesn't exist on this platform\n"); 827} 828 829int 830in6addr_local(__unused struct in6_addr *a) 831{ 832 panic("in6addr_local() doesn't exist on this platform\n"); 833 return (0); 834} 835 836void nd6_storelladdr(void); 837void 838nd6_storelladdr(void) 839{ 840 panic("nd6_storelladdr() doesn't exist on this platform\n"); 841} 842#endif /* INET6 */ 843 844/* 845 * Mbuf tag KPIs 846 */ 847 848#define MTAG_FIRST_ID FIRST_KPI_STR_ID 849 850errno_t 851mbuf_tag_id_find( 852 const char *string, 853 mbuf_tag_id_t *out_id) 854{ 855 return net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1); 856} 857 858errno_t 859mbuf_tag_allocate( 860 mbuf_t mbuf, 861 mbuf_tag_id_t id, 862 mbuf_tag_type_t type, 863 size_t length, 864 mbuf_how_t how, 865 void** data_p) 866{ 867 struct m_tag *tag; 868 u_int32_t mtag_id_first, mtag_id_last; 869 870 if (data_p != NULL) 871 *data_p = NULL; 872 873 /* Sanity check parameters */ 874 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); 875 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || 876 id > mtag_id_last || length < 1 || (length & 0xffff0000) != 0 || 877 data_p == NULL) { 878 return EINVAL; 879 } 880 881 /* Make sure this mtag hasn't already been allocated */ 882 tag = m_tag_locate(mbuf, id, type, NULL); 883 if (tag != NULL) { 884 return EEXIST; 885 } 886 887 /* Allocate an mtag */ 888 tag = m_tag_create(id, type, length, how, mbuf); 889 if (tag == NULL) { 890 return how == M_WAITOK ? ENOMEM : EWOULDBLOCK; 891 } 892 893 /* Attach the mtag and set *data_p */ 894 m_tag_prepend(mbuf, tag); 895 *data_p = tag + 1; 896 897 return 0; 898} 899 900errno_t 901mbuf_tag_find( 902 mbuf_t mbuf, 903 mbuf_tag_id_t id, 904 mbuf_tag_type_t type, 905 size_t* length, 906 void** data_p) 907{ 908 struct m_tag *tag; 909 u_int32_t mtag_id_first, mtag_id_last; 910 911 if (length != NULL) 912 *length = 0; 913 if (data_p != NULL) 914 *data_p = NULL; 915 916 /* Sanity check parameters */ 917 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); 918 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || 919 id > mtag_id_last || length == NULL || data_p == NULL) { 920 return EINVAL; 921 } 922 923 /* Locate an mtag */ 924 tag = m_tag_locate(mbuf, id, type, NULL); 925 if (tag == NULL) { 926 return ENOENT; 927 } 928 929 /* Copy out the pointer to the data and the lenght value */ 930 *length = tag->m_tag_len; 931 *data_p = tag + 1; 932 933 return 0; 934} 935 936void 937mbuf_tag_free( 938 mbuf_t mbuf, 939 mbuf_tag_id_t id, 940 mbuf_tag_type_t type) 941{ 942 struct m_tag *tag; 943 u_int32_t mtag_id_first, mtag_id_last; 944 945 /* Sanity check parameters */ 946 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); 947 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || 948 id > mtag_id_last) 949 return; 950 951 tag = m_tag_locate(mbuf, id, type, NULL); 952 if (tag == NULL) { 953 return; 954 } 955 956 m_tag_delete(mbuf, tag); 957 return; 958} 959 960/* 961 * Maximum length of driver auxiliary data; keep this small to 962 * fit in a single mbuf to avoid wasting memory, rounded down to 963 * the nearest 64-bit boundary. This takes into account mbuf 964 * tag-related (m_taghdr + m_tag) as well m_drvaux_tag structs. 965 */ 966#define MBUF_DRVAUX_MAXLEN \ 967 P2ROUNDDOWN(MLEN - sizeof (struct m_taghdr) - \ 968 M_TAG_ALIGN(sizeof (struct m_drvaux_tag)), sizeof (uint64_t)) 969 970errno_t 971mbuf_add_drvaux(mbuf_t mbuf, mbuf_how_t how, u_int32_t family, 972 u_int32_t subfamily, size_t length, void **data_p) 973{ 974 struct m_drvaux_tag *p; 975 struct m_tag *tag; 976 977 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) || 978 length == 0 || length > MBUF_DRVAUX_MAXLEN) 979 return (EINVAL); 980 981 if (data_p != NULL) 982 *data_p = NULL; 983 984 /* Check if one is already associated */ 985 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID, 986 KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL) 987 return (EEXIST); 988 989 /* Tag is (m_drvaux_tag + module specific data) */ 990 if ((tag = m_tag_create(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_DRVAUX, 991 sizeof (*p) + length, how, mbuf)) == NULL) 992 return ((how == MBUF_WAITOK) ? ENOMEM : EWOULDBLOCK); 993 994 p = (struct m_drvaux_tag *)(tag + 1); 995 p->da_family = family; 996 p->da_subfamily = subfamily; 997 p->da_length = length; 998 999 /* Associate the tag */ 1000 m_tag_prepend(mbuf, tag); 1001 1002 if (data_p != NULL) 1003 *data_p = (p + 1); 1004 1005 return (0); 1006} 1007 1008errno_t 1009mbuf_find_drvaux(mbuf_t mbuf, u_int32_t *family_p, u_int32_t *subfamily_p, 1010 u_int32_t *length_p, void **data_p) 1011{ 1012 struct m_drvaux_tag *p; 1013 struct m_tag *tag; 1014 1015 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) || data_p == NULL) 1016 return (EINVAL); 1017 1018 *data_p = NULL; 1019 1020 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID, 1021 KERNEL_TAG_TYPE_DRVAUX, NULL)) == NULL) 1022 return (ENOENT); 1023 1024 /* Must be at least size of m_drvaux_tag */ 1025 VERIFY(tag->m_tag_len >= sizeof (*p)); 1026 1027 p = (struct m_drvaux_tag *)(tag + 1); 1028 VERIFY(p->da_length > 0 && p->da_length <= MBUF_DRVAUX_MAXLEN); 1029 1030 if (family_p != NULL) 1031 *family_p = p->da_family; 1032 if (subfamily_p != NULL) 1033 *subfamily_p = p->da_subfamily; 1034 if (length_p != NULL) 1035 *length_p = p->da_length; 1036 1037 *data_p = (p + 1); 1038 1039 return (0); 1040} 1041 1042void 1043mbuf_del_drvaux(mbuf_t mbuf) 1044{ 1045 struct m_tag *tag; 1046 1047 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR)) 1048 return; 1049 1050 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID, 1051 KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL) 1052 m_tag_delete(mbuf, tag); 1053} 1054 1055/* mbuf stats */ 1056void mbuf_stats(struct mbuf_stat *stats) 1057{ 1058 stats->mbufs = mbstat.m_mbufs; 1059 stats->clusters = mbstat.m_clusters; 1060 stats->clfree = mbstat.m_clfree; 1061 stats->drops = mbstat.m_drops; 1062 stats->wait = mbstat.m_wait; 1063 stats->drain = mbstat.m_drain; 1064 __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes)); 1065 stats->mcfail = mbstat.m_mcfail; 1066 stats->mpfail = mbstat.m_mpfail; 1067 stats->msize = mbstat.m_msize; 1068 stats->mclbytes = mbstat.m_mclbytes; 1069 stats->minclsize = mbstat.m_minclsize; 1070 stats->mlen = mbstat.m_mlen; 1071 stats->mhlen = mbstat.m_mhlen; 1072 stats->bigclusters = mbstat.m_bigclusters; 1073 stats->bigclfree = mbstat.m_bigclfree; 1074 stats->bigmclbytes = mbstat.m_bigmclbytes; 1075} 1076 1077errno_t 1078mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf) 1079{ 1080 errno_t error; 1081 struct mbuf *m; 1082 unsigned int numpkts = 1; 1083 unsigned int numchunks = maxchunks ? *maxchunks : 0; 1084 1085 if (packetlen == 0) { 1086 error = EINVAL; 1087 goto out; 1088 } 1089 m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0); 1090 if (m == 0) { 1091 if (maxchunks && *maxchunks && numchunks > *maxchunks) 1092 error = ENOBUFS; 1093 else 1094 error = ENOMEM; 1095 } else { 1096 if (maxchunks) 1097 *maxchunks = numchunks; 1098 error = 0; 1099 *mbuf = m; 1100 } 1101out: 1102 return error; 1103} 1104 1105errno_t 1106mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf) 1107{ 1108 errno_t error; 1109 struct mbuf *m; 1110 unsigned int numchunks = maxchunks ? *maxchunks : 0; 1111 1112 if (numpkts == 0) { 1113 error = EINVAL; 1114 goto out; 1115 } 1116 if (packetlen == 0) { 1117 error = EINVAL; 1118 goto out; 1119 } 1120 m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0); 1121 if (m == 0) { 1122 if (maxchunks && *maxchunks && numchunks > *maxchunks) 1123 error = ENOBUFS; 1124 else 1125 error = ENOMEM; 1126 } else { 1127 if (maxchunks) 1128 *maxchunks = numchunks; 1129 error = 0; 1130 *mbuf = m; 1131 } 1132out: 1133 return error; 1134} 1135 1136__private_extern__ size_t 1137mbuf_pkt_list_len(mbuf_t m) 1138{ 1139 size_t len = 0; 1140 mbuf_t n = m; 1141 1142 while (n) { 1143 len += mbuf_pkthdr_len(n); 1144 n = mbuf_nextpkt(n); 1145 } 1146 return (len); 1147} 1148 1149__private_extern__ size_t 1150mbuf_pkt_list_maxlen(mbuf_t m) 1151{ 1152 size_t maxlen = 0; 1153 mbuf_t n = m; 1154 1155 while (n) { 1156 maxlen += mbuf_pkthdr_maxlen(n); 1157 n = mbuf_nextpkt(n); 1158 } 1159 return (maxlen); 1160} 1161 1162/* 1163 * mbuf_copyback differs from m_copyback in a few ways: 1164 * 1) mbuf_copyback will allocate clusters for new mbufs we append 1165 * 2) mbuf_copyback will grow the last mbuf in the chain if possible 1166 * 3) mbuf_copyback reports whether or not the operation succeeded 1167 * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT 1168 */ 1169errno_t 1170mbuf_copyback( 1171 mbuf_t m, 1172 size_t off, 1173 size_t len, 1174 const void *data, 1175 mbuf_how_t how) 1176{ 1177 size_t mlen; 1178 mbuf_t m_start = m; 1179 mbuf_t n; 1180 int totlen = 0; 1181 errno_t result = 0; 1182 const char *cp = data; 1183 1184 if (m == NULL || len == 0 || data == NULL) 1185 return EINVAL; 1186 1187 while (off > (mlen = m->m_len)) { 1188 off -= mlen; 1189 totlen += mlen; 1190 if (m->m_next == 0) { 1191 n = m_getclr(how, m->m_type); 1192 if (n == 0) { 1193 result = ENOBUFS; 1194 goto out; 1195 } 1196 n->m_len = MIN(MLEN, len + off); 1197 m->m_next = n; 1198 } 1199 m = m->m_next; 1200 } 1201 1202 while (len > 0) { 1203 mlen = MIN(m->m_len - off, len); 1204 if (mlen < len && m->m_next == NULL && mbuf_trailingspace(m) > 0) { 1205 size_t grow = MIN(mbuf_trailingspace(m), len - mlen); 1206 mlen += grow; 1207 m->m_len += grow; 1208 } 1209 bcopy(cp, off + (char*)mbuf_data(m), (unsigned)mlen); 1210 cp += mlen; 1211 len -= mlen; 1212 mlen += off; 1213 off = 0; 1214 totlen += mlen; 1215 if (len == 0) 1216 break; 1217 if (m->m_next == 0) { 1218 n = m_get(how, m->m_type); 1219 if (n == NULL) { 1220 result = ENOBUFS; 1221 goto out; 1222 } 1223 if (len > MINCLSIZE) { 1224 /* cluter allocation failure is okay, we can grow chain */ 1225 mbuf_mclget(how, m->m_type, &n); 1226 } 1227 n->m_len = MIN(mbuf_maxlen(n), len); 1228 m->m_next = n; 1229 } 1230 m = m->m_next; 1231 } 1232 1233out: 1234 if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen)) 1235 m_start->m_pkthdr.len = totlen; 1236 1237 return result; 1238} 1239 1240u_int32_t 1241mbuf_get_mlen(void) 1242{ 1243 return (_MLEN); 1244} 1245 1246u_int32_t 1247mbuf_get_mhlen(void) 1248{ 1249 return (_MHLEN); 1250} 1251 1252u_int32_t 1253mbuf_get_minclsize(void) 1254{ 1255 return (MHLEN + MLEN); 1256} 1257 1258u_int32_t 1259mbuf_get_traffic_class_max_count(void) 1260{ 1261 return (MBUF_TC_MAX); 1262} 1263 1264errno_t 1265mbuf_get_traffic_class_index(mbuf_traffic_class_t tc, u_int32_t *index) 1266{ 1267 if (index == NULL || (u_int32_t)tc >= MBUF_TC_MAX) 1268 return (EINVAL); 1269 1270 *index = MBUF_SCIDX(m_service_class_from_val(MBUF_TC2SCVAL(tc))); 1271 return (0); 1272} 1273 1274mbuf_traffic_class_t 1275mbuf_get_traffic_class(mbuf_t m) 1276{ 1277 if (m == NULL || !(m->m_flags & M_PKTHDR)) 1278 return (MBUF_TC_BE); 1279 1280 return (m_get_traffic_class(m)); 1281} 1282 1283errno_t 1284mbuf_set_traffic_class(mbuf_t m, mbuf_traffic_class_t tc) 1285{ 1286 if (m == NULL || !(m->m_flags & M_PKTHDR) || 1287 ((u_int32_t)tc >= MBUF_TC_MAX)) 1288 return (EINVAL); 1289 1290 return (m_set_traffic_class(m, tc)); 1291} 1292 1293int 1294mbuf_is_traffic_class_privileged(mbuf_t m) 1295{ 1296 if (m == NULL || !(m->m_flags & M_PKTHDR) || 1297 !MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) 1298 return (0); 1299 1300 return ((m->m_pkthdr.pkt_flags & PKTF_PRIO_PRIVILEGED) ? 1 : 0); 1301} 1302 1303u_int32_t 1304mbuf_get_service_class_max_count(void) 1305{ 1306 return (MBUF_SC_MAX_CLASSES); 1307} 1308 1309errno_t 1310mbuf_get_service_class_index(mbuf_svc_class_t sc, u_int32_t *index) 1311{ 1312 if (index == NULL || !MBUF_VALID_SC(sc)) 1313 return (EINVAL); 1314 1315 *index = MBUF_SCIDX(sc); 1316 return (0); 1317} 1318 1319mbuf_svc_class_t 1320mbuf_get_service_class(mbuf_t m) 1321{ 1322 if (m == NULL || !(m->m_flags & M_PKTHDR)) 1323 return (MBUF_SC_BE); 1324 1325 return (m_get_service_class(m)); 1326} 1327 1328errno_t 1329mbuf_set_service_class(mbuf_t m, mbuf_svc_class_t sc) 1330{ 1331 if (m == NULL || !(m->m_flags & M_PKTHDR)) 1332 return (EINVAL); 1333 1334 return (m_set_service_class(m, sc)); 1335} 1336 1337errno_t 1338mbuf_pkthdr_aux_flags(mbuf_t m, mbuf_pkthdr_aux_flags_t *flagsp) 1339{ 1340 u_int32_t flags; 1341 1342 if (m == NULL || !(m->m_flags & M_PKTHDR) || flagsp == NULL) 1343 return (EINVAL); 1344 1345 *flagsp = 0; 1346 flags = m->m_pkthdr.pkt_flags; 1347 if ((flags & (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR)) == 1348 (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR)) 1349 *flagsp |= MBUF_PKTAUXF_INET_RESOLVE_RTR; 1350 if ((flags & (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR)) == 1351 (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR)) 1352 *flagsp |= MBUF_PKTAUXF_INET6_RESOLVE_RTR; 1353 1354 /* These 2 flags are mutually exclusive */ 1355 VERIFY((*flagsp & 1356 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)) != 1357 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)); 1358 1359 return (0); 1360} 1361 1362errno_t 1363mbuf_get_driver_scratch(mbuf_t m, u_int8_t **area, size_t *area_len) 1364{ 1365 if (m == NULL || area == NULL || area_len == NULL || 1366 !(m->m_flags & M_PKTHDR)) 1367 return (EINVAL); 1368 1369 *area_len = m_scratch_get(m, area); 1370 return (0); 1371} 1372