1/* 2 * Copyright (c) 2004-2011 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#define __KPI__ 30//#include <sys/kpi_interface.h> 31 32#include <sys/param.h> 33#include <sys/mbuf.h> 34#include <sys/mcache.h> 35#include <sys/socket.h> 36#include <kern/debug.h> 37#include <libkern/OSAtomic.h> 38#include <kern/kalloc.h> 39#include <string.h> 40#include <netinet/in.h> 41 42#include "net/net_str_id.h" 43 44static const mbuf_flags_t mbuf_flags_mask = (MBUF_EXT | MBUF_PKTHDR | MBUF_EOR | 45 MBUF_LOOP | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG | 46 MBUF_LASTFRAG | MBUF_PROMISC | MBUF_HASFCS); 47 48#define MBUF_PKTAUXF_MASK \ 49 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR) 50 51void* mbuf_data(mbuf_t mbuf) 52{ 53 return mbuf->m_data; 54} 55 56void* mbuf_datastart(mbuf_t mbuf) 57{ 58 if (mbuf->m_flags & M_EXT) 59 return mbuf->m_ext.ext_buf; 60 if (mbuf->m_flags & M_PKTHDR) 61 return mbuf->m_pktdat; 62 return mbuf->m_dat; 63} 64 65errno_t mbuf_setdata(mbuf_t mbuf, void* data, size_t len) 66{ 67 size_t start = (size_t)((char*)mbuf_datastart(mbuf)); 68 size_t maxlen = mbuf_maxlen(mbuf); 69 70 if ((size_t)data < start || ((size_t)data) + len > start + maxlen) 71 return EINVAL; 72 mbuf->m_data = data; 73 mbuf->m_len = len; 74 75 return 0; 76} 77 78errno_t mbuf_align_32(mbuf_t mbuf, size_t len) 79{ 80 if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf)) 81 return ENOTSUP; 82 mbuf->m_data = mbuf_datastart(mbuf); 83 mbuf->m_data += ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1)); 84 85 return 0; 86} 87 88/* This function is used to provide mcl_to_paddr via symbol indirection, 89 * please avoid any change in behavior or remove the indirection in 90 * config/Unsupported* 91 */ 92addr64_t mbuf_data_to_physical(void* ptr) 93{ 94 return (addr64_t)(uintptr_t)mcl_to_paddr(ptr); 95} 96 97errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) 98{ 99 /* Must set *mbuf to NULL in failure case */ 100 *mbuf = m_get(how, type); 101 102 return (*mbuf == NULL) ? ENOMEM : 0; 103} 104 105errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) 106{ 107 /* Must set *mbuf to NULL in failure case */ 108 *mbuf = m_gethdr(how, type); 109 110 return (*mbuf == NULL) ? ENOMEM : 0; 111} 112 113errno_t 114mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf, 115 caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t), 116 size_t extsize, caddr_t extarg) 117{ 118 if (mbuf == NULL || extbuf == NULL || extfree == NULL || extsize == 0) 119 return (EINVAL); 120 121 if ((*mbuf = m_clattach(*mbuf, type, extbuf, 122 extfree, extsize, extarg, how)) == NULL) 123 return (ENOMEM); 124 125 return (0); 126} 127 128errno_t 129mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr) 130{ 131 if (size == NULL || *size == 0 || addr == NULL) 132 return (EINVAL); 133 134 *addr = NULL; 135 136 /* Jumbo cluster pool not available? */ 137 if (*size > MBIGCLBYTES && njcl == 0) 138 return (ENOTSUP); 139 140 if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL) 141 *size = MCLBYTES; 142 else if (*size > MCLBYTES && *size <= MBIGCLBYTES && 143 (*addr = m_bigalloc(how)) != NULL) 144 *size = MBIGCLBYTES; 145 else if (*size > MBIGCLBYTES && *size <= M16KCLBYTES && 146 (*addr = m_16kalloc(how)) != NULL) 147 *size = M16KCLBYTES; 148 else 149 *size = 0; 150 151 if (*addr == NULL) 152 return (ENOMEM); 153 154 return (0); 155} 156 157void 158mbuf_freecluster(caddr_t addr, size_t size) 159{ 160 if (size != MCLBYTES && size != MBIGCLBYTES && size != M16KCLBYTES) 161 panic("%s: invalid size (%ld) for cluster %p", __func__, 162 size, (void *)addr); 163 164 if (size == MCLBYTES) 165 m_mclfree(addr); 166 else if (size == MBIGCLBYTES) 167 m_bigfree(addr, MBIGCLBYTES, NULL); 168 else if (njcl > 0) 169 m_16kfree(addr, M16KCLBYTES, NULL); 170 else 171 panic("%s: freeing jumbo cluster to an empty pool", __func__); 172} 173 174errno_t 175mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf) 176{ 177 /* Must set *mbuf to NULL in failure case */ 178 errno_t error = 0; 179 int created = 0; 180 181 if (mbuf == NULL) 182 return EINVAL; 183 if (*mbuf == NULL) { 184 *mbuf = m_get(how, type); 185 if (*mbuf == NULL) 186 return ENOMEM; 187 created = 1; 188 } 189 /* 190 * At the time this code was written, m_{mclget,mbigget,m16kget} 191 * would always return the same value that was passed in to it. 192 */ 193 if (size == MCLBYTES) { 194 *mbuf = m_mclget(*mbuf, how); 195 } else if (size == MBIGCLBYTES) { 196 *mbuf = m_mbigget(*mbuf, how); 197 } else if (size == M16KCLBYTES) { 198 if (njcl > 0) { 199 *mbuf = m_m16kget(*mbuf, how); 200 } else { 201 /* Jumbo cluster pool not available? */ 202 error = ENOTSUP; 203 goto out; 204 } 205 } else { 206 error = EINVAL; 207 goto out; 208 } 209 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0) 210 error = ENOMEM; 211out: 212 if (created && error != 0) { 213 mbuf_free(*mbuf); 214 *mbuf = NULL; 215 } 216 return error; 217} 218 219errno_t mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) 220{ 221 /* Must set *mbuf to NULL in failure case */ 222 errno_t error = 0; 223 int created = 0; 224 if (mbuf == NULL) return EINVAL; 225 if (*mbuf == NULL) { 226 error = mbuf_get(how, type, mbuf); 227 if (error) 228 return error; 229 created = 1; 230 } 231 232 /* 233 * At the time this code was written, m_mclget would always 234 * return the same value that was passed in to it. 235 */ 236 *mbuf = m_mclget(*mbuf, how); 237 238 if (created && ((*mbuf)->m_flags & M_EXT) == 0) { 239 mbuf_free(*mbuf); 240 *mbuf = NULL; 241 } 242 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0) 243 error = ENOMEM; 244 return error; 245} 246 247 248errno_t mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf) 249{ 250 /* Must set *mbuf to NULL in failure case */ 251 errno_t error = 0; 252 253 *mbuf = m_getpacket_how(how); 254 255 if (*mbuf == NULL) { 256 if (how == MBUF_WAITOK) 257 error = ENOMEM; 258 else 259 error = EWOULDBLOCK; 260 } 261 262 return error; 263} 264 265/* This function is used to provide m_free via symbol indirection, please avoid 266 * any change in behavior or remove the indirection in config/Unsupported* 267 */ 268mbuf_t mbuf_free(mbuf_t mbuf) 269{ 270 return m_free(mbuf); 271} 272 273/* This function is used to provide m_freem via symbol indirection, please avoid 274 * any change in behavior or remove the indirection in config/Unsupported* 275 */ 276void mbuf_freem(mbuf_t mbuf) 277{ 278 m_freem(mbuf); 279} 280 281int mbuf_freem_list(mbuf_t mbuf) 282{ 283 return m_freem_list(mbuf); 284} 285 286size_t mbuf_leadingspace(const mbuf_t mbuf) 287{ 288 return m_leadingspace(mbuf); 289} 290 291/* This function is used to provide m_trailingspace via symbol indirection, 292 * please avoid any change in behavior or remove the indirection in 293 * config/Unsupported* 294 */ 295size_t mbuf_trailingspace(const mbuf_t mbuf) 296{ 297 return m_trailingspace(mbuf); 298} 299 300/* Manipulation */ 301errno_t mbuf_copym(const mbuf_t src, size_t offset, size_t len, 302 mbuf_how_t how, mbuf_t *new_mbuf) 303{ 304 /* Must set *mbuf to NULL in failure case */ 305 *new_mbuf = m_copym(src, offset, len, how); 306 307 return (*new_mbuf == NULL) ? ENOMEM : 0; 308} 309 310errno_t mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf) 311{ 312 /* Must set *new_mbuf to NULL in failure case */ 313 *new_mbuf = m_dup(src, how); 314 315 return (*new_mbuf == NULL) ? ENOMEM : 0; 316} 317 318errno_t mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how) 319{ 320 /* Must set *orig to NULL in failure case */ 321 *orig = m_prepend_2(*orig, len, how); 322 323 return (*orig == NULL) ? ENOMEM : 0; 324} 325 326errno_t mbuf_split(mbuf_t src, size_t offset, 327 mbuf_how_t how, mbuf_t *new_mbuf) 328{ 329 /* Must set *new_mbuf to NULL in failure case */ 330 *new_mbuf = m_split(src, offset, how); 331 332 return (*new_mbuf == NULL) ? ENOMEM : 0; 333} 334 335errno_t mbuf_pullup(mbuf_t *mbuf, size_t len) 336{ 337 /* Must set *mbuf to NULL in failure case */ 338 *mbuf = m_pullup(*mbuf, len); 339 340 return (*mbuf == NULL) ? ENOMEM : 0; 341} 342 343errno_t mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location) 344{ 345 /* Must set *location to NULL in failure case */ 346 int new_offset; 347 *location = m_pulldown(src, *offset, len, &new_offset); 348 *offset = new_offset; 349 350 return (*location == NULL) ? ENOMEM : 0; 351} 352 353/* This function is used to provide m_adj via symbol indirection, please avoid 354 * any change in behavior or remove the indirection in config/Unsupported* 355 */ 356void mbuf_adj(mbuf_t mbuf, int len) 357{ 358 m_adj(mbuf, len); 359} 360 361errno_t mbuf_adjustlen(mbuf_t m, int amount) 362{ 363 /* Verify m_len will be valid after adding amount */ 364 if (amount > 0) { 365 int used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) + 366 m->m_len; 367 368 if ((size_t)(amount + used) > mbuf_maxlen(m)) 369 return EINVAL; 370 } 371 else if (-amount > m->m_len) { 372 return EINVAL; 373 } 374 375 m->m_len += amount; 376 return 0; 377} 378 379mbuf_t 380mbuf_concatenate(mbuf_t dst, mbuf_t src) 381{ 382 if (dst == NULL) 383 return (NULL); 384 385 m_cat(dst, src); 386 387 /* return dst as is in the current implementation */ 388 return (dst); 389} 390errno_t mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void* out_data) 391{ 392 /* Copied m_copydata, added error handling (don't just panic) */ 393 int count; 394 mbuf_t m = m0; 395 396 while (off > 0) { 397 if (m == 0) 398 return EINVAL; 399 if (off < (size_t)m->m_len) 400 break; 401 off -= m->m_len; 402 m = m->m_next; 403 } 404 while (len > 0) { 405 if (m == 0) 406 return EINVAL; 407 count = m->m_len - off > len ? len : m->m_len - off; 408 bcopy(mtod(m, caddr_t) + off, out_data, count); 409 len -= count; 410 out_data = ((char*)out_data) + count; 411 off = 0; 412 m = m->m_next; 413 } 414 415 return 0; 416} 417 418int mbuf_mclhasreference(mbuf_t mbuf) 419{ 420 if ((mbuf->m_flags & M_EXT)) 421 return m_mclhasreference(mbuf); 422 else 423 return 0; 424} 425 426 427/* mbuf header */ 428mbuf_t mbuf_next(const mbuf_t mbuf) 429{ 430 return mbuf->m_next; 431} 432 433errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next) 434{ 435 if (next && ((next)->m_nextpkt != NULL || 436 (next)->m_type == MT_FREE)) return EINVAL; 437 mbuf->m_next = next; 438 439 return 0; 440} 441 442mbuf_t mbuf_nextpkt(const mbuf_t mbuf) 443{ 444 return mbuf->m_nextpkt; 445} 446 447void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt) 448{ 449 mbuf->m_nextpkt = nextpkt; 450} 451 452size_t mbuf_len(const mbuf_t mbuf) 453{ 454 return mbuf->m_len; 455} 456 457void mbuf_setlen(mbuf_t mbuf, size_t len) 458{ 459 mbuf->m_len = len; 460} 461 462size_t mbuf_maxlen(const mbuf_t mbuf) 463{ 464 if (mbuf->m_flags & M_EXT) 465 return mbuf->m_ext.ext_size; 466 return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf)); 467} 468 469mbuf_type_t mbuf_type(const mbuf_t mbuf) 470{ 471 return mbuf->m_type; 472} 473 474errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type) 475{ 476 if (new_type == MBUF_TYPE_FREE) return EINVAL; 477 478 m_mchtype(mbuf, new_type); 479 480 return 0; 481} 482 483mbuf_flags_t mbuf_flags(const mbuf_t mbuf) 484{ 485 return mbuf->m_flags & mbuf_flags_mask; 486} 487 488errno_t mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags) 489{ 490 if ((flags & ~mbuf_flags_mask) != 0) return EINVAL; 491 mbuf->m_flags = flags | 492 (mbuf->m_flags & ~mbuf_flags_mask); 493 494 return 0; 495} 496 497errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask) 498{ 499 if (((flags | mask) & ~mbuf_flags_mask) != 0) return EINVAL; 500 501 mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask); 502 503 return 0; 504} 505 506errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src) 507{ 508 if (((src)->m_flags & M_PKTHDR) == 0) 509 return EINVAL; 510 511 m_copy_pkthdr(dest, src); 512 513 return 0; 514} 515 516size_t mbuf_pkthdr_len(const mbuf_t mbuf) 517{ 518 return mbuf->m_pkthdr.len; 519} 520 521void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len) 522{ 523 mbuf->m_pkthdr.len = len; 524} 525 526void mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount) 527{ 528 mbuf->m_pkthdr.len += amount; 529} 530 531ifnet_t mbuf_pkthdr_rcvif(const mbuf_t mbuf) 532{ 533 // If we reference count ifnets, we should take a reference here before returning 534 return mbuf->m_pkthdr.rcvif; 535} 536 537errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet) 538{ 539 /* May want to walk ifnet list to determine if interface is valid */ 540 mbuf->m_pkthdr.rcvif = (struct ifnet*)ifnet; 541 return 0; 542} 543 544void* mbuf_pkthdr_header(const mbuf_t mbuf) 545{ 546 return mbuf->m_pkthdr.header; 547} 548 549void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header) 550{ 551 mbuf->m_pkthdr.header = (void*)header; 552} 553 554void 555mbuf_inbound_modified(mbuf_t mbuf) 556{ 557 /* Invalidate hardware generated checksum flags */ 558 mbuf->m_pkthdr.csum_flags = 0; 559} 560 561extern void in_cksum_offset(struct mbuf* m, size_t ip_offset); 562extern void in_delayed_cksum_offset(struct mbuf *m, int ip_offset); 563 564void 565mbuf_outbound_finalize(mbuf_t mbuf, u_int32_t protocol_family, size_t protocol_offset) 566{ 567 if ((mbuf->m_pkthdr.csum_flags & 568 (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_TCP_SUM16 | CSUM_DELAY_IPV6_DATA)) == 0) 569 return; 570 571 /* Generate the packet in software, client needs it */ 572 switch (protocol_family) { 573 case PF_INET: 574 if (mbuf->m_pkthdr.csum_flags & CSUM_TCP_SUM16) { 575 /* 576 * If you're wondering where this lovely code comes 577 * from, we're trying to undo what happens in ip_output. 578 * Look for CSUM_TCP_SUM16 in ip_output. 579 */ 580 u_int16_t first, second; 581 mbuf->m_pkthdr.csum_flags &= ~CSUM_TCP_SUM16; 582 mbuf->m_pkthdr.csum_flags |= CSUM_TCP; 583 first = mbuf->m_pkthdr.csum_data >> 16; 584 second = mbuf->m_pkthdr.csum_data & 0xffff; 585 mbuf->m_pkthdr.csum_data = first - second; 586 } 587 if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_DATA) { 588 in_delayed_cksum_offset(mbuf, protocol_offset); 589 } 590 591 if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) { 592 in_cksum_offset(mbuf, protocol_offset); 593 } 594 595 mbuf->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DELAY_IP); 596 break; 597 598 case PF_INET6: 599 600 if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IPV6_DATA) { 601 in_delayed_cksum_offset(mbuf, protocol_offset); 602 } 603 mbuf->m_pkthdr.csum_flags &= ~CSUM_DELAY_IPV6_DATA; 604 break; 605 606 607 default: 608 /* 609 * Not sure what to do here if anything. 610 * Hardware checksum code looked pretty IPv4/IPv6 specific. 611 */ 612 if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_DELAY_IPV6_DATA)) != 0) 613 panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 or IPv6 packet (%u)!\n", protocol_family); 614 } 615} 616 617errno_t 618mbuf_set_vlan_tag( 619 mbuf_t mbuf, 620 u_int16_t vlan) 621{ 622 mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID; 623 mbuf->m_pkthdr.vlan_tag = vlan; 624 625 return 0; 626} 627 628errno_t 629mbuf_get_vlan_tag( 630 mbuf_t mbuf, 631 u_int16_t *vlan) 632{ 633 if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) 634 return ENXIO; // No vlan tag set 635 636 *vlan = mbuf->m_pkthdr.vlan_tag; 637 638 return 0; 639} 640 641errno_t 642mbuf_clear_vlan_tag( 643 mbuf_t mbuf) 644{ 645 mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID; 646 mbuf->m_pkthdr.vlan_tag = 0; 647 648 return 0; 649} 650 651static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags = 652 MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP | 653 MBUF_CSUM_REQ_SUM16 | MBUF_CSUM_REQ_TCPIPV6 | MBUF_CSUM_REQ_UDPIPV6; 654 655errno_t 656mbuf_set_csum_requested( 657 mbuf_t mbuf, 658 mbuf_csum_request_flags_t request, 659 u_int32_t value) 660{ 661 request &= mbuf_valid_csum_request_flags; 662 mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request; 663 mbuf->m_pkthdr.csum_data = value; 664 665 return 0; 666} 667 668static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags = 669 MBUF_TSO_IPV4 | MBUF_TSO_IPV6; 670 671errno_t 672mbuf_get_tso_requested( 673 mbuf_t mbuf, 674 mbuf_tso_request_flags_t *request, 675 u_int32_t *value) 676{ 677 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || 678 request == NULL || value == NULL) 679 return EINVAL; 680 681 *request = mbuf->m_pkthdr.csum_flags; 682 *request &= mbuf_valid_tso_request_flags; 683 if (*request && value != NULL) 684 *value = mbuf->m_pkthdr.tso_segsz; 685 686 return 0; 687} 688 689errno_t 690mbuf_get_csum_requested( 691 mbuf_t mbuf, 692 mbuf_csum_request_flags_t *request, 693 u_int32_t *value) 694{ 695 *request = mbuf->m_pkthdr.csum_flags; 696 *request &= mbuf_valid_csum_request_flags; 697 if (value != NULL) { 698 *value = mbuf->m_pkthdr.csum_data; 699 } 700 701 return 0; 702} 703 704errno_t 705mbuf_clear_csum_requested( 706 mbuf_t mbuf) 707{ 708 mbuf->m_pkthdr.csum_flags &= 0xffff0000; 709 mbuf->m_pkthdr.csum_data = 0; 710 711 return 0; 712} 713 714static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags = 715 MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA | 716 MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_TCP_SUM16; 717 718errno_t 719mbuf_set_csum_performed( 720 mbuf_t mbuf, 721 mbuf_csum_performed_flags_t performed, 722 u_int32_t value) 723{ 724 performed &= mbuf_valid_csum_performed_flags; 725 mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed; 726 mbuf->m_pkthdr.csum_data = value; 727 728 return 0; 729} 730 731errno_t 732mbuf_get_csum_performed( 733 mbuf_t mbuf, 734 mbuf_csum_performed_flags_t *performed, 735 u_int32_t *value) 736{ 737 *performed = mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags; 738 *value = mbuf->m_pkthdr.csum_data; 739 740 return 0; 741} 742 743errno_t 744mbuf_clear_csum_performed( 745 mbuf_t mbuf) 746{ 747 mbuf->m_pkthdr.csum_flags &= 0xffff0000; 748 mbuf->m_pkthdr.csum_data = 0; 749 750 return 0; 751} 752 753errno_t 754mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, 755 u_int16_t *csum) 756{ 757 if (mbuf == NULL || length == 0 || csum == NULL || 758 (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) 759 return (EINVAL); 760 761 *csum = inet_cksum(mbuf, protocol, offset, length); 762 return (0); 763} 764 765#if INET6 766errno_t 767mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, 768 u_int16_t *csum) 769{ 770 if (mbuf == NULL || length == 0 || csum == NULL || 771 (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) 772 return (EINVAL); 773 774 *csum = inet6_cksum(mbuf, protocol, offset, length); 775 return (0); 776} 777#else /* INET6 */ 778errno_t 779mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol, 780 __unused u_int32_t offset, __unused u_int32_t length, 781 __unused u_int16_t *csum) 782{ 783 panic("mbuf_inet6_cksum() doesn't exist on this platform\n"); 784 return (0); 785} 786 787u_int16_t 788inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt, 789 __unused unsigned int off, __unused unsigned int len) 790{ 791 panic("inet6_cksum() doesn't exist on this platform\n"); 792 return (0); 793} 794 795void nd6_lookup_ipv6(void); 796void 797nd6_lookup_ipv6(void) 798{ 799 panic("nd6_lookup_ipv6() doesn't exist on this platform\n"); 800} 801 802int 803in6addr_local(__unused struct in6_addr *a) 804{ 805 panic("in6addr_local() doesn't exist on this platform\n"); 806 return (0); 807} 808 809void nd6_storelladdr(void); 810void 811nd6_storelladdr(void) 812{ 813 panic("nd6_storelladdr() doesn't exist on this platform\n"); 814} 815#endif /* INET6 */ 816 817/* 818 * Mbuf tag KPIs 819 */ 820 821#define MTAG_FIRST_ID FIRST_KPI_STR_ID 822 823errno_t 824mbuf_tag_id_find( 825 const char *string, 826 mbuf_tag_id_t *out_id) 827{ 828 return net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1); 829} 830 831errno_t 832mbuf_tag_allocate( 833 mbuf_t mbuf, 834 mbuf_tag_id_t id, 835 mbuf_tag_type_t type, 836 size_t length, 837 mbuf_how_t how, 838 void** data_p) 839{ 840 struct m_tag *tag; 841 u_int32_t mtag_id_first, mtag_id_last; 842 843 if (data_p != NULL) 844 *data_p = NULL; 845 846 /* Sanity check parameters */ 847 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); 848 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || 849 id > mtag_id_last || length < 1 || (length & 0xffff0000) != 0 || 850 data_p == NULL) { 851 return EINVAL; 852 } 853 854 /* Make sure this mtag hasn't already been allocated */ 855 tag = m_tag_locate(mbuf, id, type, NULL); 856 if (tag != NULL) { 857 return EEXIST; 858 } 859 860 /* Allocate an mtag */ 861 tag = m_tag_create(id, type, length, how, mbuf); 862 if (tag == NULL) { 863 return how == M_WAITOK ? ENOMEM : EWOULDBLOCK; 864 } 865 866 /* Attach the mtag and set *data_p */ 867 m_tag_prepend(mbuf, tag); 868 *data_p = tag + 1; 869 870 return 0; 871} 872 873errno_t 874mbuf_tag_find( 875 mbuf_t mbuf, 876 mbuf_tag_id_t id, 877 mbuf_tag_type_t type, 878 size_t* length, 879 void** data_p) 880{ 881 struct m_tag *tag; 882 u_int32_t mtag_id_first, mtag_id_last; 883 884 if (length != NULL) 885 *length = 0; 886 if (data_p != NULL) 887 *data_p = NULL; 888 889 /* Sanity check parameters */ 890 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); 891 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || 892 id > mtag_id_last || length == NULL || data_p == NULL) { 893 return EINVAL; 894 } 895 896 /* Locate an mtag */ 897 tag = m_tag_locate(mbuf, id, type, NULL); 898 if (tag == NULL) { 899 return ENOENT; 900 } 901 902 /* Copy out the pointer to the data and the lenght value */ 903 *length = tag->m_tag_len; 904 *data_p = tag + 1; 905 906 return 0; 907} 908 909void 910mbuf_tag_free( 911 mbuf_t mbuf, 912 mbuf_tag_id_t id, 913 mbuf_tag_type_t type) 914{ 915 struct m_tag *tag; 916 u_int32_t mtag_id_first, mtag_id_last; 917 918 /* Sanity check parameters */ 919 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); 920 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || 921 id > mtag_id_last) 922 return; 923 924 tag = m_tag_locate(mbuf, id, type, NULL); 925 if (tag == NULL) { 926 return; 927 } 928 929 m_tag_delete(mbuf, tag); 930 return; 931} 932 933/* mbuf stats */ 934void mbuf_stats(struct mbuf_stat *stats) 935{ 936 stats->mbufs = mbstat.m_mbufs; 937 stats->clusters = mbstat.m_clusters; 938 stats->clfree = mbstat.m_clfree; 939 stats->drops = mbstat.m_drops; 940 stats->wait = mbstat.m_wait; 941 stats->drain = mbstat.m_drain; 942 __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes)); 943 stats->mcfail = mbstat.m_mcfail; 944 stats->mpfail = mbstat.m_mpfail; 945 stats->msize = mbstat.m_msize; 946 stats->mclbytes = mbstat.m_mclbytes; 947 stats->minclsize = mbstat.m_minclsize; 948 stats->mlen = mbstat.m_mlen; 949 stats->mhlen = mbstat.m_mhlen; 950 stats->bigclusters = mbstat.m_bigclusters; 951 stats->bigclfree = mbstat.m_bigclfree; 952 stats->bigmclbytes = mbstat.m_bigmclbytes; 953} 954 955errno_t 956mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf) 957{ 958 errno_t error; 959 struct mbuf *m; 960 unsigned int numpkts = 1; 961 unsigned int numchunks = maxchunks ? *maxchunks : 0; 962 963 if (packetlen == 0) { 964 error = EINVAL; 965 goto out; 966 } 967 m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0); 968 if (m == 0) { 969 if (maxchunks && *maxchunks && numchunks > *maxchunks) 970 error = ENOBUFS; 971 else 972 error = ENOMEM; 973 } else { 974 if (maxchunks) 975 *maxchunks = numchunks; 976 error = 0; 977 *mbuf = m; 978 } 979out: 980 return error; 981} 982 983errno_t 984mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf) 985{ 986 errno_t error; 987 struct mbuf *m; 988 unsigned int numchunks = maxchunks ? *maxchunks : 0; 989 990 if (numpkts == 0) { 991 error = EINVAL; 992 goto out; 993 } 994 if (packetlen == 0) { 995 error = EINVAL; 996 goto out; 997 } 998 m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0); 999 if (m == 0) { 1000 if (maxchunks && *maxchunks && numchunks > *maxchunks) 1001 error = ENOBUFS; 1002 else 1003 error = ENOMEM; 1004 } else { 1005 if (maxchunks) 1006 *maxchunks = numchunks; 1007 error = 0; 1008 *mbuf = m; 1009 } 1010out: 1011 return error; 1012} 1013 1014 1015 1016/* 1017 * mbuf_copyback differs from m_copyback in a few ways: 1018 * 1) mbuf_copyback will allocate clusters for new mbufs we append 1019 * 2) mbuf_copyback will grow the last mbuf in the chain if possible 1020 * 3) mbuf_copyback reports whether or not the operation succeeded 1021 * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT 1022 */ 1023errno_t 1024mbuf_copyback( 1025 mbuf_t m, 1026 size_t off, 1027 size_t len, 1028 const void *data, 1029 mbuf_how_t how) 1030{ 1031 size_t mlen; 1032 mbuf_t m_start = m; 1033 mbuf_t n; 1034 int totlen = 0; 1035 errno_t result = 0; 1036 const char *cp = data; 1037 1038 if (m == NULL || len == 0 || data == NULL) 1039 return EINVAL; 1040 1041 while (off > (mlen = m->m_len)) { 1042 off -= mlen; 1043 totlen += mlen; 1044 if (m->m_next == 0) { 1045 n = m_getclr(how, m->m_type); 1046 if (n == 0) { 1047 result = ENOBUFS; 1048 goto out; 1049 } 1050 n->m_len = MIN(MLEN, len + off); 1051 m->m_next = n; 1052 } 1053 m = m->m_next; 1054 } 1055 1056 while (len > 0) { 1057 mlen = MIN(m->m_len - off, len); 1058 if (mlen < len && m->m_next == NULL && mbuf_trailingspace(m) > 0) { 1059 size_t grow = MIN(mbuf_trailingspace(m), len - mlen); 1060 mlen += grow; 1061 m->m_len += grow; 1062 } 1063 bcopy(cp, off + (char*)mbuf_data(m), (unsigned)mlen); 1064 cp += mlen; 1065 len -= mlen; 1066 mlen += off; 1067 off = 0; 1068 totlen += mlen; 1069 if (len == 0) 1070 break; 1071 if (m->m_next == 0) { 1072 n = m_get(how, m->m_type); 1073 if (n == NULL) { 1074 result = ENOBUFS; 1075 goto out; 1076 } 1077 if (len > MINCLSIZE) { 1078 /* cluter allocation failure is okay, we can grow chain */ 1079 mbuf_mclget(how, m->m_type, &n); 1080 } 1081 n->m_len = MIN(mbuf_maxlen(n), len); 1082 m->m_next = n; 1083 } 1084 m = m->m_next; 1085 } 1086 1087out: 1088 if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen)) 1089 m_start->m_pkthdr.len = totlen; 1090 1091 return result; 1092} 1093 1094u_int32_t 1095mbuf_get_mlen(void) 1096{ 1097 return (_MLEN); 1098} 1099 1100u_int32_t 1101mbuf_get_mhlen(void) 1102{ 1103 return (_MHLEN); 1104} 1105 1106u_int32_t 1107mbuf_get_minclsize(void) 1108{ 1109 return (MHLEN + MLEN); 1110} 1111 1112mbuf_traffic_class_t 1113mbuf_get_traffic_class(mbuf_t m) 1114{ 1115 if (m == NULL || !(m->m_flags & M_PKTHDR)) 1116 return (MBUF_TC_BE); 1117 1118 return (m_get_traffic_class(m)); 1119} 1120 1121errno_t 1122mbuf_set_traffic_class(mbuf_t m, mbuf_traffic_class_t tc) 1123{ 1124 if (m == NULL || !(m->m_flags & M_PKTHDR) || 1125 ((u_int32_t)tc >= MBUF_TC_MAX)) 1126 return (EINVAL); 1127 1128 return (m_set_traffic_class(m, tc)); 1129} 1130 1131int 1132mbuf_is_traffic_class_privileged(mbuf_t m) 1133{ 1134 if (m == NULL || !(m->m_flags & M_PKTHDR) || 1135 !MBUF_VALID_SC(m->m_pkthdr.svc)) 1136 return (0); 1137 1138 return (m->m_pkthdr.aux_flags & MAUXF_PRIO_PRIVILEGED); 1139} 1140 1141mbuf_svc_class_t 1142mbuf_get_service_class(mbuf_t m) 1143{ 1144 if (m == NULL || !(m->m_flags & M_PKTHDR)) 1145 return (MBUF_SC_BE); 1146 1147 return (m_get_service_class(m)); 1148} 1149 1150errno_t 1151mbuf_set_service_class(mbuf_t m, mbuf_svc_class_t sc) 1152{ 1153 if (m == NULL || !(m->m_flags & M_PKTHDR)) 1154 return (EINVAL); 1155 1156 return (m_set_service_class(m, sc)); 1157} 1158 1159errno_t 1160mbuf_pkthdr_aux_flags(mbuf_t m, mbuf_pkthdr_aux_flags_t *flagsp) 1161{ 1162 u_int32_t flags; 1163 if (m == NULL || !(m->m_flags & M_PKTHDR) || flagsp == NULL) 1164 return (EINVAL); 1165 1166 flags = m->m_pkthdr.aux_flags & MBUF_PKTAUXF_MASK; 1167 1168 /* These 2 flags are mutually exclusive */ 1169 VERIFY((flags & 1170 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)) != 1171 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)); 1172 1173 *flagsp = flags; 1174 return (0); 1175} 1176