ipsec_mbuf.c revision 1.21
1/* $NetBSD: ipsec_mbuf.c,v 1.21 2018/03/05 12:42:28 maxv Exp $ */ 2 3/* 4 * Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD: /repoman/r/ncvs/src/sys/netipsec/ipsec_mbuf.c,v 1.5.2.2 2003/03/28 20:32:53 sam Exp $ 29 */ 30 31#include <sys/cdefs.h> 32__KERNEL_RCSID(0, "$NetBSD: ipsec_mbuf.c,v 1.21 2018/03/05 12:42:28 maxv Exp $"); 33 34/* 35 * IPsec-specific mbuf routines. 36 */ 37 38#include <sys/param.h> 39#include <sys/systm.h> 40#include <sys/mbuf.h> 41#include <sys/socket.h> 42 43#include <net/route.h> 44#include <netinet/in.h> 45 46#include <netipsec/ipsec.h> 47#include <netipsec/ipsec_var.h> 48#include <netipsec/ipsec_private.h> 49 50/* 51 * Create a writable copy of the mbuf chain. While doing this 52 * we compact the chain with a goal of producing a chain with 53 * at most two mbufs. The second mbuf in this chain is likely 54 * to be a cluster. The primary purpose of this work is to create 55 * a writable packet for encryption, compression, etc. The 56 * secondary goal is to linearize the data so the data can be 57 * passed to crypto hardware in the most efficient manner possible. 58 */ 59struct mbuf * 60m_clone(struct mbuf *m0) 61{ 62 struct mbuf *m, *mprev; 63 struct mbuf *n, *mfirst, *mlast; 64 int len, off; 65 66 KASSERT(m0 != NULL); 67 68 mprev = NULL; 69 for (m = m0; m != NULL; m = mprev->m_next) { 70 /* 71 * Regular mbufs are ignored unless there's a cluster 72 * in front of it that we can use to coalesce. We do 73 * the latter mainly so later clusters can be coalesced 74 * also w/o having to handle them specially (i.e. convert 75 * mbuf+cluster -> cluster). This optimization is heavily 76 * influenced by the assumption that we're running over 77 * Ethernet where MCLBYTES is large enough that the max 78 * packet size will permit lots of coalescing into a 79 * single cluster. This in turn permits efficient 80 * crypto operations, especially when using hardware. 81 */ 82 if ((m->m_flags & M_EXT) == 0) { 83 if (mprev && (mprev->m_flags & M_EXT) && 84 m->m_len <= M_TRAILINGSPACE(mprev)) { 85 /* XXX: this ignores mbuf types */ 86 memcpy(mtod(mprev, char *) + mprev->m_len, 87 mtod(m, char *), m->m_len); 88 mprev->m_len += m->m_len; 89 mprev->m_next = m->m_next; /* unlink from chain */ 90 m_free(m); /* reclaim mbuf */ 91 IPSEC_STATINC(IPSEC_STAT_MBCOALESCED); 92 } else { 93 mprev = m; 94 } 95 continue; 96 } 97 /* 98 * Writable mbufs are left alone (for now). Note 99 * that for 4.x systems it's not possible to identify 100 * whether or not mbufs with external buffers are 101 * writable unless they use clusters. 102 */ 103 if (M_EXT_WRITABLE(m)) { 104 mprev = m; 105 continue; 106 } 107 108 /* 109 * Not writable, replace with a copy or coalesce with 110 * the previous mbuf if possible (since we have to copy 111 * it anyway, we try to reduce the number of mbufs and 112 * clusters so that future work is easier). 113 */ 114 KASSERTMSG(m->m_flags & M_EXT, "m_flags 0x%x", m->m_flags); 115 /* NB: we only coalesce into a cluster or larger */ 116 if (mprev != NULL && (mprev->m_flags & M_EXT) && 117 m->m_len <= M_TRAILINGSPACE(mprev)) { 118 /* XXX: this ignores mbuf types */ 119 memcpy(mtod(mprev, char *) + mprev->m_len, 120 mtod(m, char *), m->m_len); 121 mprev->m_len += m->m_len; 122 mprev->m_next = m->m_next; /* unlink from chain */ 123 m_free(m); /* reclaim mbuf */ 124 IPSEC_STATINC(IPSEC_STAT_CLCOALESCED); 125 continue; 126 } 127 128 /* 129 * Allocate new space to hold the copy... 130 */ 131 /* XXX why can M_PKTHDR be set past the first mbuf? */ 132 if (mprev == NULL && (m->m_flags & M_PKTHDR)) { 133 /* 134 * NB: if a packet header is present we must 135 * allocate the mbuf separately from any cluster 136 * because M_MOVE_PKTHDR will smash the data 137 * pointer and drop the M_EXT marker. 138 */ 139 MGETHDR(n, M_DONTWAIT, m->m_type); 140 if (n == NULL) { 141 m_freem(m0); 142 return (NULL); 143 } 144 M_MOVE_PKTHDR(n, m); 145 MCLGET(n, M_DONTWAIT); 146 if ((n->m_flags & M_EXT) == 0) { 147 m_free(n); 148 m_freem(m0); 149 return (NULL); 150 } 151 } else { 152 n = m_getcl(M_DONTWAIT, m->m_type, m->m_flags); 153 if (n == NULL) { 154 m_freem(m0); 155 return (NULL); 156 } 157 } 158 /* 159 * ... and copy the data. We deal with jumbo mbufs 160 * (i.e. m_len > MCLBYTES) by splitting them into 161 * clusters. We could just malloc a buffer and make 162 * it external but too many device drivers don't know 163 * how to break up the non-contiguous memory when 164 * doing DMA. 165 */ 166 len = m->m_len; 167 off = 0; 168 mfirst = n; 169 mlast = NULL; 170 for (;;) { 171 int cc = min(len, MCLBYTES); 172 memcpy(mtod(n, char *), mtod(m, char *) + off, cc); 173 n->m_len = cc; 174 if (mlast != NULL) 175 mlast->m_next = n; 176 mlast = n; 177 IPSEC_STATINC(IPSEC_STAT_CLCOPIED); 178 179 len -= cc; 180 if (len <= 0) 181 break; 182 off += cc; 183 184 n = m_getcl(M_DONTWAIT, m->m_type, m->m_flags); 185 if (n == NULL) { 186 m_freem(mfirst); 187 m_freem(m0); 188 return (NULL); 189 } 190 } 191 n->m_next = m->m_next; 192 if (mprev == NULL) 193 m0 = mfirst; /* new head of chain */ 194 else 195 mprev->m_next = mfirst; /* replace old mbuf */ 196 m_free(m); /* release old mbuf */ 197 mprev = mfirst; 198 } 199 return (m0); 200} 201 202/* 203 * Make space for a new header of length hlen at skip bytes 204 * into the packet. When doing this we allocate new mbufs only 205 * when absolutely necessary. The mbuf where the new header 206 * is to go is returned together with an offset into the mbuf. 207 * If NULL is returned then the mbuf chain may have been modified; 208 * the caller is assumed to always free the chain. 209 */ 210struct mbuf * 211m_makespace(struct mbuf *m0, int skip, int hlen, int *off) 212{ 213 struct mbuf *m; 214 unsigned remain; 215 216 KASSERT(m0 != NULL); 217 KASSERTMSG(hlen < MHLEN, "hlen too big: %u", hlen); 218 219 for (m = m0; m && skip > m->m_len; m = m->m_next) 220 skip -= m->m_len; 221 if (m == NULL) 222 return (NULL); 223 /* 224 * At this point skip is the offset into the mbuf m 225 * where the new header should be placed. Figure out 226 * if there's space to insert the new header. If so, 227 * and copying the remainder makes sense then do so. 228 * Otherwise insert a new mbuf in the chain, splitting 229 * the contents of m as needed. 230 */ 231 remain = m->m_len - skip; /* data to move */ 232 if (hlen > M_TRAILINGSPACE(m)) { 233 struct mbuf *n0, *n, **np; 234 int todo, len, done, alloc; 235 236 n0 = NULL; 237 np = &n0; 238 alloc = 0; 239 done = 0; 240 todo = remain; 241 while (todo > 0) { 242 if (todo > MHLEN) { 243 n = m_getcl(M_DONTWAIT, m->m_type, 0); 244 len = MCLBYTES; 245 } else { 246 n = m_get(M_DONTWAIT, m->m_type); 247 len = MHLEN; 248 } 249 if (n == NULL) { 250 m_freem(n0); 251 return NULL; 252 } 253 *np = n; 254 np = &n->m_next; 255 alloc++; 256 len = min(todo, len); 257 memcpy(n->m_data, mtod(m, char *) + skip + done, len); 258 n->m_len = len; 259 done += len; 260 todo -= len; 261 } 262 263 if (hlen <= M_TRAILINGSPACE(m) + remain) { 264 m->m_len = skip + hlen; 265 *off = skip; 266 if (n0 != NULL) { 267 *np = m->m_next; 268 m->m_next = n0; 269 } 270 } else { 271 n = m_get(M_DONTWAIT, m->m_type); 272 if (n == NULL) { 273 m_freem(n0); 274 return NULL; 275 } 276 alloc++; 277 278 if ((n->m_next = n0) == NULL) 279 np = &n->m_next; 280 n0 = n; 281 282 *np = m->m_next; 283 m->m_next = n0; 284 285 n->m_len = hlen; 286 m->m_len = skip; 287 288 m = n; /* header is at front ... */ 289 *off = 0; /* ... of new mbuf */ 290 } 291 292 IPSEC_STATADD(IPSEC_STAT_MBINSERTED, alloc); 293 } else { 294 /* 295 * Copy the remainder to the back of the mbuf 296 * so there's space to write the new header. 297 */ 298 /* XXX can this be memcpy? does it handle overlap? */ 299 memmove(mtod(m, char *) + skip + hlen, 300 mtod(m, char *) + skip, remain); 301 m->m_len += hlen; 302 *off = skip; 303 } 304 m0->m_pkthdr.len += hlen; /* adjust packet length */ 305 return m; 306} 307 308/* 309 * m_pad(m, n) pads <m> with <n> bytes at the end. The packet header 310 * length is updated, and a pointer to the first byte of the padding 311 * (which is guaranteed to be all in one mbuf) is returned. 312 */ 313void * 314m_pad(struct mbuf *m, int n) 315{ 316 register struct mbuf *m0, *m1; 317 register int len, pad; 318 void *retval; 319 320 if (__predict_false(n > MLEN)) { 321 panic("%s: %d > MLEN", __func__, n); 322 } 323 324 len = m->m_pkthdr.len; 325 pad = n; 326 m0 = m; 327 328 while (m0->m_len < len) { 329 KASSERTMSG(m0->m_next != NULL, 330 "m0 null, len %u m_len %u", len, m0->m_len); 331 len -= m0->m_len; 332 m0 = m0->m_next; 333 } 334 335 if (m0->m_len != len) { 336 IPSECLOG(LOG_DEBUG, 337 "length mismatch (should be %d instead of %d)\n", 338 m->m_pkthdr.len, m->m_pkthdr.len + m0->m_len - len); 339 m_freem(m); 340 return NULL; 341 } 342 343 /* Check for zero-length trailing mbufs, and find the last one. */ 344 for (m1 = m0; m1->m_next; m1 = m1->m_next) { 345 if (m1->m_next->m_len != 0) { 346 IPSECLOG(LOG_DEBUG, 347 "length mismatch (should be %d instead of %d)\n", 348 m->m_pkthdr.len, 349 m->m_pkthdr.len + m1->m_next->m_len); 350 m_freem(m); 351 return NULL; 352 } 353 354 m0 = m1->m_next; 355 } 356 357 if (pad > M_TRAILINGSPACE(m0)) { 358 /* Add an mbuf to the chain. */ 359 MGET(m1, M_DONTWAIT, MT_DATA); 360 if (m1 == NULL) { 361 m_freem(m); 362 IPSECLOG(LOG_DEBUG, "unable to get extra mbuf\n"); 363 return NULL; 364 } 365 366 m0->m_next = m1; 367 m0 = m1; 368 m0->m_len = 0; 369 } 370 371 retval = m0->m_data + m0->m_len; 372 m0->m_len += pad; 373 m->m_pkthdr.len += pad; 374 375 return retval; 376} 377 378/* 379 * Remove hlen data at offset skip in the packet. This is used by 380 * the protocols strip protocol headers and associated data (e.g. IV, 381 * authenticator) on input. 382 */ 383int 384m_striphdr(struct mbuf *m, int skip, int hlen) 385{ 386 struct mbuf *m1; 387 int roff; 388 389 /* Find beginning of header */ 390 m1 = m_getptr(m, skip, &roff); 391 if (m1 == NULL) 392 return (EINVAL); 393 394 /* Remove the header and associated data from the mbuf. */ 395 if (roff == 0) { 396 /* The header was at the beginning of the mbuf */ 397 IPSEC_STATINC(IPSEC_STAT_INPUT_FRONT); 398 m_adj(m1, hlen); 399 if ((m1->m_flags & M_PKTHDR) == 0) 400 m->m_pkthdr.len -= hlen; 401 } else if (roff + hlen >= m1->m_len) { 402 struct mbuf *mo; 403 404 /* 405 * Part or all of the header is at the end of this mbuf, 406 * so first let's remove the remainder of the header from 407 * the beginning of the remainder of the mbuf chain, if any. 408 */ 409 IPSEC_STATINC(IPSEC_STAT_INPUT_END); 410 if (roff + hlen > m1->m_len) { 411 /* Adjust the next mbuf by the remainder */ 412 m_adj(m1->m_next, roff + hlen - m1->m_len); 413 414 /* The second mbuf is guaranteed not to have a pkthdr... */ 415 m->m_pkthdr.len -= (roff + hlen - m1->m_len); 416 } 417 418 /* Now, let's unlink the mbuf chain for a second...*/ 419 mo = m1->m_next; 420 m1->m_next = NULL; 421 422 /* ...and trim the end of the first part of the chain...sick */ 423 m_adj(m1, -(m1->m_len - roff)); 424 if ((m1->m_flags & M_PKTHDR) == 0) 425 m->m_pkthdr.len -= (m1->m_len - roff); 426 427 /* Finally, let's relink */ 428 m1->m_next = mo; 429 } else { 430 /* 431 * The header lies in the "middle" of the mbuf; copy 432 * the remainder of the mbuf down over the header. 433 */ 434 IPSEC_STATINC(IPSEC_STAT_INPUT_MIDDLE); 435 memmove(mtod(m1, u_char *) + roff, 436 mtod(m1, u_char *) + roff + hlen, 437 m1->m_len - (roff + hlen)); 438 m1->m_len -= hlen; 439 m->m_pkthdr.len -= hlen; 440 } 441 return (0); 442} 443