2 3/* 4 * IPsec-specific mbuf routines. 5 */ 6 7#include "opt_param.h" 8 9#include <sys/param.h> 10#include <sys/systm.h> 11#include <sys/mbuf.h> 12#include <sys/socket.h> 13 14#include <net/route.h> 15#include <netinet/in.h> 16 17#include <netipsec/ipsec.h> 18 19extern struct mbuf *m_getptr(struct mbuf *, int, int *); 20 21/* 22 * Create a writable copy of the mbuf chain. While doing this 23 * we compact the chain with a goal of producing a chain with 24 * at most two mbufs. The second mbuf in this chain is likely 25 * to be a cluster. The primary purpose of this work is to create 26 * a writable packet for encryption, compression, etc. The 27 * secondary goal is to linearize the data so the data can be 28 * passed to crypto hardware in the most efficient manner possible. 29 */ 30struct mbuf * 31m_clone(struct mbuf *m0) 32{ 33 struct mbuf *m, *mprev; 34 35 KASSERT(m0 != NULL, ("m_clone: null mbuf")); 36 37 mprev = NULL; 38 for (m = m0; m != NULL; m = mprev->m_next) { 39 /* 40 * Regular mbufs are ignored unless there's a cluster 41 * in front of it that we can use to coalesce. We do 42 * the latter mainly so later clusters can be coalesced 43 * also w/o having to handle them specially (i.e. convert 44 * mbuf+cluster -> cluster). This optimization is heavily 45 * influenced by the assumption that we're running over 46 * Ethernet where MCBYTES is large enough that the max 47 * packet size will permit lots of coalescing into a 48 * single cluster. This in turn permits efficient 49 * crypto operations, especially when using hardware. 50 */ 51 if ((m->m_flags & M_EXT) == 0) { 52 if (mprev && (mprev->m_flags & M_EXT) && 53 m->m_len <= M_TRAILINGSPACE(mprev)) { 54 /* XXX: this ignores mbuf types */ 55 memcpy(mtod(mprev, caddr_t) + mprev->m_len, 56 mtod(m, caddr_t), m->m_len); 57 mprev->m_len += m->m_len; 58 mprev->m_next = m->m_next; /* unlink from chain */ 59 m_free(m); /* reclaim mbuf */ 60 newipsecstat.ips_mbcoalesced++; 61 } else { 62 mprev = m; 63 } 64 continue; 65 } 66 /* 67 * Cluster'd mbufs are left alone (for now). 68 */ 69 if (!MEXT_IS_REF(m)) { 70 mprev = m; 71 continue; 72 } 73 /* 74 * Not writable, replace with a copy or coalesce with 75 * the previous mbuf if possible (since we have to copy 76 * it anyway, we try to reduce the number of mbufs and 77 * clusters so that future work is easier). 78 */ 79 /* XXX why can M_PKTHDR be set past the first mbuf? */ 80 KASSERT(m->m_flags & M_EXT, 81 ("m_clone: m_flags 0x%x", m->m_flags)); 82 /* NB: we only coalesce into a cluster */ 83 if (mprev == NULL || (mprev->m_flags & M_EXT) == 0 || 84 m->m_len > M_TRAILINGSPACE(mprev)) { 85 struct mbuf *n; 86 87 /* 88 * Allocate a new page, copy the data to the front 89 * and release the reference to the old page. 90 */ 91 if (mprev == NULL && (m->m_flags & M_PKTHDR)) { 92 /* 93 * NB: if a packet header is present we 94 * must allocate the mbuf separately from 95 * the cluster 'cuz M_COPY_PKTHDR will 96 * smash the data pointer and drop the 97 * M_EXT marker. 98 */ 99 MGETHDR(n, M_DONTWAIT, m->m_type); 100 if (n == NULL) { 101 m_freem(m0); 102 return (NULL); 103 }
|
105 MCLGET(n, M_DONTWAIT); 106 if ((n->m_flags & M_EXT) == 0) { 107 m_free(n); 108 m_freem(m0); 109 return (NULL); 110 } 111 } else { 112 n = m_getcl(M_DONTWAIT, m->m_type, m->m_flags); 113 if (n == NULL) { 114 m_freem(m0); 115 return (NULL); 116 } 117 } 118 memcpy(mtod(n, caddr_t), mtod(m, caddr_t), m->m_len); 119 n->m_len = m->m_len; 120 n->m_next = m->m_next; 121 if (mprev == NULL) 122 m0 = n; /* new head of chain */ 123 else 124 mprev->m_next = n; /* replace old mbuf */ 125 m_free(m); /* release old mbuf */ 126 mprev = n; 127 newipsecstat.ips_clcopied++; 128 } else { 129 /* XXX: this ignores mbuf types */ 130 memcpy(mtod(mprev, caddr_t) + mprev->m_len, 131 mtod(m, caddr_t), m->m_len); 132 mprev->m_len += m->m_len; 133 mprev->m_next = m->m_next; /* unlink from chain */ 134 m_free(m); /* reclaim mbuf */ 135 newipsecstat.ips_clcoalesced++; 136 } 137 } 138 return (m0); 139} 140 141/* 142 * Make space for a new header of length hlen at offset off 143 * in the packet. When doing this we allocate new mbufs only 144 * when absolutely necessary. The mbuf where the new header 145 * is to go is returned together with an offset into the mbuf. 146 * If NULL is returned then the mbuf chain may have been modified; 147 * the caller is assumed to always free the chain. 148 */ 149struct mbuf * 150m_makespace(struct mbuf *m0, int skip, int hlen, int *off) 151{ 152 struct mbuf *m; 153 unsigned remain; 154 155 KASSERT(m0 != NULL, ("m_dmakespace: null mbuf")); 156 KASSERT(hlen < MHLEN, ("m_makespace: hlen too big: %u", hlen)); 157 158 for (m = m0; m && skip > m->m_len; m = m->m_next) 159 skip -= m->m_len; 160 if (m == NULL) 161 return (NULL); 162 /* 163 * At this point skip is the offset into the mbuf m 164 * where the new header should be placed. Figure out 165 * if there's space to insert the new header. If so, 166 * and copying the remainder makese sense then do so. 167 * Otherwise insert a new mbuf in the chain, splitting 168 * the contents of m as needed. 169 */ 170 remain = m->m_len - skip; /* data to move */ 171 if (hlen > M_TRAILINGSPACE(m)) { 172 struct mbuf *n; 173 174 /* XXX code doesn't handle clusters XXX */ 175 KASSERT(remain < MLEN, 176 ("m_makespace: remainder too big: %u", remain)); 177 /* 178 * Not enough space in m, split the contents 179 * of m, inserting new mbufs as required. 180 * 181 * NB: this ignores mbuf types. 182 */ 183 MGET(n, M_DONTWAIT, MT_DATA); 184 if (n == NULL) 185 return (NULL); 186 n->m_next = m->m_next; /* splice new mbuf */ 187 m->m_next = n; 188 newipsecstat.ips_mbinserted++; 189 if (hlen <= M_TRAILINGSPACE(m) + remain) { 190 /* 191 * New header fits in the old mbuf if we copy 192 * the remainder; just do the copy to the new 193 * mbuf and we're good to go. 194 */ 195 memcpy(mtod(n, caddr_t), 196 mtod(m, caddr_t) + skip, remain); 197 n->m_len = remain; 198 m->m_len = skip + hlen; 199 *off = skip; 200 } else { 201 /* 202 * No space in the old mbuf for the new header. 203 * Make space in the new mbuf and check the 204 * remainder'd data fits too. If not then we 205 * must allocate an additional mbuf (yech). 206 */ 207 n->m_len = 0; 208 if (remain + hlen > M_TRAILINGSPACE(n)) { 209 struct mbuf *n2; 210 211 MGET(n2, M_DONTWAIT, MT_DATA); 212 /* NB: new mbuf is on chain, let caller free */ 213 if (n2 == NULL) 214 return (NULL); 215 n2->m_len = 0; 216 memcpy(mtod(n2, caddr_t), 217 mtod(m, caddr_t) + skip, remain); 218 n2->m_len = remain; 219 /* splice in second mbuf */ 220 n2->m_next = n->m_next; 221 n->m_next = n2; 222 newipsecstat.ips_mbinserted++; 223 } else { 224 memcpy(mtod(n, caddr_t) + hlen, 225 mtod(m, caddr_t) + skip, remain); 226 n->m_len += remain; 227 } 228 m->m_len -= remain; 229 n->m_len += hlen; 230 m = n; /* header is at front ... */ 231 *off = 0; /* ... of new mbuf */ 232 } 233 } else { 234 /* 235 * Copy the remainder to the back of the mbuf 236 * so there's space to write the new header. 237 */ 238 /* XXX can this be memcpy? does it handle overlap? */ 239 ovbcopy(mtod(m, caddr_t) + skip, 240 mtod(m, caddr_t) + skip + hlen, remain); 241 m->m_len += hlen; 242 *off = skip; 243 } 244 m0->m_pkthdr.len += hlen; /* adjust packet length */ 245 return m; 246} 247 248/* 249 * m_pad(m, n) pads <m> with <n> bytes at the end. The packet header 250 * length is updated, and a pointer to the first byte of the padding 251 * (which is guaranteed to be all in one mbuf) is returned. 252 */ 253caddr_t 254m_pad(struct mbuf *m, int n) 255{ 256 register struct mbuf *m0, *m1; 257 register int len, pad; 258 caddr_t retval; 259 260 if (n <= 0) { /* No stupid arguments. */ 261 DPRINTF(("m_pad: pad length invalid (%d)\n", n)); 262 m_freem(m); 263 return NULL; 264 } 265 266 len = m->m_pkthdr.len; 267 pad = n; 268 m0 = m; 269 270 while (m0->m_len < len) { 271KASSERT(m0->m_next != NULL, ("m_pad: m0 null, len %u m_len %u", len, m0->m_len));/*XXX*/ 272 len -= m0->m_len; 273 m0 = m0->m_next; 274 } 275 276 if (m0->m_len != len) { 277 DPRINTF(("m_pad: length mismatch (should be %d instead of %d)\n", 278 m->m_pkthdr.len, m->m_pkthdr.len + m0->m_len - len)); 279 280 m_freem(m); 281 return NULL; 282 } 283 284 /* Check for zero-length trailing mbufs, and find the last one. */ 285 for (m1 = m0; m1->m_next; m1 = m1->m_next) { 286 if (m1->m_next->m_len != 0) { 287 DPRINTF(("m_pad: length mismatch (should be %d " 288 "instead of %d)\n", 289 m->m_pkthdr.len, 290 m->m_pkthdr.len + m1->m_next->m_len)); 291 292 m_freem(m); 293 return NULL; 294 } 295 296 m0 = m1->m_next; 297 } 298 299 if (pad > M_TRAILINGSPACE(m0)) { 300 /* Add an mbuf to the chain. */ 301 MGET(m1, M_DONTWAIT, MT_DATA); 302 if (m1 == 0) { 303 m_freem(m0); 304 DPRINTF(("m_pad: unable to get extra mbuf\n")); 305 return NULL; 306 } 307 308 m0->m_next = m1; 309 m0 = m1; 310 m0->m_len = 0; 311 } 312 313 retval = m0->m_data + m0->m_len; 314 m0->m_len += pad; 315 m->m_pkthdr.len += pad; 316 317 return retval; 318} 319 320/* 321 * Remove hlen data at offset skip in the packet. This is used by 322 * the protocols strip protocol headers and associated data (e.g. IV, 323 * authenticator) on input. 324 */ 325int 326m_striphdr(struct mbuf *m, int skip, int hlen) 327{ 328 struct mbuf *m1; 329 int roff; 330 331 /* Find beginning of header */ 332 m1 = m_getptr(m, skip, &roff); 333 if (m1 == NULL) 334 return (EINVAL); 335 336 /* Remove the header and associated data from the mbuf. */ 337 if (roff == 0) { 338 /* The header was at the beginning of the mbuf */ 339 newipsecstat.ips_input_front++; 340 m_adj(m1, hlen); 341 if ((m1->m_flags & M_PKTHDR) == 0) 342 m->m_pkthdr.len -= hlen; 343 } else if (roff + hlen >= m1->m_len) { 344 struct mbuf *mo; 345 346 /* 347 * Part or all of the header is at the end of this mbuf, 348 * so first let's remove the remainder of the header from 349 * the beginning of the remainder of the mbuf chain, if any. 350 */ 351 newipsecstat.ips_input_end++; 352 if (roff + hlen > m1->m_len) { 353 /* Adjust the next mbuf by the remainder */ 354 m_adj(m1->m_next, roff + hlen - m1->m_len); 355 356 /* The second mbuf is guaranteed not to have a pkthdr... */ 357 m->m_pkthdr.len -= (roff + hlen - m1->m_len); 358 } 359 360 /* Now, let's unlink the mbuf chain for a second...*/ 361 mo = m1->m_next; 362 m1->m_next = NULL; 363 364 /* ...and trim the end of the first part of the chain...sick */ 365 m_adj(m1, -(m1->m_len - roff)); 366 if ((m1->m_flags & M_PKTHDR) == 0) 367 m->m_pkthdr.len -= (m1->m_len - roff); 368 369 /* Finally, let's relink */ 370 m1->m_next = mo; 371 } else { 372 /* 373 * The header lies in the "middle" of the mbuf; copy 374 * the remainder of the mbuf down over the header. 375 */ 376 newipsecstat.ips_input_middle++; 377 bcopy(mtod(m1, u_char *) + roff + hlen, 378 mtod(m1, u_char *) + roff, 379 m1->m_len - (roff + hlen)); 380 m1->m_len -= hlen; 381 m->m_pkthdr.len -= hlen; 382 } 383 return (0); 384} 385 386/* 387 * Diagnostic routine to check mbuf alignment as required by the 388 * crypto device drivers (that use DMA). 389 */ 390void 391m_checkalignment(const char* where, struct mbuf *m0, int off, int len) 392{ 393 int roff; 394 struct mbuf *m = m_getptr(m0, off, &roff); 395 caddr_t addr; 396 397 if (m == NULL) 398 return; 399 printf("%s (off %u len %u): ", where, off, len); 400 addr = mtod(m, caddr_t) + roff; 401 do { 402 int mlen; 403 404 if (((uintptr_t) addr) & 3) { 405 printf("addr misaligned %p,", addr); 406 break; 407 } 408 mlen = m->m_len; 409 if (mlen > len) 410 mlen = len; 411 len -= mlen; 412 if (len && (mlen & 3)) { 413 printf("len mismatch %u,", mlen); 414 break; 415 } 416 m = m->m_next; 417 addr = m ? mtod(m, caddr_t) : NULL; 418 } while (m && len > 0); 419 for (m = m0; m; m = m->m_next) 420 printf(" [%p:%u]", mtod(m, caddr_t), m->m_len); 421 printf("\n"); 422}
|