mvec.h revision 175347
1/************************************************************************** 2 * 3 * Copyright (c) 2007, Kip Macy kmacy@freebsd.org 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, 10 * this list of conditions and the following disclaimer. 11 * 12 * 2. The name of Kip Macy nor the names of other 13 * contributors may be used to endorse or promote products derived from 14 * this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 * 28 * $FreeBSD: head/sys/dev/cxgb/sys/mvec.h 175347 2008-01-15 08:08:09Z kmacy $ 29 * 30 ***************************************************************************/ 31 32#ifndef _MVEC_H_ 33#define _MVEC_H_ 34 35int cxgb_cache_init(void); 36 37void cxgb_cache_flush(void); 38 39caddr_t cxgb_cache_get(uma_zone_t zone); 40 41void cxgb_cache_put(uma_zone_t zone, void *cl); 42 43void cxgb_cache_refill(void); 44 45extern int cxgb_cached_allocations; 46extern int cxgb_cached; 47extern int cxgb_ext_freed; 48extern int mbufs_outstanding; 49 50#define mtomv(m) ((struct mbuf_vec *)((m)->m_pktdat)) 51#define M_IOVEC 0x100000 /* mbuf immediate data area is used for cluster ptrs */ 52 53/* 54 * duplication from mbuf.h - can't use directly because 55 * m_ext is a define 56 */ 57struct m_ext_ { 58 caddr_t ext_buf; /* start of buffer */ 59 void (*ext_free) /* free routine if not the usual */ 60 (void *, void *); 61 void *ext_args; /* optional argument pointer */ 62 u_int ext_size; /* size of buffer, for ext_free */ 63 volatile u_int *ref_cnt; /* pointer to ref count info */ 64 int ext_type; /* type of external storage */ 65}; 66 67#define MT_IOVEC 9 68#define MT_CLIOVEC 10 69 70#define EXT_IOVEC 8 71#define EXT_CLIOVEC 9 72#define EXT_JMPIOVEC 10 73 74 75extern uma_zone_t zone_miovec; 76 77struct mbuf_iovec { 78 struct m_ext_ mi_ext; 79 uint32_t mi_flags; 80 uint32_t mi_len; 81 caddr_t mi_data; 82 uint16_t mi_tso_segsz; 83 uint16_t mi_ether_vtag; 84 uint16_t mi_rss_hash; /* this can be shrunk down if something comes 85 * along that needs 1 byte 86 */ 87 uint16_t mi_pad; 88 struct mbuf *mi_mbuf; /* need to be able to handle the @#$@@#%$ing packet zone */ 89#define mi_size mi_ext.ext_size 90#define mi_base mi_ext.ext_buf 91#define mi_args mi_ext.ext_args 92#define mi_size mi_ext.ext_size 93#define mi_size mi_ext.ext_size 94#define mi_refcnt mi_ext.ref_cnt 95#define mi_ext_free mi_ext.ext_free 96#define mi_ext_flags mi_ext.ext_flags 97#define mi_type mi_ext.ext_type 98}; 99 100#define MIOVBYTES 512 101#define MAX_MBUF_IOV ((MHLEN-8)/sizeof(struct mbuf_iovec)) 102#define MAX_MIOVEC_IOV ((MIOVBYTES-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) 103#define MAX_CL_IOV ((MCLBYTES-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) 104#define MAX_PAGE_IOV ((MJUMPAGESIZE-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) 105 106struct mbuf_vec { 107 uint16_t mv_first; /* first valid cluster */ 108 uint16_t mv_count; /* # of clusters */ 109 uint32_t mv_flags; /* flags for iovec */ 110 struct mbuf_iovec mv_vec[0]; /* depends on whether or not this is in a cluster or an mbuf */ 111}; 112void mi_init(void); 113void mi_deinit(void); 114 115int _m_explode(struct mbuf *); 116int _m_collapse(struct mbuf *, int maxbufs, struct mbuf **); 117void mb_free_vec(struct mbuf *m); 118 119static __inline void 120m_iovinit(struct mbuf *m) 121{ 122 struct mbuf_vec *mv = mtomv(m); 123 124 mv->mv_first = mv->mv_count = 0; 125 m->m_pkthdr.len = m->m_len = 0; 126 m->m_flags |= M_IOVEC; 127} 128 129static __inline void 130m_iovappend(struct mbuf *m, uint8_t *cl, int size, int len, caddr_t data, volatile uint32_t *ref) 131{ 132 struct mbuf_vec *mv = mtomv(m); 133 struct mbuf_iovec *iov; 134 int idx = mv->mv_first + mv->mv_count; 135 136 KASSERT(idx <= MAX_MBUF_IOV, ("tried to append too many clusters to mbuf iovec")); 137 if ((m->m_flags & M_EXT) != 0) 138 panic("invalid flags in %s", __func__); 139 140 if (mv->mv_count == 0) 141 m->m_data = data; 142 143 iov = &mv->mv_vec[idx]; 144 iov->mi_type = m_gettype(size); 145 iov->mi_base = cl; 146 iov->mi_len = len; 147 iov->mi_data = data; 148 iov->mi_refcnt = ref; 149 m->m_pkthdr.len += len; 150 m->m_len += len; 151 mv->mv_count++; 152} 153 154static __inline int 155m_explode(struct mbuf *m) 156{ 157 if ((m->m_flags & M_IOVEC) == 0) 158 return (0); 159 160 return _m_explode(m); 161} 162 163static __inline void 164busdma_map_mbuf_fast(struct mbuf *m, bus_dma_segment_t *seg) 165{ 166 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 167 seg->ds_len = m->m_len; 168} 169 170int busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs); 171int busdma_map_sg_vec(struct mbuf **m, struct mbuf **mp, bus_dma_segment_t *segs, int count); 172static __inline int busdma_map_sgl(bus_dma_segment_t *vsegs, bus_dma_segment_t *segs, int count) 173{ 174 while (count--) { 175 segs->ds_addr = pmap_kextract((vm_offset_t)vsegs->ds_addr); 176 segs->ds_len = vsegs->ds_len; 177 segs++; 178 vsegs++; 179 } 180 return (0); 181} 182 183struct mbuf *mi_collapse_mbuf(struct mbuf_iovec *mi, struct mbuf *m); 184struct mbuf *mi_collapse_sge(struct mbuf_iovec *mi, bus_dma_segment_t *seg); 185void *mcl_alloc(int seg_count, int *type); 186 187static __inline int 188m_collapse(struct mbuf *m, int maxbufs, struct mbuf **mnew) 189{ 190#if (!defined(__sparc64__) && !defined(__sun4v__)) 191 if (m->m_next == NULL) 192#endif 193 { 194 *mnew = m; 195 return (0); 196 } 197 return _m_collapse(m, maxbufs, mnew); 198} 199 200void mb_free_ext_fast(struct mbuf_iovec *mi, int type, int idx); 201 202static __inline void 203m_free_iovec(struct mbuf *m, int type) 204{ 205 int i; 206 struct mbuf_vec *mv; 207 struct mbuf_iovec *mi; 208 209 mv = mtomv(m); 210 mi = mv->mv_vec; 211 for (i = 0; i < mv->mv_count; i++, mi++) { 212 DPRINTF("freeing buf=%d of %d\n", i, mv->mv_count); 213 mb_free_ext_fast(mi, mi->mi_type, i); 214 } 215 switch (type) { 216 case EXT_IOVEC: 217 uma_zfree(zone_miovec, m); 218 break; 219 case EXT_CLIOVEC: 220 cxgb_cache_put(zone_clust, m); 221 break; 222 case EXT_JMPIOVEC: 223 cxgb_cache_put(zone_jumbop, m); 224 break; 225 default: 226 panic("unexpected type %d\n", type); 227 } 228} 229 230static __inline void 231m_freem_iovec(struct mbuf_iovec *mi) 232{ 233 struct mbuf *m = (struct mbuf *)mi->mi_base; 234 235 switch (mi->mi_type) { 236 case EXT_MBUF: 237#ifdef PIO_LEN 238 KASSERT(m->m_pkthdr.len > PIO_LEN, ("freeing PIO buf")); 239#endif 240 KASSERT((mi->mi_flags & M_NOFREE) == 0, ("no free set on mbuf")); 241 KASSERT(m->m_next == NULL, ("freeing chain")); 242 mbufs_outstanding--; 243 m_free_fast(m); 244 break; 245 case EXT_IOVEC: 246 case EXT_CLIOVEC: 247 case EXT_JMPIOVEC: 248 m = (struct mbuf *)mi->mi_base; 249 m_free_iovec(m, mi->mi_type); 250 break; 251 case EXT_CLUSTER: 252 case EXT_JUMBOP: 253 case EXT_JUMBO9: 254 case EXT_JUMBO16: 255 case EXT_SFBUF: 256 case EXT_NET_DRV: 257 case EXT_MOD_TYPE: 258 case EXT_DISPOSABLE: 259 case EXT_PACKET: 260 case EXT_EXTREF: 261 mb_free_ext_fast(mi, mi->mi_type, -1); 262 break; 263 default: 264 panic("unknown miov type: %d\n", mi->mi_type); 265 break; 266 } 267} 268 269static __inline uma_zone_t 270m_getzonefromtype(int type) 271{ 272 uma_zone_t zone; 273 274 switch (type) { 275 case EXT_MBUF: 276 zone = zone_mbuf; 277 break; 278 case EXT_CLUSTER: 279 zone = zone_clust; 280 break; 281#if MJUMPAGESIZE != MCLBYTES 282 case EXT_JUMBOP: 283 zone = zone_jumbop; 284 break; 285#endif 286 case EXT_JUMBO9: 287 zone = zone_jumbo9; 288 break; 289 case EXT_JUMBO16: 290 zone = zone_jumbo16; 291 break; 292 default: 293 panic("%s: invalid cluster type %d", __func__, type); 294 } 295 return (zone); 296} 297 298static __inline int 299m_getsizefromtype(int type) 300{ 301 int size; 302 303 switch (type) { 304 case EXT_MBUF: 305 size = MSIZE; 306 break; 307 case EXT_CLUSTER: 308 size = MCLBYTES; 309 break; 310#if MJUMPAGESIZE != MCLBYTES 311 case EXT_JUMBOP: 312 size = MJUMPAGESIZE; 313 break; 314#endif 315 case EXT_JUMBO9: 316 size = MJUM9BYTES; 317 break; 318 case EXT_JUMBO16: 319 size = MJUM16BYTES; 320 break; 321 default: 322 panic("%s: unrecognized cluster type %d", __func__, type); 323 } 324 return (size); 325} 326 327void dump_mi(struct mbuf_iovec *mi); 328 329#endif /* _MVEC_H_ */ 330