mvec.h revision 175414
1/************************************************************************** 2 * 3 * Copyright (c) 2007, Kip Macy kmacy@freebsd.org 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, 10 * this list of conditions and the following disclaimer. 11 * 12 * 2. The name of Kip Macy nor the names of other 13 * contributors may be used to endorse or promote products derived from 14 * this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 * 28 * $FreeBSD: head/sys/dev/cxgb/sys/mvec.h 175414 2008-01-17 21:25:09Z sam $ 29 * 30 ***************************************************************************/ 31 32#ifndef _MVEC_H_ 33#define _MVEC_H_ 34 35int cxgb_cache_init(void); 36 37void cxgb_cache_flush(void); 38 39caddr_t cxgb_cache_get(uma_zone_t zone); 40 41void cxgb_cache_put(uma_zone_t zone, void *cl); 42 43void cxgb_cache_refill(void); 44 45extern int cxgb_cached_allocations; 46extern int cxgb_cached; 47extern int cxgb_ext_freed; 48extern int cxgb_mbufs_outstanding; 49extern int cxgb_pack_outstanding; 50 51#define mtomv(m) ((struct mbuf_vec *)((m)->m_pktdat)) 52#define M_IOVEC 0x100000 /* mbuf immediate data area is used for cluster ptrs */ 53 54/* 55 * duplication from mbuf.h - can't use directly because 56 * m_ext is a define 57 */ 58struct m_ext_ { 59 caddr_t ext_buf; /* start of buffer */ 60 void (*ext_free) /* free routine if not the usual */ 61 (void *, void *); 62 void *ext_args; /* optional argument pointer */ 63 u_int ext_size; /* size of buffer, for ext_free */ 64 volatile u_int *ref_cnt; /* pointer to ref count info */ 65 int ext_type; /* type of external storage */ 66}; 67 68#define MT_IOVEC 9 69#define MT_CLIOVEC 10 70 71#define EXT_IOVEC 8 72#define EXT_CLIOVEC 9 73#define EXT_JMPIOVEC 10 74 75 76extern uma_zone_t zone_miovec; 77 78struct mbuf_iovec { 79 struct m_ext_ mi_ext; 80 uint32_t mi_flags; 81 uint32_t mi_len; 82 caddr_t mi_data; 83 uint16_t mi_tso_segsz; 84 uint16_t mi_ether_vtag; 85 uint16_t mi_rss_hash; /* this can be shrunk down if something comes 86 * along that needs 1 byte 87 */ 88 uint16_t mi_pad; 89 struct mbuf *mi_mbuf; /* need to be able to handle the @#$@@#%$ing packet zone */ 90#define mi_size mi_ext.ext_size 91#define mi_base mi_ext.ext_buf 92#define mi_args mi_ext.ext_args 93#define mi_size mi_ext.ext_size 94#define mi_size mi_ext.ext_size 95#define mi_refcnt mi_ext.ref_cnt 96#define mi_ext_free mi_ext.ext_free 97#define mi_ext_flags mi_ext.ext_flags 98#define mi_type mi_ext.ext_type 99}; 100 101#define MIOVBYTES 512 102#define MAX_MBUF_IOV ((MHLEN-8)/sizeof(struct mbuf_iovec)) 103#define MAX_MIOVEC_IOV ((MIOVBYTES-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) 104#define MAX_CL_IOV ((MCLBYTES-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) 105#define MAX_PAGE_IOV ((MJUMPAGESIZE-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) 106 107struct mbuf_vec { 108 uint16_t mv_first; /* first valid cluster */ 109 uint16_t mv_count; /* # of clusters */ 110 uint32_t mv_flags; /* flags for iovec */ 111 struct mbuf_iovec mv_vec[0]; /* depends on whether or not this is in a cluster or an mbuf */ 112}; 113void mi_init(void); 114void mi_deinit(void); 115 116int _m_explode(struct mbuf *); 117void mb_free_vec(struct mbuf *m); 118 119static __inline void 120m_iovinit(struct mbuf *m) 121{ 122 struct mbuf_vec *mv = mtomv(m); 123 124 mv->mv_first = mv->mv_count = 0; 125 m->m_pkthdr.len = m->m_len = 0; 126 m->m_flags |= M_IOVEC; 127} 128 129static __inline void 130m_iovappend(struct mbuf *m, uint8_t *cl, int size, int len, caddr_t data, volatile uint32_t *ref) 131{ 132 struct mbuf_vec *mv = mtomv(m); 133 struct mbuf_iovec *iov; 134 int idx = mv->mv_first + mv->mv_count; 135 136 KASSERT(idx <= MAX_MBUF_IOV, ("tried to append too many clusters to mbuf iovec")); 137 if ((m->m_flags & M_EXT) != 0) 138 panic("invalid flags in %s", __func__); 139 140 if (mv->mv_count == 0) 141 m->m_data = data; 142 143 iov = &mv->mv_vec[idx]; 144 iov->mi_type = m_gettype(size); 145 iov->mi_base = cl; 146 iov->mi_len = len; 147 iov->mi_data = data; 148 iov->mi_refcnt = ref; 149 m->m_pkthdr.len += len; 150 m->m_len += len; 151 mv->mv_count++; 152} 153 154static __inline int 155m_explode(struct mbuf *m) 156{ 157 if ((m->m_flags & M_IOVEC) == 0) 158 return (0); 159 160 return _m_explode(m); 161} 162 163static __inline void 164busdma_map_mbuf_fast(struct mbuf *m, bus_dma_segment_t *seg) 165{ 166 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 167 seg->ds_len = m->m_len; 168} 169 170int busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs); 171int busdma_map_sg_vec(struct mbuf **m, struct mbuf **mp, bus_dma_segment_t *segs, int count); 172static __inline int busdma_map_sgl(bus_dma_segment_t *vsegs, bus_dma_segment_t *segs, int count) 173{ 174 while (count--) { 175 segs->ds_addr = pmap_kextract((vm_offset_t)vsegs->ds_addr); 176 segs->ds_len = vsegs->ds_len; 177 segs++; 178 vsegs++; 179 } 180 return (0); 181} 182 183struct mbuf *mi_collapse_mbuf(struct mbuf_iovec *mi, struct mbuf *m); 184struct mbuf *mi_collapse_sge(struct mbuf_iovec *mi, bus_dma_segment_t *seg); 185void *mcl_alloc(int seg_count, int *type); 186 187void mb_free_ext_fast(struct mbuf_iovec *mi, int type, int idx); 188 189static __inline void 190m_free_iovec(struct mbuf *m, int type) 191{ 192 int i; 193 struct mbuf_vec *mv; 194 struct mbuf_iovec *mi; 195 196 mv = mtomv(m); 197 mi = mv->mv_vec; 198 for (i = 0; i < mv->mv_count; i++, mi++) { 199 DPRINTF("freeing buf=%d of %d\n", i, mv->mv_count); 200 mb_free_ext_fast(mi, mi->mi_type, i); 201 } 202 switch (type) { 203 case EXT_IOVEC: 204 uma_zfree(zone_miovec, m); 205 break; 206 case EXT_CLIOVEC: 207 cxgb_cache_put(zone_clust, m); 208 break; 209 case EXT_JMPIOVEC: 210 cxgb_cache_put(zone_jumbop, m); 211 break; 212 default: 213 panic("unexpected type %d\n", type); 214 } 215} 216 217static __inline void 218m_freem_iovec(struct mbuf_iovec *mi) 219{ 220 struct mbuf *m = (struct mbuf *)mi->mi_base; 221 222 switch (mi->mi_type) { 223 case EXT_MBUF: 224#ifdef PIO_LEN 225 KASSERT(m->m_pkthdr.len > PIO_LEN, ("freeing PIO buf")); 226#endif 227 KASSERT((mi->mi_flags & M_NOFREE) == 0, ("no free set on mbuf")); 228 KASSERT(m->m_next == NULL, ("freeing chain")); 229 cxgb_mbufs_outstanding--; 230 m_free_fast(m); 231 break; 232 case EXT_PACKET: 233 cxgb_pack_outstanding--; 234 m_free(mi->mi_mbuf); 235 break; 236 case EXT_IOVEC: 237 case EXT_CLIOVEC: 238 case EXT_JMPIOVEC: 239 m = (struct mbuf *)mi->mi_base; 240 m_free_iovec(m, mi->mi_type); 241 break; 242 case EXT_CLUSTER: 243 case EXT_JUMBOP: 244 case EXT_JUMBO9: 245 case EXT_JUMBO16: 246 case EXT_SFBUF: 247 case EXT_NET_DRV: 248 case EXT_MOD_TYPE: 249 case EXT_DISPOSABLE: 250 case EXT_EXTREF: 251 mb_free_ext_fast(mi, mi->mi_type, -1); 252 break; 253 default: 254 panic("unknown miov type: %d\n", mi->mi_type); 255 break; 256 } 257} 258 259static __inline uma_zone_t 260m_getzonefromtype(int type) 261{ 262 uma_zone_t zone; 263 264 switch (type) { 265 case EXT_MBUF: 266 zone = zone_mbuf; 267 break; 268 case EXT_CLUSTER: 269 zone = zone_clust; 270 break; 271#if MJUMPAGESIZE != MCLBYTES 272 case EXT_JUMBOP: 273 zone = zone_jumbop; 274 break; 275#endif 276 case EXT_JUMBO9: 277 zone = zone_jumbo9; 278 break; 279 case EXT_JUMBO16: 280 zone = zone_jumbo16; 281 break; 282 case EXT_PACKET: 283 zone = zone_pack; 284 break; 285 default: 286 panic("%s: invalid cluster type %d", __func__, type); 287 } 288 return (zone); 289} 290 291static __inline int 292m_getsizefromtype(int type) 293{ 294 int size; 295 296 switch (type) { 297 case EXT_MBUF: 298 size = MSIZE; 299 break; 300 case EXT_CLUSTER: 301 case EXT_PACKET: 302 size = MCLBYTES; 303 break; 304#if MJUMPAGESIZE != MCLBYTES 305 case EXT_JUMBOP: 306 size = MJUMPAGESIZE; 307 break; 308#endif 309 case EXT_JUMBO9: 310 size = MJUM9BYTES; 311 break; 312 case EXT_JUMBO16: 313 size = MJUM16BYTES; 314 break; 315 default: 316 panic("%s: unrecognized cluster type %d", __func__, type); 317 } 318 return (size); 319} 320 321void dump_mi(struct mbuf_iovec *mi); 322 323#endif /* _MVEC_H_ */ 324