1139804Simp/*- 2117624Sharti * Copyright (c) 2003 3117624Sharti * Fraunhofer Institute for Open Communication Systems (FhG Fokus). 4117624Sharti * All rights reserved. 5117624Sharti * 6117624Sharti * Redistribution and use in source and binary forms, with or without 7117624Sharti * modification, are permitted provided that the following conditions 8117624Sharti * are met: 9117624Sharti * 1. Redistributions of source code must retain the above copyright 10117624Sharti * notice, this list of conditions and the following disclaimer. 11117624Sharti * 2. Redistributions in binary form must reproduce the above copyright 12117624Sharti * notice, this list of conditions and the following disclaimer in the 13117624Sharti * documentation and/or other materials provided with the distribution. 14117624Sharti * 15117624Sharti * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16117624Sharti * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17117624Sharti * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18117624Sharti * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19117624Sharti * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20117624Sharti * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21117624Sharti * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22117624Sharti * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23117624Sharti * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24117624Sharti * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25117624Sharti * SUCH DAMAGE. 26117624Sharti * 27117624Sharti * Author: Hartmut Brandt <harti@freebsd.org> 28117624Sharti */ 29117624Sharti 30117624Sharti#include <sys/cdefs.h> 31117624Sharti__FBSDID("$FreeBSD: releng/10.2/sys/kern/subr_mbpool.c 254842 2013-08-25 10:57:09Z andre $"); 32117624Sharti 33117624Sharti#include <sys/param.h> 34117624Sharti#include <sys/lock.h> 35117624Sharti#include <sys/mutex.h> 36117624Sharti#include <sys/kernel.h> 37117624Sharti#include <sys/systm.h> 38117624Sharti#include <sys/malloc.h> 39117624Sharti#include <sys/module.h> 40117624Sharti 41117624Sharti#include <machine/bus.h> 42117624Sharti 43254842Sandre#include <sys/mbuf.h> 44117624Sharti#include <sys/mbpool.h> 45117624Sharti 46117624ShartiMODULE_VERSION(libmbpool, 1); 47117624Sharti 48117624Sharti/* 49117624Sharti * Memory is allocated as DMA-able pages. Each page is divided into a number 50117624Sharti * of equal chunks where the last 4 bytes of each chunk are occupied by 51117624Sharti * the page number and the chunk number. The caller must take these four 52117624Sharti * bytes into account when specifying the chunk size. Each page is mapped by 53117624Sharti * its own DMA map using the user specified DMA tag. 54117624Sharti * 55117624Sharti * Each chunk has a used and a card bit in the high bits of its page number. 56117624Sharti * 0 0 chunk is free and may be allocated 57117624Sharti * 1 1 chunk has been given to the interface 58117624Sharti * 0 1 chunk is traveling through the system 59117624Sharti * 1 0 illegal 60117624Sharti */ 61117624Shartistruct mbtrail { 62117624Sharti uint16_t chunk; 63117624Sharti uint16_t page; 64117624Sharti}; 65117624Sharti#define MBP_CARD 0x8000 66117624Sharti#define MBP_USED 0x4000 67117624Sharti#define MBP_PMSK 0x3fff /* page number mask */ 68117624Sharti#define MBP_CMSK 0x01ff /* chunk number mask */ 69117624Sharti 70117624Shartistruct mbfree { 71117624Sharti SLIST_ENTRY(mbfree) link; /* link on free list */ 72117624Sharti}; 73117624Sharti 74117624Shartistruct mbpage { 75117624Sharti bus_dmamap_t map; /* map for this page */ 76117624Sharti bus_addr_t phy; /* physical address */ 77117624Sharti void *va; /* the memory */ 78117624Sharti}; 79117624Sharti 80117624Shartistruct mbpool { 81117624Sharti const char *name; /* a name for this pool */ 82117624Sharti bus_dma_tag_t dmat; /* tag for mapping */ 83117624Sharti u_int max_pages; /* maximum number of pages */ 84117624Sharti size_t page_size; /* size of each allocation */ 85117624Sharti size_t chunk_size; /* size of each external mbuf */ 86117624Sharti 87117624Sharti struct mtx free_lock; /* lock of free list */ 88117624Sharti SLIST_HEAD(, mbfree) free_list; /* free list */ 89117624Sharti u_int npages; /* current number of pages */ 90117624Sharti u_int nchunks; /* chunks per page */ 91117624Sharti struct mbpage pages[]; /* pages */ 92117624Sharti}; 93117624Sharti 94117624Shartistatic MALLOC_DEFINE(M_MBPOOL, "mbpools", "mbuf pools"); 95117624Sharti 96117624Sharti/* 97117624Sharti * Make a trail pointer from a chunk pointer 98117624Sharti */ 99117624Sharti#define C2T(P, C) ((struct mbtrail *)((char *)(C) + (P)->chunk_size - \ 100117624Sharti sizeof(struct mbtrail))) 101117624Sharti 102117624Sharti/* 103117624Sharti * Make a free chunk pointer from a chunk number 104117624Sharti */ 105117624Sharti#define N2C(P, PG, C) ((struct mbfree *)((char *)(PG)->va + \ 106117624Sharti (C) * (P)->chunk_size)) 107117624Sharti 108117624Sharti/* 109117624Sharti * Make/parse handles 110117624Sharti */ 111117624Sharti#define HMAKE(P, C) ((((P) & MBP_PMSK) << 16) | ((C) << 7)) 112117624Sharti#define HPAGE(H) (((H) >> 16) & MBP_PMSK) 113117624Sharti#define HCHUNK(H) (((H) >> 7) & MBP_CMSK) 114117624Sharti 115117624Sharti/* 116117624Sharti * initialize a pool 117117624Sharti */ 118117624Shartiint 119117624Shartimbp_create(struct mbpool **pp, const char *name, bus_dma_tag_t dmat, 120117624Sharti u_int max_pages, size_t page_size, size_t chunk_size) 121117624Sharti{ 122117624Sharti u_int nchunks; 123117624Sharti 124117624Sharti if (max_pages > MBPOOL_MAX_MAXPAGES || chunk_size == 0) 125117624Sharti return (EINVAL); 126117624Sharti nchunks = page_size / chunk_size; 127117624Sharti if (nchunks == 0 || nchunks > MBPOOL_MAX_CHUNKS) 128117624Sharti return (EINVAL); 129117624Sharti 130117624Sharti (*pp) = malloc(sizeof(struct mbpool) + 131117624Sharti max_pages * sizeof(struct mbpage), 132117624Sharti M_MBPOOL, M_WAITOK | M_ZERO); 133117624Sharti 134117624Sharti (*pp)->name = name; 135117624Sharti (*pp)->dmat = dmat; 136117624Sharti (*pp)->max_pages = max_pages; 137117624Sharti (*pp)->page_size = page_size; 138117624Sharti (*pp)->chunk_size = chunk_size; 139117624Sharti (*pp)->nchunks = nchunks; 140117624Sharti 141117624Sharti SLIST_INIT(&(*pp)->free_list); 142123263Struckman mtx_init(&(*pp)->free_lock, name, NULL, MTX_DEF); 143117624Sharti 144117624Sharti return (0); 145117624Sharti} 146117624Sharti 147117624Sharti/* 148117624Sharti * destroy a pool 149117624Sharti */ 150117624Shartivoid 151117624Shartimbp_destroy(struct mbpool *p) 152117624Sharti{ 153117624Sharti u_int i; 154117624Sharti struct mbpage *pg; 155117624Sharti#ifdef DIAGNOSTIC 156117624Sharti struct mbtrail *tr; 157117624Sharti u_int b; 158117624Sharti#endif 159117624Sharti 160117624Sharti for (i = 0; i < p->npages; i++) { 161117624Sharti pg = &p->pages[i]; 162117624Sharti#ifdef DIAGNOSTIC 163117624Sharti for (b = 0; b < p->nchunks; b++) { 164117624Sharti tr = C2T(p, N2C(p, pg, b)); 165117624Sharti if (tr->page & MBP_CARD) 166117624Sharti printf("%s: (%s) buf still on card" 167117624Sharti " %u/%u\n", __func__, p->name, i, b); 168117624Sharti if (tr->page & MBP_USED) 169117624Sharti printf("%s: (%s) sbuf still in use" 170117624Sharti " %u/%u\n", __func__, p->name, i, b); 171117624Sharti } 172117624Sharti#endif 173117624Sharti bus_dmamap_unload(p->dmat, pg->map); 174117624Sharti bus_dmamem_free(p->dmat, pg->va, pg->map); 175117624Sharti } 176117624Sharti mtx_destroy(&p->free_lock); 177117624Sharti 178117624Sharti free(p, M_MBPOOL); 179117624Sharti} 180117624Sharti 181117624Sharti/* 182117624Sharti * Helper function when loading a one segment DMA buffer. 183117624Sharti */ 184117624Shartistatic void 185117624Shartimbp_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 186117624Sharti{ 187117624Sharti if (error == 0) 188117624Sharti *(bus_addr_t *)arg = segs[0].ds_addr; 189117624Sharti} 190117624Sharti 191117624Sharti/* 192117624Sharti * Allocate a new page 193117624Sharti */ 194117624Shartistatic void 195117624Shartimbp_alloc_page(struct mbpool *p) 196117624Sharti{ 197117624Sharti int error; 198117624Sharti struct mbpage *pg; 199117624Sharti u_int i; 200117624Sharti struct mbfree *f; 201117624Sharti struct mbtrail *t; 202117624Sharti 203117624Sharti if (p->npages == p->max_pages) { 204117624Sharti#ifdef DIAGNOSTIC 205117624Sharti printf("%s: (%s) page limit reached %u\n", __func__, 206117624Sharti p->name, p->max_pages); 207117624Sharti#endif 208117624Sharti return; 209117624Sharti } 210117624Sharti pg = &p->pages[p->npages]; 211117624Sharti 212117624Sharti error = bus_dmamem_alloc(p->dmat, &pg->va, BUS_DMA_NOWAIT, &pg->map); 213117624Sharti if (error != 0) { 214117624Sharti free(pg, M_MBPOOL); 215117624Sharti return; 216117624Sharti } 217117624Sharti 218117624Sharti error = bus_dmamap_load(p->dmat, pg->map, pg->va, p->page_size, 219117624Sharti mbp_callback, &pg->phy, 0); 220117624Sharti if (error != 0) { 221117624Sharti bus_dmamem_free(p->dmat, pg->va, pg->map); 222117624Sharti free(pg, M_MBPOOL); 223117624Sharti return; 224117624Sharti } 225117624Sharti 226117624Sharti for (i = 0; i < p->nchunks; i++) { 227117624Sharti f = N2C(p, pg, i); 228117624Sharti t = C2T(p, f); 229117624Sharti t->page = p->npages; 230117624Sharti t->chunk = i; 231117624Sharti SLIST_INSERT_HEAD(&p->free_list, f, link); 232117624Sharti } 233117624Sharti 234117624Sharti p->npages++; 235117624Sharti} 236117624Sharti 237117624Sharti/* 238117624Sharti * allocate a chunk 239117624Sharti */ 240117624Shartivoid * 241117624Shartimbp_alloc(struct mbpool *p, bus_addr_t *pap, uint32_t *hp) 242117624Sharti{ 243117624Sharti struct mbfree *cf; 244117624Sharti struct mbtrail *t; 245117624Sharti 246117624Sharti mtx_lock(&p->free_lock); 247117624Sharti if ((cf = SLIST_FIRST(&p->free_list)) == NULL) { 248117624Sharti mbp_alloc_page(p); 249117624Sharti cf = SLIST_FIRST(&p->free_list); 250117624Sharti } 251117624Sharti if (cf == NULL) { 252117624Sharti mtx_unlock(&p->free_lock); 253117624Sharti return (NULL); 254117624Sharti } 255117624Sharti SLIST_REMOVE_HEAD(&p->free_list, link); 256117624Sharti mtx_unlock(&p->free_lock); 257117624Sharti 258117624Sharti t = C2T(p, cf); 259117624Sharti 260117624Sharti *pap = p->pages[t->page].phy + t->chunk * p->chunk_size; 261117624Sharti *hp = HMAKE(t->page, t->chunk); 262117624Sharti 263117624Sharti t->page |= MBP_CARD | MBP_USED; 264117624Sharti 265117624Sharti return (cf); 266117624Sharti} 267117624Sharti 268117624Sharti/* 269117624Sharti * Free a chunk 270117624Sharti */ 271117624Shartivoid 272117624Shartimbp_free(struct mbpool *p, void *ptr) 273117624Sharti{ 274117624Sharti struct mbtrail *t; 275117624Sharti 276117624Sharti mtx_lock(&p->free_lock); 277117624Sharti t = C2T(p, ptr); 278117624Sharti t->page &= ~(MBP_USED | MBP_CARD); 279117624Sharti SLIST_INSERT_HEAD(&p->free_list, (struct mbfree *)ptr, link); 280117624Sharti mtx_unlock(&p->free_lock); 281117624Sharti} 282117624Sharti 283117624Sharti/* 284117624Sharti * Mbuf system external mbuf free routine 285117624Sharti */ 286254842Sandreint 287254799Sandrembp_ext_free(struct mbuf *m, void *buf, void *arg) 288117624Sharti{ 289117624Sharti mbp_free(arg, buf); 290254842Sandre 291254842Sandre return (EXT_FREE_OK); 292117624Sharti} 293117624Sharti 294117624Sharti/* 295117624Sharti * Free all buffers that are marked as beeing on the card 296117624Sharti */ 297117624Shartivoid 298117624Shartimbp_card_free(struct mbpool *p) 299117624Sharti{ 300117624Sharti u_int i, b; 301117624Sharti struct mbpage *pg; 302117624Sharti struct mbtrail *tr; 303117624Sharti struct mbfree *cf; 304117624Sharti 305117624Sharti mtx_lock(&p->free_lock); 306117624Sharti for (i = 0; i < p->npages; i++) { 307117624Sharti pg = &p->pages[i]; 308117624Sharti for (b = 0; b < p->nchunks; b++) { 309117624Sharti cf = N2C(p, pg, b); 310117624Sharti tr = C2T(p, cf); 311117624Sharti if (tr->page & MBP_CARD) { 312117624Sharti tr->page &= MBP_PMSK; 313117624Sharti SLIST_INSERT_HEAD(&p->free_list, cf, link); 314117624Sharti } 315117624Sharti } 316117624Sharti } 317117624Sharti mtx_unlock(&p->free_lock); 318117624Sharti} 319117624Sharti 320117624Sharti/* 321117624Sharti * Count buffers 322117624Sharti */ 323117624Shartivoid 324117624Shartimbp_count(struct mbpool *p, u_int *used, u_int *card, u_int *free) 325117624Sharti{ 326117624Sharti u_int i, b; 327117624Sharti struct mbpage *pg; 328117624Sharti struct mbtrail *tr; 329117624Sharti struct mbfree *cf; 330117624Sharti 331117624Sharti *used = *card = *free = 0; 332117624Sharti for (i = 0; i < p->npages; i++) { 333117624Sharti pg = &p->pages[i]; 334117624Sharti for (b = 0; b < p->nchunks; b++) { 335117624Sharti tr = C2T(p, N2C(p, pg, b)); 336117624Sharti if (tr->page & MBP_CARD) 337117624Sharti (*card)++; 338117624Sharti if (tr->page & MBP_USED) 339117624Sharti (*used)++; 340117624Sharti } 341117624Sharti } 342117624Sharti mtx_lock(&p->free_lock); 343117624Sharti SLIST_FOREACH(cf, &p->free_list, link) 344170023Srwatson (*free)++; 345117624Sharti mtx_unlock(&p->free_lock); 346117624Sharti} 347117624Sharti 348117624Sharti/* 349117624Sharti * Get the buffer from a handle and clear the card flag. 350117624Sharti */ 351117624Shartivoid * 352117624Shartimbp_get(struct mbpool *p, uint32_t h) 353117624Sharti{ 354117624Sharti struct mbfree *cf; 355117624Sharti struct mbtrail *tr; 356117624Sharti 357117624Sharti cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h)); 358117624Sharti tr = C2T(p, cf); 359117624Sharti 360117624Sharti#ifdef DIAGNOSTIC 361117624Sharti if (!(tr->page & MBP_CARD)) 362117624Sharti printf("%s: (%s) chunk %u page %u not on card\n", __func__, 363117624Sharti p->name, HCHUNK(h), HPAGE(h)); 364117624Sharti#endif 365117624Sharti 366117624Sharti tr->page &= ~MBP_CARD; 367117624Sharti return (cf); 368117624Sharti} 369117624Sharti 370117624Sharti/* 371117624Sharti * Get the buffer from a handle and keep the card flag. 372117624Sharti */ 373117624Shartivoid * 374117624Shartimbp_get_keep(struct mbpool *p, uint32_t h) 375117624Sharti{ 376117624Sharti struct mbfree *cf; 377117624Sharti struct mbtrail *tr; 378117624Sharti 379117624Sharti cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h)); 380117624Sharti tr = C2T(p, cf); 381117624Sharti 382117624Sharti#ifdef DIAGNOSTIC 383117624Sharti if (!(tr->page & MBP_CARD)) 384117624Sharti printf("%s: (%s) chunk %u page %u not on card\n", __func__, 385117624Sharti p->name, HCHUNK(h), HPAGE(h)); 386117624Sharti#endif 387117624Sharti 388117624Sharti return (cf); 389117624Sharti} 390117624Sharti 391117624Sharti/* 392117624Sharti * sync the chunk 393117624Sharti */ 394117624Shartivoid 395117624Shartimbp_sync(struct mbpool *p, uint32_t h, bus_addr_t off, bus_size_t len, u_int op) 396117624Sharti{ 397117624Sharti 398117624Sharti#if 0 399117624Sharti bus_dmamap_sync_size(p->dmat, p->pages[HPAGE(h)].map, 400117624Sharti HCHUNK(h) * p->chunk_size + off, len, op); 401117624Sharti#endif 402117624Sharti} 403