netmap_mem2.c revision 234228
1234228Sluigi/* 2234228Sluigi * Copyright (C) 2012 Matteo Landi, Luigi Rizzo. All rights reserved. 3234228Sluigi * 4234228Sluigi * Redistribution and use in source and binary forms, with or without 5234228Sluigi * modification, are permitted provided that the following conditions 6234228Sluigi * are met: 7234228Sluigi * 1. Redistributions of source code must retain the above copyright 8234228Sluigi * notice, this list of conditions and the following disclaimer. 9234228Sluigi * 2. Redistributions in binary form must reproduce the above copyright 10234228Sluigi * notice, this list of conditions and the following disclaimer in the 11234228Sluigi * documentation and/or other materials provided with the distribution. 12234228Sluigi * 13234228Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14234228Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15234228Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16234228Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17234228Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18234228Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19234228Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20234228Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21234228Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22234228Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23234228Sluigi * SUCH DAMAGE. 24234228Sluigi */ 25234228Sluigi 26234228Sluigi/* 27234228Sluigi * $FreeBSD: head/sys/dev/netmap/netmap_mem2.c 234228 2012-04-13 16:32:33Z luigi $ 28234228Sluigi * $Id: netmap_mem2.c 10830 2012-03-22 18:06:01Z luigi $ 29234228Sluigi * 30234228Sluigi * New memory allocator for netmap 31234228Sluigi */ 32234228Sluigi 33234228Sluigi/* 34234228Sluigi * The new version allocates three regions: 35234228Sluigi * nm_if_pool for the struct netmap_if 36234228Sluigi * nm_ring_pool for the struct netmap_ring 37234228Sluigi * nm_buf_pool for the packet buffers. 38234228Sluigi * 39234228Sluigi * All regions need to be page-sized as we export them to 40234228Sluigi * userspace through mmap. Only the latter need to be dma-able, 41234228Sluigi * but for convenience use the same type of allocator for all. 42234228Sluigi * 43234228Sluigi * Once mapped, the three regions are exported to userspace 44234228Sluigi * as a contiguous block, starting from nm_if_pool. Each 45234228Sluigi * cluster (and pool) is an integral number of pages. 46234228Sluigi * [ . . . ][ . . . . . .][ . . . . . . . . . .] 47234228Sluigi * nm_if nm_ring nm_buf 48234228Sluigi * 49234228Sluigi * The userspace areas contain offsets of the objects in userspace. 50234228Sluigi * When (at init time) we write these offsets, we find out the index 51234228Sluigi * of the object, and from there locate the offset from the beginning 52234228Sluigi * of the region. 53234228Sluigi * 54234228Sluigi * Allocator for a pool of memory objects of the same size. 55234228Sluigi * The pool is split into smaller clusters, whose size is a 56234228Sluigi * multiple of the page size. The cluster size is chosen 57234228Sluigi * to minimize the waste for a given max cluster size 58234228Sluigi * (we do it by brute force, as we have relatively few object 59234228Sluigi * per cluster). 60234228Sluigi * 61234228Sluigi * To be polite with the cache, objects are aligned to 62234228Sluigi * the cache line, or 64 bytes. Sizes are rounded to multiple of 64. 63234228Sluigi * For each object we have 64234228Sluigi * one entry in the bitmap to signal the state. Allocation scans 65234228Sluigi * the bitmap, but since this is done only on attach, we are not 66234228Sluigi * too worried about performance 67234228Sluigi */ 68234228Sluigi 69234228Sluigi/* 70234228Sluigi * MEMORY SIZES: 71234228Sluigi * 72234228Sluigi * (all the parameters below will become tunables) 73234228Sluigi * 74234228Sluigi * struct netmap_if is variable size but small. 75234228Sluigi * Assuming each NIC has 8+2 rings, (4+1 tx, 4+1 rx) the netmap_if 76234228Sluigi * uses 120 bytes on a 64-bit machine. 77234228Sluigi * We allocate NETMAP_IF_MAX_SIZE (1024) which should work even for 78234228Sluigi * cards with 48 ring pairs. 79234228Sluigi * The total number of 'struct netmap_if' could be slightly larger 80234228Sluigi * that the total number of rings on all interfaces on the system. 81234228Sluigi */ 82234228Sluigi#define NETMAP_IF_MAX_SIZE 1024 83234228Sluigi#define NETMAP_IF_MAX_NUM 512 84234228Sluigi 85234228Sluigi/* 86234228Sluigi * netmap rings are up to 2..4k descriptors, 8 bytes each, 87234228Sluigi * plus some glue at the beginning (32 bytes). 88234228Sluigi * We set the default ring size to 9 pages (36K) and enable 89234228Sluigi * a few hundreds of them. 90234228Sluigi */ 91234228Sluigi#define NETMAP_RING_MAX_SIZE (9*PAGE_SIZE) 92234228Sluigi#define NETMAP_RING_MAX_NUM 200 /* approx 8MB */ 93234228Sluigi 94234228Sluigi/* 95234228Sluigi * Buffers: the more the better. Buffer size is NETMAP_BUF_SIZE, 96234228Sluigi * 2k or slightly less, aligned to 64 bytes. 97234228Sluigi * A large 10G interface can have 2k*18 = 36k buffers per interface, 98234228Sluigi * or about 72MB of memory. Up to us to use more. 99234228Sluigi */ 100234228Sluigi#ifndef CONSERVATIVE 101234228Sluigi#define NETMAP_BUF_MAX_NUM 100000 /* 200MB */ 102234228Sluigi#else /* CONSERVATIVE */ 103234228Sluigi#define NETMAP_BUF_MAX_NUM 20000 /* 40MB */ 104234228Sluigi#endif 105234228Sluigi 106234228Sluigi 107234228Sluigistruct netmap_obj_pool { 108234228Sluigi char name[16]; /* name of the allocator */ 109234228Sluigi u_int objtotal; /* actual total number of objects. */ 110234228Sluigi u_int objfree; /* number of free objects. */ 111234228Sluigi u_int clustentries; /* actual objects per cluster */ 112234228Sluigi 113234228Sluigi /* the total memory space is _numclusters*_clustsize */ 114234228Sluigi u_int _numclusters; /* how many clusters */ 115234228Sluigi u_int _clustsize; /* cluster size */ 116234228Sluigi u_int _objsize; /* actual object size */ 117234228Sluigi 118234228Sluigi u_int _memtotal; /* _numclusters*_clustsize */ 119234228Sluigi struct lut_entry *lut; /* virt,phys addresses, objtotal entries */ 120234228Sluigi uint32_t *bitmap; /* one bit per buffer, 1 means free */ 121234228Sluigi}; 122234228Sluigi 123234228Sluigistruct netmap_mem_d { 124234228Sluigi NM_LOCK_T nm_mtx; /* protect the allocator ? */ 125234228Sluigi u_int nm_totalsize; /* shorthand */ 126234228Sluigi 127234228Sluigi /* pointers to the three allocators */ 128234228Sluigi struct netmap_obj_pool *nm_if_pool; 129234228Sluigi struct netmap_obj_pool *nm_ring_pool; 130234228Sluigi struct netmap_obj_pool *nm_buf_pool; 131234228Sluigi}; 132234228Sluigi 133234228Sluigistruct lut_entry *netmap_buffer_lut; /* exported */ 134234228Sluigi 135234228Sluigi 136234228Sluigi/* 137234228Sluigi * Convert a userspace offset to a phisical address. 138234228Sluigi * XXX re-do in a simpler way. 139234228Sluigi * 140234228Sluigi * The idea here is to hide userspace applications the fact that pre-allocated 141234228Sluigi * memory is not contiguous, but fragmented across different clusters and 142234228Sluigi * smaller memory allocators. Consequently, first of all we need to find which 143234228Sluigi * allocator is owning provided offset, then we need to find out the physical 144234228Sluigi * address associated to target page (this is done using the look-up table. 145234228Sluigi */ 146234228Sluigistatic inline vm_paddr_t 147234228Sluiginetmap_ofstophys(vm_offset_t offset) 148234228Sluigi{ 149234228Sluigi const struct netmap_obj_pool *p[] = { 150234228Sluigi nm_mem->nm_if_pool, 151234228Sluigi nm_mem->nm_ring_pool, 152234228Sluigi nm_mem->nm_buf_pool }; 153234228Sluigi int i; 154234228Sluigi vm_offset_t o = offset; 155234228Sluigi 156234228Sluigi 157234228Sluigi for (i = 0; i < 3; offset -= p[i]->_memtotal, i++) { 158234228Sluigi if (offset >= p[i]->_memtotal) 159234228Sluigi continue; 160234228Sluigi // XXX now scan the clusters 161234228Sluigi return p[i]->lut[offset / p[i]->_objsize].paddr + 162234228Sluigi offset % p[i]->_objsize; 163234228Sluigi } 164234228Sluigi D("invalid ofs 0x%x out of 0x%x 0x%x 0x%x", o, 165234228Sluigi p[0]->_memtotal, p[0]->_memtotal + p[1]->_memtotal, 166234228Sluigi p[0]->_memtotal + p[1]->_memtotal + p[2]->_memtotal); 167234228Sluigi return 0; // XXX bad address 168234228Sluigi} 169234228Sluigi 170234228Sluigi/* 171234228Sluigi * we store objects by kernel address, need to find the offset 172234228Sluigi * within the pool to export the value to userspace. 173234228Sluigi * Algorithm: scan until we find the cluster, then add the 174234228Sluigi * actual offset in the cluster 175234228Sluigi */ 176234228Sluigissize_t 177234228Sluiginetmap_obj_offset(struct netmap_obj_pool *p, const void *vaddr) 178234228Sluigi{ 179234228Sluigi int i, k = p->clustentries, n = p->objtotal; 180234228Sluigi ssize_t ofs = 0; 181234228Sluigi 182234228Sluigi for (i = 0; i < n; i += k, ofs += p->_clustsize) { 183234228Sluigi const char *base = p->lut[i].vaddr; 184234228Sluigi ssize_t relofs = (const char *) vaddr - base; 185234228Sluigi 186234228Sluigi if (relofs < 0 || relofs > p->_clustsize) 187234228Sluigi continue; 188234228Sluigi 189234228Sluigi ofs = ofs + relofs; 190234228Sluigi ND("%s: return offset %d (cluster %d) for pointer %p", 191234228Sluigi p->name, ofs, i, vaddr); 192234228Sluigi return ofs; 193234228Sluigi } 194234228Sluigi D("address %p is not contained inside any cluster (%s)", 195234228Sluigi vaddr, p->name); 196234228Sluigi return 0; /* An error occurred */ 197234228Sluigi} 198234228Sluigi 199234228Sluigi/* Helper functions which convert virtual addresses to offsets */ 200234228Sluigi#define netmap_if_offset(v) \ 201234228Sluigi netmap_obj_offset(nm_mem->nm_if_pool, (v)) 202234228Sluigi 203234228Sluigi#define netmap_ring_offset(v) \ 204234228Sluigi (nm_mem->nm_if_pool->_memtotal + \ 205234228Sluigi netmap_obj_offset(nm_mem->nm_ring_pool, (v))) 206234228Sluigi 207234228Sluigi#define netmap_buf_offset(v) \ 208234228Sluigi (nm_mem->nm_if_pool->_memtotal + \ 209234228Sluigi nm_mem->nm_ring_pool->_memtotal + \ 210234228Sluigi netmap_obj_offset(nm_mem->nm_buf_pool, (v))) 211234228Sluigi 212234228Sluigi 213234228Sluigistatic void * 214234228Sluiginetmap_obj_malloc(struct netmap_obj_pool *p, int len) 215234228Sluigi{ 216234228Sluigi uint32_t i = 0; /* index in the bitmap */ 217234228Sluigi uint32_t mask, j; /* slot counter */ 218234228Sluigi void *vaddr = NULL; 219234228Sluigi 220234228Sluigi if (len > p->_objsize) { 221234228Sluigi D("%s request size %d too large", p->name, len); 222234228Sluigi // XXX cannot reduce the size 223234228Sluigi return NULL; 224234228Sluigi } 225234228Sluigi 226234228Sluigi if (p->objfree == 0) { 227234228Sluigi D("%s allocator: run out of memory", p->name); 228234228Sluigi return NULL; 229234228Sluigi } 230234228Sluigi 231234228Sluigi /* termination is guaranteed by p->free */ 232234228Sluigi while (vaddr == NULL) { 233234228Sluigi uint32_t cur = p->bitmap[i]; 234234228Sluigi if (cur == 0) { /* bitmask is fully used */ 235234228Sluigi i++; 236234228Sluigi continue; 237234228Sluigi } 238234228Sluigi /* locate a slot */ 239234228Sluigi for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) 240234228Sluigi ; 241234228Sluigi 242234228Sluigi p->bitmap[i] &= ~mask; /* mark object as in use */ 243234228Sluigi p->objfree--; 244234228Sluigi 245234228Sluigi vaddr = p->lut[i * 32 + j].vaddr; 246234228Sluigi } 247234228Sluigi ND("%s allocator: allocated object @ [%d][%d]: vaddr %p", i, j, vaddr); 248234228Sluigi 249234228Sluigi return vaddr; 250234228Sluigi} 251234228Sluigi 252234228Sluigi 253234228Sluigi/* 254234228Sluigi * free by index, not by address 255234228Sluigi */ 256234228Sluigistatic void 257234228Sluiginetmap_obj_free(struct netmap_obj_pool *p, uint32_t j) 258234228Sluigi{ 259234228Sluigi if (j >= p->objtotal) { 260234228Sluigi D("invalid index %u, max %u", j, p->objtotal); 261234228Sluigi return; 262234228Sluigi } 263234228Sluigi p->bitmap[j / 32] |= (1 << (j % 32)); 264234228Sluigi p->objfree++; 265234228Sluigi return; 266234228Sluigi} 267234228Sluigi 268234228Sluigistatic void 269234228Sluiginetmap_obj_free_va(struct netmap_obj_pool *p, void *vaddr) 270234228Sluigi{ 271234228Sluigi int i, j, n = p->_memtotal / p->_clustsize; 272234228Sluigi 273234228Sluigi for (i = 0, j = 0; i < n; i++, j += p->clustentries) { 274234228Sluigi void *base = p->lut[i * p->clustentries].vaddr; 275234228Sluigi ssize_t relofs = (ssize_t) vaddr - (ssize_t) base; 276234228Sluigi 277234228Sluigi /* Given address, is out of the scope of the current cluster.*/ 278234228Sluigi if (vaddr < base || relofs > p->_clustsize) 279234228Sluigi continue; 280234228Sluigi 281234228Sluigi j = j + relofs / p->_objsize; 282234228Sluigi KASSERT(j != 0, ("Cannot free object 0")); 283234228Sluigi netmap_obj_free(p, j); 284234228Sluigi return; 285234228Sluigi } 286234228Sluigi ND("address %p is not contained inside any cluster (%s)", 287234228Sluigi vaddr, p->name); 288234228Sluigi} 289234228Sluigi 290234228Sluigi#define netmap_if_malloc(len) netmap_obj_malloc(nm_mem->nm_if_pool, len) 291234228Sluigi#define netmap_if_free(v) netmap_obj_free_va(nm_mem->nm_if_pool, (v)) 292234228Sluigi#define netmap_ring_malloc(len) netmap_obj_malloc(nm_mem->nm_ring_pool, len) 293234228Sluigi#define netmap_buf_malloc() \ 294234228Sluigi netmap_obj_malloc(nm_mem->nm_buf_pool, NETMAP_BUF_SIZE) 295234228Sluigi 296234228Sluigi 297234228Sluigi/* Return the index associated to the given packet buffer */ 298234228Sluigi#define netmap_buf_index(v) \ 299234228Sluigi (netmap_obj_offset(nm_mem->nm_buf_pool, (v)) / nm_mem->nm_buf_pool->_objsize) 300234228Sluigi 301234228Sluigi 302234228Sluigistatic void 303234228Sluiginetmap_new_bufs(struct netmap_if *nifp __unused, 304234228Sluigi struct netmap_slot *slot, u_int n) 305234228Sluigi{ 306234228Sluigi struct netmap_obj_pool *p = nm_mem->nm_buf_pool; 307234228Sluigi uint32_t i = 0; /* slot counter */ 308234228Sluigi 309234228Sluigi for (i = 0; i < n; i++) { 310234228Sluigi void *vaddr = netmap_buf_malloc(); 311234228Sluigi if (vaddr == NULL) { 312234228Sluigi D("unable to locate empty packet buffer"); 313234228Sluigi goto cleanup; 314234228Sluigi } 315234228Sluigi 316234228Sluigi slot[i].buf_idx = netmap_buf_index(vaddr); 317234228Sluigi KASSERT(slot[i].buf_idx != 0, 318234228Sluigi ("Assigning buf_idx=0 to just created slot")); 319234228Sluigi slot[i].len = p->_objsize; 320234228Sluigi slot[i].flags = NS_BUF_CHANGED; // XXX GAETANO hack 321234228Sluigi } 322234228Sluigi 323234228Sluigi ND("allocated %d buffers, %d available", n, p->objfree); 324234228Sluigi return; 325234228Sluigi 326234228Sluigicleanup: 327234228Sluigi for (i--; i >= 0; i--) { 328234228Sluigi netmap_obj_free(nm_mem->nm_buf_pool, slot[i].buf_idx); 329234228Sluigi } 330234228Sluigi} 331234228Sluigi 332234228Sluigi 333234228Sluigistatic void 334234228Sluiginetmap_free_buf(struct netmap_if *nifp, uint32_t i) 335234228Sluigi{ 336234228Sluigi struct netmap_obj_pool *p = nm_mem->nm_buf_pool; 337234228Sluigi if (i < 2 || i >= p->objtotal) { 338234228Sluigi D("Cannot free buf#%d: should be in [2, %d[", i, p->objtotal); 339234228Sluigi return; 340234228Sluigi } 341234228Sluigi netmap_obj_free(nm_mem->nm_buf_pool, i); 342234228Sluigi} 343234228Sluigi 344234228Sluigi 345234228Sluigi/* 346234228Sluigi * Free all resources related to an allocator. 347234228Sluigi */ 348234228Sluigistatic void 349234228Sluiginetmap_destroy_obj_allocator(struct netmap_obj_pool *p) 350234228Sluigi{ 351234228Sluigi if (p == NULL) 352234228Sluigi return; 353234228Sluigi if (p->bitmap) 354234228Sluigi free(p->bitmap, M_NETMAP); 355234228Sluigi if (p->lut) { 356234228Sluigi int i; 357234228Sluigi for (i = 0; i < p->objtotal; i += p->clustentries) { 358234228Sluigi if (p->lut[i].vaddr) 359234228Sluigi contigfree(p->lut[i].vaddr, p->_clustsize, M_NETMAP); 360234228Sluigi } 361234228Sluigi bzero(p->lut, sizeof(struct lut_entry) * p->objtotal); 362234228Sluigi free(p->lut, M_NETMAP); 363234228Sluigi } 364234228Sluigi bzero(p, sizeof(*p)); 365234228Sluigi free(p, M_NETMAP); 366234228Sluigi} 367234228Sluigi 368234228Sluigi/* 369234228Sluigi * We receive a request for objtotal objects, of size objsize each. 370234228Sluigi * Internally we may round up both numbers, as we allocate objects 371234228Sluigi * in small clusters multiple of the page size. 372234228Sluigi * In the allocator we don't need to store the objsize, 373234228Sluigi * but we do need to keep track of objtotal' and clustentries, 374234228Sluigi * as they are needed when freeing memory. 375234228Sluigi * 376234228Sluigi * XXX note -- userspace needs the buffers to be contiguous, 377234228Sluigi * so we cannot afford gaps at the end of a cluster. 378234228Sluigi */ 379234228Sluigistatic struct netmap_obj_pool * 380234228Sluiginetmap_new_obj_allocator(const char *name, u_int objtotal, u_int objsize) 381234228Sluigi{ 382234228Sluigi struct netmap_obj_pool *p; 383234228Sluigi int i, n; 384234228Sluigi u_int clustsize; /* the cluster size, multiple of page size */ 385234228Sluigi u_int clustentries; /* how many objects per entry */ 386234228Sluigi 387234228Sluigi#define MAX_CLUSTSIZE (1<<17) 388234228Sluigi#define LINE_ROUND 64 389234228Sluigi if (objsize >= MAX_CLUSTSIZE) { 390234228Sluigi /* we could do it but there is no point */ 391234228Sluigi D("unsupported allocation for %d bytes", objsize); 392234228Sluigi return NULL; 393234228Sluigi } 394234228Sluigi /* make sure objsize is a multiple of LINE_ROUND */ 395234228Sluigi i = (objsize & (LINE_ROUND - 1)); 396234228Sluigi if (i) { 397234228Sluigi D("XXX aligning object by %d bytes", LINE_ROUND - i); 398234228Sluigi objsize += LINE_ROUND - i; 399234228Sluigi } 400234228Sluigi /* 401234228Sluigi * Compute number of objects using a brute-force approach: 402234228Sluigi * given a max cluster size, 403234228Sluigi * we try to fill it with objects keeping track of the 404234228Sluigi * wasted space to the next page boundary. 405234228Sluigi */ 406234228Sluigi for (clustentries = 0, i = 1;; i++) { 407234228Sluigi u_int delta, used = i * objsize; 408234228Sluigi if (used > MAX_CLUSTSIZE) 409234228Sluigi break; 410234228Sluigi delta = used % PAGE_SIZE; 411234228Sluigi if (delta == 0) { // exact solution 412234228Sluigi clustentries = i; 413234228Sluigi break; 414234228Sluigi } 415234228Sluigi if (delta > ( (clustentries*objsize) % PAGE_SIZE) ) 416234228Sluigi clustentries = i; 417234228Sluigi } 418234228Sluigi // D("XXX --- ouch, delta %d (bad for buffers)", delta); 419234228Sluigi /* compute clustsize and round to the next page */ 420234228Sluigi clustsize = clustentries * objsize; 421234228Sluigi i = (clustsize & (PAGE_SIZE - 1)); 422234228Sluigi if (i) 423234228Sluigi clustsize += PAGE_SIZE - i; 424234228Sluigi D("objsize %d clustsize %d objects %d", 425234228Sluigi objsize, clustsize, clustentries); 426234228Sluigi 427234228Sluigi p = malloc(sizeof(struct netmap_obj_pool), M_NETMAP, 428234228Sluigi M_WAITOK | M_ZERO); 429234228Sluigi if (p == NULL) { 430234228Sluigi D("Unable to create '%s' allocator", name); 431234228Sluigi return NULL; 432234228Sluigi } 433234228Sluigi /* 434234228Sluigi * Allocate and initialize the lookup table. 435234228Sluigi * 436234228Sluigi * The number of clusters is n = ceil(objtotal/clustentries) 437234228Sluigi * objtotal' = n * clustentries 438234228Sluigi */ 439234228Sluigi strncpy(p->name, name, sizeof(p->name)); 440234228Sluigi p->clustentries = clustentries; 441234228Sluigi p->_clustsize = clustsize; 442234228Sluigi n = (objtotal + clustentries - 1) / clustentries; 443234228Sluigi p->_numclusters = n; 444234228Sluigi p->objtotal = n * clustentries; 445234228Sluigi p->objfree = p->objtotal - 2; /* obj 0 and 1 are reserved */ 446234228Sluigi p->_objsize = objsize; 447234228Sluigi p->_memtotal = p->_numclusters * p->_clustsize; 448234228Sluigi 449234228Sluigi p->lut = malloc(sizeof(struct lut_entry) * p->objtotal, 450234228Sluigi M_NETMAP, M_WAITOK | M_ZERO); 451234228Sluigi if (p->lut == NULL) { 452234228Sluigi D("Unable to create lookup table for '%s' allocator", name); 453234228Sluigi goto clean; 454234228Sluigi } 455234228Sluigi 456234228Sluigi /* Allocate the bitmap */ 457234228Sluigi n = (p->objtotal + 31) / 32; 458234228Sluigi p->bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP, M_WAITOK | M_ZERO); 459234228Sluigi if (p->bitmap == NULL) { 460234228Sluigi D("Unable to create bitmap (%d entries) for allocator '%s'", n, 461234228Sluigi name); 462234228Sluigi goto clean; 463234228Sluigi } 464234228Sluigi 465234228Sluigi /* 466234228Sluigi * Allocate clusters, init pointers and bitmap 467234228Sluigi */ 468234228Sluigi for (i = 0; i < p->objtotal;) { 469234228Sluigi int lim = i + clustentries; 470234228Sluigi char *clust; 471234228Sluigi 472234228Sluigi clust = contigmalloc(clustsize, M_NETMAP, M_WAITOK | M_ZERO, 473234228Sluigi 0, -1UL, PAGE_SIZE, 0); 474234228Sluigi if (clust == NULL) { 475234228Sluigi /* 476234228Sluigi * If we get here, there is a severe memory shortage, 477234228Sluigi * so halve the allocated memory to reclaim some. 478234228Sluigi */ 479234228Sluigi D("Unable to create cluster at %d for '%s' allocator", 480234228Sluigi i, name); 481234228Sluigi lim = i / 2; 482234228Sluigi for (; i >= lim; i--) { 483234228Sluigi p->bitmap[ (i>>5) ] &= ~( 1 << (i & 31) ); 484234228Sluigi if (i % clustentries == 0 && p->lut[i].vaddr) 485234228Sluigi contigfree(p->lut[i].vaddr, 486234228Sluigi p->_clustsize, M_NETMAP); 487234228Sluigi } 488234228Sluigi p->objtotal = i; 489234228Sluigi p->objfree = p->objtotal - 2; 490234228Sluigi p->_numclusters = i / clustentries; 491234228Sluigi p->_memtotal = p->_numclusters * p->_clustsize; 492234228Sluigi break; 493234228Sluigi } 494234228Sluigi for (; i < lim; i++, clust += objsize) { 495234228Sluigi p->bitmap[ (i>>5) ] |= ( 1 << (i & 31) ); 496234228Sluigi p->lut[i].vaddr = clust; 497234228Sluigi p->lut[i].paddr = vtophys(clust); 498234228Sluigi } 499234228Sluigi } 500234228Sluigi p->bitmap[0] = ~3; /* objs 0 and 1 is always busy */ 501234228Sluigi D("Pre-allocated %d clusters (%d/%dKB) for '%s'", 502234228Sluigi p->_numclusters, p->_clustsize >> 10, 503234228Sluigi p->_memtotal >> 10, name); 504234228Sluigi 505234228Sluigi return p; 506234228Sluigi 507234228Sluigiclean: 508234228Sluigi netmap_destroy_obj_allocator(p); 509234228Sluigi return NULL; 510234228Sluigi} 511234228Sluigi 512234228Sluigistatic int 513234228Sluiginetmap_memory_init(void) 514234228Sluigi{ 515234228Sluigi struct netmap_obj_pool *p; 516234228Sluigi 517234228Sluigi nm_mem = malloc(sizeof(struct netmap_mem_d), M_NETMAP, 518234228Sluigi M_WAITOK | M_ZERO); 519234228Sluigi if (nm_mem == NULL) 520234228Sluigi goto clean; 521234228Sluigi 522234228Sluigi p = netmap_new_obj_allocator("netmap_if", 523234228Sluigi NETMAP_IF_MAX_NUM, NETMAP_IF_MAX_SIZE); 524234228Sluigi if (p == NULL) 525234228Sluigi goto clean; 526234228Sluigi nm_mem->nm_if_pool = p; 527234228Sluigi 528234228Sluigi p = netmap_new_obj_allocator("netmap_ring", 529234228Sluigi NETMAP_RING_MAX_NUM, NETMAP_RING_MAX_SIZE); 530234228Sluigi if (p == NULL) 531234228Sluigi goto clean; 532234228Sluigi nm_mem->nm_ring_pool = p; 533234228Sluigi 534234228Sluigi p = netmap_new_obj_allocator("netmap_buf", 535234228Sluigi NETMAP_BUF_MAX_NUM, NETMAP_BUF_SIZE); 536234228Sluigi if (p == NULL) 537234228Sluigi goto clean; 538234228Sluigi netmap_total_buffers = p->objtotal; 539234228Sluigi netmap_buffer_lut = p->lut; 540234228Sluigi nm_mem->nm_buf_pool = p; 541234228Sluigi netmap_buffer_base = p->lut[0].vaddr; 542234228Sluigi 543234228Sluigi mtx_init(&nm_mem->nm_mtx, "netmap memory allocator lock", NULL, 544234228Sluigi MTX_DEF); 545234228Sluigi nm_mem->nm_totalsize = 546234228Sluigi nm_mem->nm_if_pool->_memtotal + 547234228Sluigi nm_mem->nm_ring_pool->_memtotal + 548234228Sluigi nm_mem->nm_buf_pool->_memtotal; 549234228Sluigi 550234228Sluigi D("Have %d KB for interfaces, %d KB for rings and %d MB for buffers", 551234228Sluigi nm_mem->nm_if_pool->_memtotal >> 10, 552234228Sluigi nm_mem->nm_ring_pool->_memtotal >> 10, 553234228Sluigi nm_mem->nm_buf_pool->_memtotal >> 20); 554234228Sluigi return 0; 555234228Sluigi 556234228Sluigiclean: 557234228Sluigi if (nm_mem) { 558234228Sluigi netmap_destroy_obj_allocator(nm_mem->nm_ring_pool); 559234228Sluigi netmap_destroy_obj_allocator(nm_mem->nm_if_pool); 560234228Sluigi free(nm_mem, M_NETMAP); 561234228Sluigi } 562234228Sluigi return ENOMEM; 563234228Sluigi} 564234228Sluigi 565234228Sluigi 566234228Sluigistatic void 567234228Sluiginetmap_memory_fini(void) 568234228Sluigi{ 569234228Sluigi if (!nm_mem) 570234228Sluigi return; 571234228Sluigi netmap_destroy_obj_allocator(nm_mem->nm_if_pool); 572234228Sluigi netmap_destroy_obj_allocator(nm_mem->nm_ring_pool); 573234228Sluigi netmap_destroy_obj_allocator(nm_mem->nm_buf_pool); 574234228Sluigi mtx_destroy(&nm_mem->nm_mtx); 575234228Sluigi free(nm_mem, M_NETMAP); 576234228Sluigi} 577234228Sluigi 578234228Sluigi 579234228Sluigi 580234228Sluigistatic void * 581234228Sluiginetmap_if_new(const char *ifname, struct netmap_adapter *na) 582234228Sluigi{ 583234228Sluigi struct netmap_if *nifp; 584234228Sluigi struct netmap_ring *ring; 585234228Sluigi ssize_t base; /* handy for relative offsets between rings and nifp */ 586234228Sluigi u_int i, len, ndesc; 587234228Sluigi u_int ntx = na->num_tx_rings + 1; /* shorthand, include stack ring */ 588234228Sluigi u_int nrx = na->num_rx_rings + 1; /* shorthand, include stack ring */ 589234228Sluigi struct netmap_kring *kring; 590234228Sluigi 591234228Sluigi NMA_LOCK(); 592234228Sluigi /* 593234228Sluigi * the descriptor is followed inline by an array of offsets 594234228Sluigi * to the tx and rx rings in the shared memory region. 595234228Sluigi */ 596234228Sluigi len = sizeof(struct netmap_if) + (nrx + ntx) * sizeof(ssize_t); 597234228Sluigi nifp = netmap_if_malloc(len); 598234228Sluigi if (nifp == NULL) { 599234228Sluigi NMA_UNLOCK(); 600234228Sluigi return NULL; 601234228Sluigi } 602234228Sluigi 603234228Sluigi /* initialize base fields -- override const */ 604234228Sluigi *(int *)(uintptr_t)&nifp->ni_tx_rings = na->num_tx_rings; 605234228Sluigi *(int *)(uintptr_t)&nifp->ni_rx_rings = na->num_rx_rings; 606234228Sluigi strncpy(nifp->ni_name, ifname, IFNAMSIZ); 607234228Sluigi 608234228Sluigi (na->refcount)++; /* XXX atomic ? we are under lock */ 609234228Sluigi if (na->refcount > 1) { /* already setup, we are done */ 610234228Sluigi NMA_UNLOCK(); 611234228Sluigi goto final; 612234228Sluigi } 613234228Sluigi 614234228Sluigi /* 615234228Sluigi * First instance, allocate netmap rings and buffers for this card 616234228Sluigi * The rings are contiguous, but have variable size. 617234228Sluigi */ 618234228Sluigi for (i = 0; i < ntx; i++) { /* Transmit rings */ 619234228Sluigi kring = &na->tx_rings[i]; 620234228Sluigi ndesc = na->num_tx_desc; 621234228Sluigi bzero(kring, sizeof(*kring)); 622234228Sluigi len = sizeof(struct netmap_ring) + 623234228Sluigi ndesc * sizeof(struct netmap_slot); 624234228Sluigi ring = netmap_ring_malloc(len); 625234228Sluigi if (ring == NULL) { 626234228Sluigi D("Cannot allocate tx_ring[%d] for %s", i, ifname); 627234228Sluigi goto cleanup; 628234228Sluigi } 629234228Sluigi ND("txring[%d] at %p ofs %d", i, ring); 630234228Sluigi kring->na = na; 631234228Sluigi kring->ring = ring; 632234228Sluigi *(int *)(uintptr_t)&ring->num_slots = kring->nkr_num_slots = ndesc; 633234228Sluigi *(ssize_t *)(uintptr_t)&ring->buf_ofs = 634234228Sluigi (nm_mem->nm_if_pool->_memtotal + 635234228Sluigi nm_mem->nm_ring_pool->_memtotal) - 636234228Sluigi netmap_ring_offset(ring); 637234228Sluigi 638234228Sluigi /* 639234228Sluigi * IMPORTANT: 640234228Sluigi * Always keep one slot empty, so we can detect new 641234228Sluigi * transmissions comparing cur and nr_hwcur (they are 642234228Sluigi * the same only if there are no new transmissions). 643234228Sluigi */ 644234228Sluigi ring->avail = kring->nr_hwavail = ndesc - 1; 645234228Sluigi ring->cur = kring->nr_hwcur = 0; 646234228Sluigi *(int *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE; 647234228Sluigi ND("initializing slots for txring[%d]", i); 648234228Sluigi netmap_new_bufs(nifp, ring->slot, ndesc); 649234228Sluigi } 650234228Sluigi 651234228Sluigi for (i = 0; i < nrx; i++) { /* Receive rings */ 652234228Sluigi kring = &na->rx_rings[i]; 653234228Sluigi ndesc = na->num_rx_desc; 654234228Sluigi bzero(kring, sizeof(*kring)); 655234228Sluigi len = sizeof(struct netmap_ring) + 656234228Sluigi ndesc * sizeof(struct netmap_slot); 657234228Sluigi ring = netmap_ring_malloc(len); 658234228Sluigi if (ring == NULL) { 659234228Sluigi D("Cannot allocate rx_ring[%d] for %s", i, ifname); 660234228Sluigi goto cleanup; 661234228Sluigi } 662234228Sluigi ND("rxring[%d] at %p ofs %d", i, ring); 663234228Sluigi 664234228Sluigi kring->na = na; 665234228Sluigi kring->ring = ring; 666234228Sluigi *(int *)(uintptr_t)&ring->num_slots = kring->nkr_num_slots = ndesc; 667234228Sluigi *(ssize_t *)(uintptr_t)&ring->buf_ofs = 668234228Sluigi (nm_mem->nm_if_pool->_memtotal + 669234228Sluigi nm_mem->nm_ring_pool->_memtotal) - 670234228Sluigi netmap_ring_offset(ring); 671234228Sluigi 672234228Sluigi ring->cur = kring->nr_hwcur = 0; 673234228Sluigi ring->avail = kring->nr_hwavail = 0; /* empty */ 674234228Sluigi *(int *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE; 675234228Sluigi ND("initializing slots for rxring[%d]", i); 676234228Sluigi netmap_new_bufs(nifp, ring->slot, ndesc); 677234228Sluigi } 678234228Sluigi NMA_UNLOCK(); 679234228Sluigi#ifdef linux 680234228Sluigi // XXX initialize the selrecord structs. 681234228Sluigi for (i = 0; i < ntx; i++) 682234228Sluigi init_waitqueue_head(&na->rx_rings[i].si); 683234228Sluigi for (i = 0; i < nrx; i++) 684234228Sluigi init_waitqueue_head(&na->tx_rings[i].si); 685234228Sluigi init_waitqueue_head(&na->rx_si); 686234228Sluigi init_waitqueue_head(&na->tx_si); 687234228Sluigi#endif 688234228Sluigifinal: 689234228Sluigi /* 690234228Sluigi * fill the slots for the rx and tx rings. They contain the offset 691234228Sluigi * between the ring and nifp, so the information is usable in 692234228Sluigi * userspace to reach the ring from the nifp. 693234228Sluigi */ 694234228Sluigi base = netmap_if_offset(nifp); 695234228Sluigi for (i = 0; i < ntx; i++) { 696234228Sluigi *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = 697234228Sluigi netmap_ring_offset(na->tx_rings[i].ring) - base; 698234228Sluigi } 699234228Sluigi for (i = 0; i < nrx; i++) { 700234228Sluigi *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+ntx] = 701234228Sluigi netmap_ring_offset(na->rx_rings[i].ring) - base; 702234228Sluigi } 703234228Sluigi return (nifp); 704234228Sluigicleanup: 705234228Sluigi // XXX missing 706234228Sluigi NMA_UNLOCK(); 707234228Sluigi return NULL; 708234228Sluigi} 709234228Sluigi 710234228Sluigistatic void 711234228Sluiginetmap_free_rings(struct netmap_adapter *na) 712234228Sluigi{ 713234228Sluigi int i; 714234228Sluigi for (i = 0; i < na->num_tx_rings + 1; i++) 715234228Sluigi netmap_obj_free_va(nm_mem->nm_ring_pool, 716234228Sluigi na->tx_rings[i].ring); 717234228Sluigi for (i = 0; i < na->num_rx_rings + 1; i++) 718234228Sluigi netmap_obj_free_va(nm_mem->nm_ring_pool, 719234228Sluigi na->rx_rings[i].ring); 720234228Sluigi} 721