netmap_mem2.c revision 259412
1/* 2 * Copyright (C) 2012-2013 Matteo Landi, Luigi Rizzo, Giuseppe Lettieri. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26#ifdef linux 27#include "bsd_glue.h" 28#endif /* linux */ 29 30#ifdef __APPLE__ 31#include "osx_glue.h" 32#endif /* __APPLE__ */ 33 34#ifdef __FreeBSD__ 35#include <sys/cdefs.h> /* prerequisite */ 36__FBSDID("$FreeBSD: head/sys/dev/netmap/netmap_mem2.c 259412 2013-12-15 08:37:24Z luigi $"); 37 38#include <sys/types.h> 39#include <sys/malloc.h> 40#include <sys/proc.h> 41#include <vm/vm.h> /* vtophys */ 42#include <vm/pmap.h> /* vtophys */ 43#include <sys/socket.h> /* sockaddrs */ 44#include <sys/selinfo.h> 45#include <sys/sysctl.h> 46#include <net/if.h> 47#include <net/if_var.h> 48#include <net/vnet.h> 49#include <machine/bus.h> /* bus_dmamap_* */ 50 51#endif /* __FreeBSD__ */ 52 53#include <net/netmap.h> 54#include <dev/netmap/netmap_kern.h> 55#include "netmap_mem2.h" 56 57#ifdef linux 58#define NMA_LOCK_INIT(n) sema_init(&(n)->nm_mtx, 1) 59#define NMA_LOCK_DESTROY(n) 60#define NMA_LOCK(n) down(&(n)->nm_mtx) 61#define NMA_UNLOCK(n) up(&(n)->nm_mtx) 62#else /* !linux */ 63#define NMA_LOCK_INIT(n) mtx_init(&(n)->nm_mtx, "netmap memory allocator lock", NULL, MTX_DEF) 64#define NMA_LOCK_DESTROY(n) mtx_destroy(&(n)->nm_mtx) 65#define NMA_LOCK(n) mtx_lock(&(n)->nm_mtx) 66#define NMA_UNLOCK(n) mtx_unlock(&(n)->nm_mtx) 67#endif /* linux */ 68 69 70struct netmap_obj_params netmap_params[NETMAP_POOLS_NR] = { 71 [NETMAP_IF_POOL] = { 72 .size = 1024, 73 .num = 100, 74 }, 75 [NETMAP_RING_POOL] = { 76 .size = 9*PAGE_SIZE, 77 .num = 200, 78 }, 79 [NETMAP_BUF_POOL] = { 80 .size = 2048, 81 .num = NETMAP_BUF_MAX_NUM, 82 }, 83}; 84 85 86/* 87 * nm_mem is the memory allocator used for all physical interfaces 88 * running in netmap mode. 89 * Virtual (VALE) ports will have each its own allocator. 90 */ 91static int netmap_mem_global_config(struct netmap_mem_d *nmd); 92static int netmap_mem_global_finalize(struct netmap_mem_d *nmd); 93static void netmap_mem_global_deref(struct netmap_mem_d *nmd); 94struct netmap_mem_d nm_mem = { /* Our memory allocator. */ 95 .pools = { 96 [NETMAP_IF_POOL] = { 97 .name = "netmap_if", 98 .objminsize = sizeof(struct netmap_if), 99 .objmaxsize = 4096, 100 .nummin = 10, /* don't be stingy */ 101 .nummax = 10000, /* XXX very large */ 102 }, 103 [NETMAP_RING_POOL] = { 104 .name = "netmap_ring", 105 .objminsize = sizeof(struct netmap_ring), 106 .objmaxsize = 32*PAGE_SIZE, 107 .nummin = 2, 108 .nummax = 1024, 109 }, 110 [NETMAP_BUF_POOL] = { 111 .name = "netmap_buf", 112 .objminsize = 64, 113 .objmaxsize = 65536, 114 .nummin = 4, 115 .nummax = 1000000, /* one million! */ 116 }, 117 }, 118 .config = netmap_mem_global_config, 119 .finalize = netmap_mem_global_finalize, 120 .deref = netmap_mem_global_deref, 121}; 122 123 124// XXX logically belongs to nm_mem 125struct lut_entry *netmap_buffer_lut; /* exported */ 126 127/* blueprint for the private memory allocators */ 128static int netmap_mem_private_config(struct netmap_mem_d *nmd); 129static int netmap_mem_private_finalize(struct netmap_mem_d *nmd); 130static void netmap_mem_private_deref(struct netmap_mem_d *nmd); 131const struct netmap_mem_d nm_blueprint = { 132 .pools = { 133 [NETMAP_IF_POOL] = { 134 .name = "%s_if", 135 .objminsize = sizeof(struct netmap_if), 136 .objmaxsize = 4096, 137 .nummin = 1, 138 .nummax = 10, 139 }, 140 [NETMAP_RING_POOL] = { 141 .name = "%s_ring", 142 .objminsize = sizeof(struct netmap_ring), 143 .objmaxsize = 32*PAGE_SIZE, 144 .nummin = 2, 145 .nummax = 1024, 146 }, 147 [NETMAP_BUF_POOL] = { 148 .name = "%s_buf", 149 .objminsize = 64, 150 .objmaxsize = 65536, 151 .nummin = 4, 152 .nummax = 1000000, /* one million! */ 153 }, 154 }, 155 .config = netmap_mem_private_config, 156 .finalize = netmap_mem_private_finalize, 157 .deref = netmap_mem_private_deref, 158 159 .flags = NETMAP_MEM_PRIVATE, 160}; 161 162/* memory allocator related sysctls */ 163 164#define STRINGIFY(x) #x 165 166 167#define DECLARE_SYSCTLS(id, name) \ 168 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_size, \ 169 CTLFLAG_RW, &netmap_params[id].size, 0, "Requested size of netmap " STRINGIFY(name) "s"); \ 170 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_curr_size, \ 171 CTLFLAG_RD, &nm_mem.pools[id]._objsize, 0, "Current size of netmap " STRINGIFY(name) "s"); \ 172 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_num, \ 173 CTLFLAG_RW, &netmap_params[id].num, 0, "Requested number of netmap " STRINGIFY(name) "s"); \ 174 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_curr_num, \ 175 CTLFLAG_RD, &nm_mem.pools[id].objtotal, 0, "Current number of netmap " STRINGIFY(name) "s") 176 177SYSCTL_DECL(_dev_netmap); 178DECLARE_SYSCTLS(NETMAP_IF_POOL, if); 179DECLARE_SYSCTLS(NETMAP_RING_POOL, ring); 180DECLARE_SYSCTLS(NETMAP_BUF_POOL, buf); 181 182/* 183 * First, find the allocator that contains the requested offset, 184 * then locate the cluster through a lookup table. 185 */ 186vm_paddr_t 187netmap_mem_ofstophys(struct netmap_mem_d* nmd, vm_ooffset_t offset) 188{ 189 int i; 190 vm_ooffset_t o = offset; 191 vm_paddr_t pa; 192 struct netmap_obj_pool *p; 193 194 NMA_LOCK(nmd); 195 p = nmd->pools; 196 197 for (i = 0; i < NETMAP_POOLS_NR; offset -= p[i].memtotal, i++) { 198 if (offset >= p[i].memtotal) 199 continue; 200 // now lookup the cluster's address 201 pa = p[i].lut[offset / p[i]._objsize].paddr + 202 offset % p[i]._objsize; 203 NMA_UNLOCK(nmd); 204 return pa; 205 } 206 /* this is only in case of errors */ 207 D("invalid ofs 0x%x out of 0x%x 0x%x 0x%x", (u_int)o, 208 p[NETMAP_IF_POOL].memtotal, 209 p[NETMAP_IF_POOL].memtotal 210 + p[NETMAP_RING_POOL].memtotal, 211 p[NETMAP_IF_POOL].memtotal 212 + p[NETMAP_RING_POOL].memtotal 213 + p[NETMAP_BUF_POOL].memtotal); 214 NMA_UNLOCK(nmd); 215 return 0; // XXX bad address 216} 217 218int 219netmap_mem_get_info(struct netmap_mem_d* nmd, u_int* size, u_int *memflags) 220{ 221 int error = 0; 222 NMA_LOCK(nmd); 223 error = nmd->config(nmd); 224 if (error) 225 goto out; 226 if (nmd->flags & NETMAP_MEM_FINALIZED) { 227 *size = nmd->nm_totalsize; 228 } else { 229 int i; 230 *size = 0; 231 for (i = 0; i < NETMAP_POOLS_NR; i++) { 232 struct netmap_obj_pool *p = nmd->pools + i; 233 *size += (p->_numclusters * p->_clustsize); 234 } 235 } 236 *memflags = nmd->flags; 237out: 238 NMA_UNLOCK(nmd); 239 return error; 240} 241 242/* 243 * we store objects by kernel address, need to find the offset 244 * within the pool to export the value to userspace. 245 * Algorithm: scan until we find the cluster, then add the 246 * actual offset in the cluster 247 */ 248static ssize_t 249netmap_obj_offset(struct netmap_obj_pool *p, const void *vaddr) 250{ 251 int i, k = p->_clustentries, n = p->objtotal; 252 ssize_t ofs = 0; 253 254 for (i = 0; i < n; i += k, ofs += p->_clustsize) { 255 const char *base = p->lut[i].vaddr; 256 ssize_t relofs = (const char *) vaddr - base; 257 258 if (relofs < 0 || relofs >= p->_clustsize) 259 continue; 260 261 ofs = ofs + relofs; 262 ND("%s: return offset %d (cluster %d) for pointer %p", 263 p->name, ofs, i, vaddr); 264 return ofs; 265 } 266 D("address %p is not contained inside any cluster (%s)", 267 vaddr, p->name); 268 return 0; /* An error occurred */ 269} 270 271/* Helper functions which convert virtual addresses to offsets */ 272#define netmap_if_offset(n, v) \ 273 netmap_obj_offset(&(n)->pools[NETMAP_IF_POOL], (v)) 274 275#define netmap_ring_offset(n, v) \ 276 ((n)->pools[NETMAP_IF_POOL].memtotal + \ 277 netmap_obj_offset(&(n)->pools[NETMAP_RING_POOL], (v))) 278 279#define netmap_buf_offset(n, v) \ 280 ((n)->pools[NETMAP_IF_POOL].memtotal + \ 281 (n)->pools[NETMAP_RING_POOL].memtotal + \ 282 netmap_obj_offset(&(n)->pools[NETMAP_BUF_POOL], (v))) 283 284 285ssize_t 286netmap_mem_if_offset(struct netmap_mem_d *nmd, const void *addr) 287{ 288 ssize_t v; 289 NMA_LOCK(nmd); 290 v = netmap_if_offset(nmd, addr); 291 NMA_UNLOCK(nmd); 292 return v; 293} 294 295/* 296 * report the index, and use start position as a hint, 297 * otherwise buffer allocation becomes terribly expensive. 298 */ 299static void * 300netmap_obj_malloc(struct netmap_obj_pool *p, u_int len, uint32_t *start, uint32_t *index) 301{ 302 uint32_t i = 0; /* index in the bitmap */ 303 uint32_t mask, j; /* slot counter */ 304 void *vaddr = NULL; 305 306 if (len > p->_objsize) { 307 D("%s request size %d too large", p->name, len); 308 // XXX cannot reduce the size 309 return NULL; 310 } 311 312 if (p->objfree == 0) { 313 D("no more %s objects", p->name); 314 return NULL; 315 } 316 if (start) 317 i = *start; 318 319 /* termination is guaranteed by p->free, but better check bounds on i */ 320 while (vaddr == NULL && i < p->bitmap_slots) { 321 uint32_t cur = p->bitmap[i]; 322 if (cur == 0) { /* bitmask is fully used */ 323 i++; 324 continue; 325 } 326 /* locate a slot */ 327 for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) 328 ; 329 330 p->bitmap[i] &= ~mask; /* mark object as in use */ 331 p->objfree--; 332 333 vaddr = p->lut[i * 32 + j].vaddr; 334 if (index) 335 *index = i * 32 + j; 336 } 337 ND("%s allocator: allocated object @ [%d][%d]: vaddr %p", i, j, vaddr); 338 339 if (start) 340 *start = i; 341 return vaddr; 342} 343 344 345/* 346 * free by index, not by address. This is slow, but is only used 347 * for a small number of objects (rings, nifp) 348 */ 349static void 350netmap_obj_free(struct netmap_obj_pool *p, uint32_t j) 351{ 352 if (j >= p->objtotal) { 353 D("invalid index %u, max %u", j, p->objtotal); 354 return; 355 } 356 p->bitmap[j / 32] |= (1 << (j % 32)); 357 p->objfree++; 358 return; 359} 360 361static void 362netmap_obj_free_va(struct netmap_obj_pool *p, void *vaddr) 363{ 364 u_int i, j, n = p->numclusters; 365 366 for (i = 0, j = 0; i < n; i++, j += p->_clustentries) { 367 void *base = p->lut[i * p->_clustentries].vaddr; 368 ssize_t relofs = (ssize_t) vaddr - (ssize_t) base; 369 370 /* Given address, is out of the scope of the current cluster.*/ 371 if (vaddr < base || relofs >= p->_clustsize) 372 continue; 373 374 j = j + relofs / p->_objsize; 375 /* KASSERT(j != 0, ("Cannot free object 0")); */ 376 netmap_obj_free(p, j); 377 return; 378 } 379 D("address %p is not contained inside any cluster (%s)", 380 vaddr, p->name); 381} 382 383#define netmap_if_malloc(n, len) netmap_obj_malloc(&(n)->pools[NETMAP_IF_POOL], len, NULL, NULL) 384#define netmap_if_free(n, v) netmap_obj_free_va(&(n)->pools[NETMAP_IF_POOL], (v)) 385#define netmap_ring_malloc(n, len) netmap_obj_malloc(&(n)->pools[NETMAP_RING_POOL], len, NULL, NULL) 386#define netmap_ring_free(n, v) netmap_obj_free_va(&(n)->pools[NETMAP_RING_POOL], (v)) 387#define netmap_buf_malloc(n, _pos, _index) \ 388 netmap_obj_malloc(&(n)->pools[NETMAP_BUF_POOL], NETMAP_BDG_BUF_SIZE(n), _pos, _index) 389 390 391/* Return the index associated to the given packet buffer */ 392#define netmap_buf_index(n, v) \ 393 (netmap_obj_offset(&(n)->pools[NETMAP_BUF_POOL], (v)) / NETMAP_BDG_BUF_SIZE(n)) 394 395 396/* Return nonzero on error */ 397static int 398netmap_new_bufs(struct netmap_mem_d *nmd, struct netmap_slot *slot, u_int n) 399{ 400 struct netmap_obj_pool *p = &nmd->pools[NETMAP_BUF_POOL]; 401 u_int i = 0; /* slot counter */ 402 uint32_t pos = 0; /* slot in p->bitmap */ 403 uint32_t index = 0; /* buffer index */ 404 405 for (i = 0; i < n; i++) { 406 void *vaddr = netmap_buf_malloc(nmd, &pos, &index); 407 if (vaddr == NULL) { 408 D("no more buffers after %d of %d", i, n); 409 goto cleanup; 410 } 411 slot[i].buf_idx = index; 412 slot[i].len = p->_objsize; 413 slot[i].flags = 0; 414 } 415 416 ND("allocated %d buffers, %d available, first at %d", n, p->objfree, pos); 417 return (0); 418 419cleanup: 420 while (i > 0) { 421 i--; 422 netmap_obj_free(p, slot[i].buf_idx); 423 } 424 bzero(slot, n * sizeof(slot[0])); 425 return (ENOMEM); 426} 427 428 429static void 430netmap_free_buf(struct netmap_mem_d *nmd, uint32_t i) 431{ 432 struct netmap_obj_pool *p = &nmd->pools[NETMAP_BUF_POOL]; 433 434 if (i < 2 || i >= p->objtotal) { 435 D("Cannot free buf#%d: should be in [2, %d[", i, p->objtotal); 436 return; 437 } 438 netmap_obj_free(p, i); 439} 440 441static void 442netmap_reset_obj_allocator(struct netmap_obj_pool *p) 443{ 444 445 if (p == NULL) 446 return; 447 if (p->bitmap) 448 free(p->bitmap, M_NETMAP); 449 p->bitmap = NULL; 450 if (p->lut) { 451 u_int i; 452 size_t sz = p->_clustsize; 453 454 for (i = 0; i < p->objtotal; i += p->_clustentries) { 455 if (p->lut[i].vaddr) 456 contigfree(p->lut[i].vaddr, sz, M_NETMAP); 457 } 458 bzero(p->lut, sizeof(struct lut_entry) * p->objtotal); 459#ifdef linux 460 vfree(p->lut); 461#else 462 free(p->lut, M_NETMAP); 463#endif 464 } 465 p->lut = NULL; 466 p->objtotal = 0; 467 p->memtotal = 0; 468 p->numclusters = 0; 469 p->objfree = 0; 470} 471 472/* 473 * Free all resources related to an allocator. 474 */ 475static void 476netmap_destroy_obj_allocator(struct netmap_obj_pool *p) 477{ 478 if (p == NULL) 479 return; 480 netmap_reset_obj_allocator(p); 481} 482 483/* 484 * We receive a request for objtotal objects, of size objsize each. 485 * Internally we may round up both numbers, as we allocate objects 486 * in small clusters multiple of the page size. 487 * We need to keep track of objtotal and clustentries, 488 * as they are needed when freeing memory. 489 * 490 * XXX note -- userspace needs the buffers to be contiguous, 491 * so we cannot afford gaps at the end of a cluster. 492 */ 493 494 495/* call with NMA_LOCK held */ 496static int 497netmap_config_obj_allocator(struct netmap_obj_pool *p, u_int objtotal, u_int objsize) 498{ 499 int i; 500 u_int clustsize; /* the cluster size, multiple of page size */ 501 u_int clustentries; /* how many objects per entry */ 502 503 /* we store the current request, so we can 504 * detect configuration changes later */ 505 p->r_objtotal = objtotal; 506 p->r_objsize = objsize; 507 508#define MAX_CLUSTSIZE (1<<17) 509#define LINE_ROUND 64 510 if (objsize >= MAX_CLUSTSIZE) { 511 /* we could do it but there is no point */ 512 D("unsupported allocation for %d bytes", objsize); 513 return EINVAL; 514 } 515 /* make sure objsize is a multiple of LINE_ROUND */ 516 i = (objsize & (LINE_ROUND - 1)); 517 if (i) { 518 D("XXX aligning object by %d bytes", LINE_ROUND - i); 519 objsize += LINE_ROUND - i; 520 } 521 if (objsize < p->objminsize || objsize > p->objmaxsize) { 522 D("requested objsize %d out of range [%d, %d]", 523 objsize, p->objminsize, p->objmaxsize); 524 return EINVAL; 525 } 526 if (objtotal < p->nummin || objtotal > p->nummax) { 527 D("requested objtotal %d out of range [%d, %d]", 528 objtotal, p->nummin, p->nummax); 529 return EINVAL; 530 } 531 /* 532 * Compute number of objects using a brute-force approach: 533 * given a max cluster size, 534 * we try to fill it with objects keeping track of the 535 * wasted space to the next page boundary. 536 */ 537 for (clustentries = 0, i = 1;; i++) { 538 u_int delta, used = i * objsize; 539 if (used > MAX_CLUSTSIZE) 540 break; 541 delta = used % PAGE_SIZE; 542 if (delta == 0) { // exact solution 543 clustentries = i; 544 break; 545 } 546 if (delta > ( (clustentries*objsize) % PAGE_SIZE) ) 547 clustentries = i; 548 } 549 // D("XXX --- ouch, delta %d (bad for buffers)", delta); 550 /* compute clustsize and round to the next page */ 551 clustsize = clustentries * objsize; 552 i = (clustsize & (PAGE_SIZE - 1)); 553 if (i) 554 clustsize += PAGE_SIZE - i; 555 if (netmap_verbose) 556 D("objsize %d clustsize %d objects %d", 557 objsize, clustsize, clustentries); 558 559 /* 560 * The number of clusters is n = ceil(objtotal/clustentries) 561 * objtotal' = n * clustentries 562 */ 563 p->_clustentries = clustentries; 564 p->_clustsize = clustsize; 565 p->_numclusters = (objtotal + clustentries - 1) / clustentries; 566 567 /* actual values (may be larger than requested) */ 568 p->_objsize = objsize; 569 p->_objtotal = p->_numclusters * clustentries; 570 571 return 0; 572} 573 574 575/* call with NMA_LOCK held */ 576static int 577netmap_finalize_obj_allocator(struct netmap_obj_pool *p) 578{ 579 int i; /* must be signed */ 580 size_t n; 581 582 /* optimistically assume we have enough memory */ 583 p->numclusters = p->_numclusters; 584 p->objtotal = p->_objtotal; 585 586 n = sizeof(struct lut_entry) * p->objtotal; 587#ifdef linux 588 p->lut = vmalloc(n); 589#else 590 p->lut = malloc(n, M_NETMAP, M_NOWAIT | M_ZERO); 591#endif 592 if (p->lut == NULL) { 593 D("Unable to create lookup table (%d bytes) for '%s'", (int)n, p->name); 594 goto clean; 595 } 596 597 /* Allocate the bitmap */ 598 n = (p->objtotal + 31) / 32; 599 p->bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP, M_NOWAIT | M_ZERO); 600 if (p->bitmap == NULL) { 601 D("Unable to create bitmap (%d entries) for allocator '%s'", (int)n, 602 p->name); 603 goto clean; 604 } 605 p->bitmap_slots = n; 606 607 /* 608 * Allocate clusters, init pointers and bitmap 609 */ 610 611 n = p->_clustsize; 612 for (i = 0; i < (int)p->objtotal;) { 613 int lim = i + p->_clustentries; 614 char *clust; 615 616 clust = contigmalloc(n, M_NETMAP, M_NOWAIT | M_ZERO, 617 (size_t)0, -1UL, PAGE_SIZE, 0); 618 if (clust == NULL) { 619 /* 620 * If we get here, there is a severe memory shortage, 621 * so halve the allocated memory to reclaim some. 622 */ 623 D("Unable to create cluster at %d for '%s' allocator", 624 i, p->name); 625 if (i < 2) /* nothing to halve */ 626 goto out; 627 lim = i / 2; 628 for (i--; i >= lim; i--) { 629 p->bitmap[ (i>>5) ] &= ~( 1 << (i & 31) ); 630 if (i % p->_clustentries == 0 && p->lut[i].vaddr) 631 contigfree(p->lut[i].vaddr, 632 n, M_NETMAP); 633 } 634 out: 635 p->objtotal = i; 636 /* we may have stopped in the middle of a cluster */ 637 p->numclusters = (i + p->_clustentries - 1) / p->_clustentries; 638 break; 639 } 640 for (; i < lim; i++, clust += p->_objsize) { 641 p->bitmap[ (i>>5) ] |= ( 1 << (i & 31) ); 642 p->lut[i].vaddr = clust; 643 p->lut[i].paddr = vtophys(clust); 644 } 645 } 646 p->objfree = p->objtotal; 647 p->memtotal = p->numclusters * p->_clustsize; 648 if (p->objfree == 0) 649 goto clean; 650 if (netmap_verbose) 651 D("Pre-allocated %d clusters (%d/%dKB) for '%s'", 652 p->numclusters, p->_clustsize >> 10, 653 p->memtotal >> 10, p->name); 654 655 return 0; 656 657clean: 658 netmap_reset_obj_allocator(p); 659 return ENOMEM; 660} 661 662/* call with lock held */ 663static int 664netmap_memory_config_changed(struct netmap_mem_d *nmd) 665{ 666 int i; 667 668 for (i = 0; i < NETMAP_POOLS_NR; i++) { 669 if (nmd->pools[i].r_objsize != netmap_params[i].size || 670 nmd->pools[i].r_objtotal != netmap_params[i].num) 671 return 1; 672 } 673 return 0; 674} 675 676static void 677netmap_mem_reset_all(struct netmap_mem_d *nmd) 678{ 679 int i; 680 D("resetting %p", nmd); 681 for (i = 0; i < NETMAP_POOLS_NR; i++) { 682 netmap_reset_obj_allocator(&nmd->pools[i]); 683 } 684 nmd->flags &= ~NETMAP_MEM_FINALIZED; 685} 686 687static int 688netmap_mem_finalize_all(struct netmap_mem_d *nmd) 689{ 690 int i; 691 if (nmd->flags & NETMAP_MEM_FINALIZED) 692 return 0; 693 nmd->lasterr = 0; 694 nmd->nm_totalsize = 0; 695 for (i = 0; i < NETMAP_POOLS_NR; i++) { 696 nmd->lasterr = netmap_finalize_obj_allocator(&nmd->pools[i]); 697 if (nmd->lasterr) 698 goto error; 699 nmd->nm_totalsize += nmd->pools[i].memtotal; 700 } 701 /* buffers 0 and 1 are reserved */ 702 nmd->pools[NETMAP_BUF_POOL].objfree -= 2; 703 nmd->pools[NETMAP_BUF_POOL].bitmap[0] = ~3; 704 nmd->flags |= NETMAP_MEM_FINALIZED; 705 706 D("Have %d KB for interfaces, %d KB for rings and %d MB for buffers", 707 nmd->pools[NETMAP_IF_POOL].memtotal >> 10, 708 nmd->pools[NETMAP_RING_POOL].memtotal >> 10, 709 nmd->pools[NETMAP_BUF_POOL].memtotal >> 20); 710 711 D("Free buffers: %d", nmd->pools[NETMAP_BUF_POOL].objfree); 712 713 714 return 0; 715error: 716 netmap_mem_reset_all(nmd); 717 return nmd->lasterr; 718} 719 720 721 722void 723netmap_mem_private_delete(struct netmap_mem_d *nmd) 724{ 725 if (nmd == NULL) 726 return; 727 D("deleting %p", nmd); 728 if (nmd->refcount > 0) 729 D("bug: deleting mem allocator with refcount=%d!", nmd->refcount); 730 D("done deleting %p", nmd); 731 NMA_LOCK_DESTROY(nmd); 732 free(nmd, M_DEVBUF); 733} 734 735static int 736netmap_mem_private_config(struct netmap_mem_d *nmd) 737{ 738 /* nothing to do, we are configured on creation 739 * and configuration never changes thereafter 740 */ 741 return 0; 742} 743 744static int 745netmap_mem_private_finalize(struct netmap_mem_d *nmd) 746{ 747 int err; 748 NMA_LOCK(nmd); 749 nmd->refcount++; 750 err = netmap_mem_finalize_all(nmd); 751 NMA_UNLOCK(nmd); 752 return err; 753 754} 755 756static void 757netmap_mem_private_deref(struct netmap_mem_d *nmd) 758{ 759 NMA_LOCK(nmd); 760 if (--nmd->refcount <= 0) 761 netmap_mem_reset_all(nmd); 762 NMA_UNLOCK(nmd); 763} 764 765struct netmap_mem_d * 766netmap_mem_private_new(const char *name, u_int txr, u_int txd, u_int rxr, u_int rxd) 767{ 768 struct netmap_mem_d *d = NULL; 769 struct netmap_obj_params p[NETMAP_POOLS_NR]; 770 int i; 771 u_int maxd; 772 773 d = malloc(sizeof(struct netmap_mem_d), 774 M_DEVBUF, M_NOWAIT | M_ZERO); 775 if (d == NULL) 776 return NULL; 777 778 *d = nm_blueprint; 779 780 /* XXX the rest of the code assumes the stack rings are alwasy present */ 781 txr++; 782 rxr++; 783 p[NETMAP_IF_POOL].size = sizeof(struct netmap_if) + 784 sizeof(ssize_t) * (txr + rxr); 785 p[NETMAP_IF_POOL].num = 2; 786 maxd = (txd > rxd) ? txd : rxd; 787 p[NETMAP_RING_POOL].size = sizeof(struct netmap_ring) + 788 sizeof(struct netmap_slot) * maxd; 789 p[NETMAP_RING_POOL].num = txr + rxr; 790 p[NETMAP_BUF_POOL].size = 2048; /* XXX find a way to let the user choose this */ 791 p[NETMAP_BUF_POOL].num = rxr * (rxd + 2) + txr * (txd + 2); 792 793 D("req if %d*%d ring %d*%d buf %d*%d", 794 p[NETMAP_IF_POOL].num, 795 p[NETMAP_IF_POOL].size, 796 p[NETMAP_RING_POOL].num, 797 p[NETMAP_RING_POOL].size, 798 p[NETMAP_BUF_POOL].num, 799 p[NETMAP_BUF_POOL].size); 800 801 for (i = 0; i < NETMAP_POOLS_NR; i++) { 802 snprintf(d->pools[i].name, NETMAP_POOL_MAX_NAMSZ, 803 nm_blueprint.pools[i].name, 804 name); 805 if (netmap_config_obj_allocator(&d->pools[i], 806 p[i].num, p[i].size)) 807 goto error; 808 } 809 810 d->flags &= ~NETMAP_MEM_FINALIZED; 811 812 NMA_LOCK_INIT(d); 813 814 return d; 815error: 816 netmap_mem_private_delete(d); 817 return NULL; 818} 819 820 821/* call with lock held */ 822static int 823netmap_mem_global_config(struct netmap_mem_d *nmd) 824{ 825 int i; 826 827 if (nmd->refcount) 828 /* already in use, we cannot change the configuration */ 829 goto out; 830 831 if (!netmap_memory_config_changed(nmd)) 832 goto out; 833 834 D("reconfiguring"); 835 836 if (nmd->flags & NETMAP_MEM_FINALIZED) { 837 /* reset previous allocation */ 838 for (i = 0; i < NETMAP_POOLS_NR; i++) { 839 netmap_reset_obj_allocator(&nmd->pools[i]); 840 } 841 nmd->flags &= ~NETMAP_MEM_FINALIZED; 842 } 843 844 for (i = 0; i < NETMAP_POOLS_NR; i++) { 845 nmd->lasterr = netmap_config_obj_allocator(&nmd->pools[i], 846 netmap_params[i].num, netmap_params[i].size); 847 if (nmd->lasterr) 848 goto out; 849 } 850 851out: 852 853 return nmd->lasterr; 854} 855 856static int 857netmap_mem_global_finalize(struct netmap_mem_d *nmd) 858{ 859 int err; 860 861 NMA_LOCK(nmd); 862 863 864 /* update configuration if changed */ 865 if (netmap_mem_global_config(nmd)) 866 goto out; 867 868 nmd->refcount++; 869 870 if (nmd->flags & NETMAP_MEM_FINALIZED) { 871 /* may happen if config is not changed */ 872 ND("nothing to do"); 873 goto out; 874 } 875 876 if (netmap_mem_finalize_all(nmd)) 877 goto out; 878 879 /* backward compatibility */ 880 netmap_buf_size = nmd->pools[NETMAP_BUF_POOL]._objsize; 881 netmap_total_buffers = nmd->pools[NETMAP_BUF_POOL].objtotal; 882 883 netmap_buffer_lut = nmd->pools[NETMAP_BUF_POOL].lut; 884 netmap_buffer_base = nmd->pools[NETMAP_BUF_POOL].lut[0].vaddr; 885 886 nmd->lasterr = 0; 887 888out: 889 if (nmd->lasterr) 890 nmd->refcount--; 891 err = nmd->lasterr; 892 893 NMA_UNLOCK(nmd); 894 895 return err; 896 897} 898 899int 900netmap_mem_init(void) 901{ 902 NMA_LOCK_INIT(&nm_mem); 903 return (0); 904} 905 906void 907netmap_mem_fini(void) 908{ 909 int i; 910 911 for (i = 0; i < NETMAP_POOLS_NR; i++) { 912 netmap_destroy_obj_allocator(&nm_mem.pools[i]); 913 } 914 NMA_LOCK_DESTROY(&nm_mem); 915} 916 917static void 918netmap_free_rings(struct netmap_adapter *na) 919{ 920 u_int i; 921 if (!na->tx_rings) 922 return; 923 for (i = 0; i < na->num_tx_rings + 1; i++) { 924 if (na->tx_rings[i].ring) { 925 netmap_ring_free(na->nm_mem, na->tx_rings[i].ring); 926 na->tx_rings[i].ring = NULL; 927 } 928 } 929 for (i = 0; i < na->num_rx_rings + 1; i++) { 930 if (na->rx_rings[i].ring) { 931 netmap_ring_free(na->nm_mem, na->rx_rings[i].ring); 932 na->rx_rings[i].ring = NULL; 933 } 934 } 935} 936 937/* call with NMA_LOCK held * 938 * 939 * Allocate netmap rings and buffers for this card 940 * The rings are contiguous, but have variable size. 941 */ 942int 943netmap_mem_rings_create(struct netmap_adapter *na) 944{ 945 struct netmap_ring *ring; 946 u_int len, ndesc; 947 struct netmap_kring *kring; 948 949 NMA_LOCK(na->nm_mem); 950 951 for (kring = na->tx_rings; kring != na->rx_rings; kring++) { /* Transmit rings */ 952 ndesc = kring->nkr_num_slots; 953 len = sizeof(struct netmap_ring) + 954 ndesc * sizeof(struct netmap_slot); 955 ring = netmap_ring_malloc(na->nm_mem, len); 956 if (ring == NULL) { 957 D("Cannot allocate tx_ring"); 958 goto cleanup; 959 } 960 ND("txring[%d] at %p ofs %d", i, ring); 961 kring->ring = ring; 962 *(uint32_t *)(uintptr_t)&ring->num_slots = ndesc; 963 *(ssize_t *)(uintptr_t)&ring->buf_ofs = 964 (na->nm_mem->pools[NETMAP_IF_POOL].memtotal + 965 na->nm_mem->pools[NETMAP_RING_POOL].memtotal) - 966 netmap_ring_offset(na->nm_mem, ring); 967 968 ring->avail = kring->nr_hwavail; 969 ring->cur = kring->nr_hwcur; 970 *(uint16_t *)(uintptr_t)&ring->nr_buf_size = 971 NETMAP_BDG_BUF_SIZE(na->nm_mem); 972 ND("initializing slots for txring"); 973 if (netmap_new_bufs(na->nm_mem, ring->slot, ndesc)) { 974 D("Cannot allocate buffers for tx_ring"); 975 goto cleanup; 976 } 977 } 978 979 for ( ; kring != na->tailroom; kring++) { /* Receive rings */ 980 ndesc = kring->nkr_num_slots; 981 len = sizeof(struct netmap_ring) + 982 ndesc * sizeof(struct netmap_slot); 983 ring = netmap_ring_malloc(na->nm_mem, len); 984 if (ring == NULL) { 985 D("Cannot allocate rx_ring"); 986 goto cleanup; 987 } 988 ND("rxring at %p ofs %d", ring); 989 990 kring->ring = ring; 991 *(uint32_t *)(uintptr_t)&ring->num_slots = ndesc; 992 *(ssize_t *)(uintptr_t)&ring->buf_ofs = 993 (na->nm_mem->pools[NETMAP_IF_POOL].memtotal + 994 na->nm_mem->pools[NETMAP_RING_POOL].memtotal) - 995 netmap_ring_offset(na->nm_mem, ring); 996 997 ring->cur = kring->nr_hwcur; 998 ring->avail = kring->nr_hwavail; 999 *(int *)(uintptr_t)&ring->nr_buf_size = 1000 NETMAP_BDG_BUF_SIZE(na->nm_mem); 1001 ND("initializing slots for rxring[%d]", i); 1002 if (netmap_new_bufs(na->nm_mem, ring->slot, ndesc)) { 1003 D("Cannot allocate buffers for rx_ring"); 1004 goto cleanup; 1005 } 1006 } 1007 1008 NMA_UNLOCK(na->nm_mem); 1009 1010 return 0; 1011 1012cleanup: 1013 netmap_free_rings(na); 1014 1015 NMA_UNLOCK(na->nm_mem); 1016 1017 return ENOMEM; 1018} 1019 1020void 1021netmap_mem_rings_delete(struct netmap_adapter *na) 1022{ 1023 /* last instance, release bufs and rings */ 1024 u_int i, lim; 1025 struct netmap_kring *kring; 1026 struct netmap_ring *ring; 1027 1028 NMA_LOCK(na->nm_mem); 1029 1030 for (kring = na->tx_rings; kring != na->tailroom; kring++) { 1031 ring = kring->ring; 1032 if (ring == NULL) 1033 continue; 1034 lim = kring->nkr_num_slots; 1035 for (i = 0; i < lim; i++) 1036 netmap_free_buf(na->nm_mem, ring->slot[i].buf_idx); 1037 } 1038 netmap_free_rings(na); 1039 1040 NMA_UNLOCK(na->nm_mem); 1041} 1042 1043 1044/* call with NMA_LOCK held */ 1045/* 1046 * Allocate the per-fd structure netmap_if. 1047 * 1048 * We assume that the configuration stored in na 1049 * (number of tx/rx rings and descs) does not change while 1050 * the interface is in netmap mode. 1051 */ 1052struct netmap_if * 1053netmap_mem_if_new(const char *ifname, struct netmap_adapter *na) 1054{ 1055 struct netmap_if *nifp; 1056 ssize_t base; /* handy for relative offsets between rings and nifp */ 1057 u_int i, len, ntx, nrx; 1058 1059 /* 1060 * verify whether virtual port need the stack ring 1061 */ 1062 ntx = na->num_tx_rings + 1; /* shorthand, include stack ring */ 1063 nrx = na->num_rx_rings + 1; /* shorthand, include stack ring */ 1064 /* 1065 * the descriptor is followed inline by an array of offsets 1066 * to the tx and rx rings in the shared memory region. 1067 * For virtual rx rings we also allocate an array of 1068 * pointers to assign to nkr_leases. 1069 */ 1070 1071 NMA_LOCK(na->nm_mem); 1072 1073 len = sizeof(struct netmap_if) + (nrx + ntx) * sizeof(ssize_t); 1074 nifp = netmap_if_malloc(na->nm_mem, len); 1075 if (nifp == NULL) { 1076 NMA_UNLOCK(na->nm_mem); 1077 return NULL; 1078 } 1079 1080 /* initialize base fields -- override const */ 1081 *(u_int *)(uintptr_t)&nifp->ni_tx_rings = na->num_tx_rings; 1082 *(u_int *)(uintptr_t)&nifp->ni_rx_rings = na->num_rx_rings; 1083 strncpy(nifp->ni_name, ifname, (size_t)IFNAMSIZ); 1084 1085 /* 1086 * fill the slots for the rx and tx rings. They contain the offset 1087 * between the ring and nifp, so the information is usable in 1088 * userspace to reach the ring from the nifp. 1089 */ 1090 base = netmap_if_offset(na->nm_mem, nifp); 1091 for (i = 0; i < ntx; i++) { 1092 *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = 1093 netmap_ring_offset(na->nm_mem, na->tx_rings[i].ring) - base; 1094 } 1095 for (i = 0; i < nrx; i++) { 1096 *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+ntx] = 1097 netmap_ring_offset(na->nm_mem, na->rx_rings[i].ring) - base; 1098 } 1099 1100 NMA_UNLOCK(na->nm_mem); 1101 1102 return (nifp); 1103} 1104 1105void 1106netmap_mem_if_delete(struct netmap_adapter *na, struct netmap_if *nifp) 1107{ 1108 if (nifp == NULL) 1109 /* nothing to do */ 1110 return; 1111 NMA_LOCK(na->nm_mem); 1112 1113 netmap_if_free(na->nm_mem, nifp); 1114 1115 NMA_UNLOCK(na->nm_mem); 1116} 1117 1118static void 1119netmap_mem_global_deref(struct netmap_mem_d *nmd) 1120{ 1121 NMA_LOCK(nmd); 1122 1123 nmd->refcount--; 1124 if (netmap_verbose) 1125 D("refcount = %d", nmd->refcount); 1126 1127 NMA_UNLOCK(nmd); 1128} 1129 1130int 1131netmap_mem_finalize(struct netmap_mem_d *nmd) 1132{ 1133 return nmd->finalize(nmd); 1134} 1135 1136void 1137netmap_mem_deref(struct netmap_mem_d *nmd) 1138{ 1139 return nmd->deref(nmd); 1140} 1141