1171095Ssam/*- 2171095Ssam * Copyright (c) 2002-2007 Neterion, Inc. 3171095Ssam * All rights reserved. 4171095Ssam * 5171095Ssam * Redistribution and use in source and binary forms, with or without 6171095Ssam * modification, are permitted provided that the following conditions 7171095Ssam * are met: 8171095Ssam * 1. Redistributions of source code must retain the above copyright 9171095Ssam * notice, this list of conditions and the following disclaimer. 10171095Ssam * 2. Redistributions in binary form must reproduce the above copyright 11171095Ssam * notice, this list of conditions and the following disclaimer in the 12171095Ssam * documentation and/or other materials provided with the distribution. 13171095Ssam * 14171095Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15171095Ssam * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16171095Ssam * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17171095Ssam * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18171095Ssam * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19171095Ssam * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20171095Ssam * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21171095Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22171095Ssam * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23171095Ssam * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24171095Ssam * SUCH DAMAGE. 25171095Ssam * 26171095Ssam * $FreeBSD: releng/11.0/sys/dev/nxge/xgehal/xgehal-mm.c 173139 2007-10-29 14:19:32Z rwatson $ 27171095Ssam */ 28171095Ssam 29171095Ssam#include <dev/nxge/include/xge-os-pal.h> 30171095Ssam#include <dev/nxge/include/xgehal-mm.h> 31171095Ssam#include <dev/nxge/include/xge-debug.h> 32171095Ssam 33171095Ssam/* 34171095Ssam * __hal_mempool_grow 35171095Ssam * 36171095Ssam * Will resize mempool up to %num_allocate value. 37171095Ssam */ 38171095Ssamxge_hal_status_e 39171095Ssam__hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate, 40173139Srwatson int *num_allocated) 41171095Ssam{ 42171095Ssam int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0; 43171095Ssam int n_items = mempool->items_per_memblock; 44171095Ssam 45171095Ssam *num_allocated = 0; 46171095Ssam 47171095Ssam if ((mempool->memblocks_allocated + num_allocate) > 48173139Srwatson mempool->memblocks_max) { 49173139Srwatson xge_debug_mm(XGE_ERR, "%s", 50173139Srwatson "__hal_mempool_grow: can grow anymore"); 51173139Srwatson return XGE_HAL_ERR_OUT_OF_MEMORY; 52171095Ssam } 53171095Ssam 54171095Ssam for (i = mempool->memblocks_allocated; 55171095Ssam i < mempool->memblocks_allocated + num_allocate; i++) { 56173139Srwatson int j; 57173139Srwatson int is_last = 58173139Srwatson ((mempool->memblocks_allocated+num_allocate-1) == i); 59173139Srwatson xge_hal_mempool_dma_t *dma_object = 60173139Srwatson mempool->memblocks_dma_arr + i; 61173139Srwatson void *the_memblock; 62173139Srwatson int dma_flags; 63171095Ssam 64173139Srwatson dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED; 65171095Ssam#ifdef XGE_HAL_DMA_DTR_CONSISTENT 66173139Srwatson dma_flags |= XGE_OS_DMA_CONSISTENT; 67171095Ssam#else 68173139Srwatson dma_flags |= XGE_OS_DMA_STREAMING; 69171095Ssam#endif 70171095Ssam 71173139Srwatson /* allocate DMA-capable memblock */ 72173139Srwatson mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev, 73173139Srwatson mempool->memblock_size, 74173139Srwatson dma_flags, 75173139Srwatson &dma_object->handle, 76173139Srwatson &dma_object->acc_handle); 77173139Srwatson if (mempool->memblocks_arr[i] == NULL) { 78173139Srwatson xge_debug_mm(XGE_ERR, 79173139Srwatson "memblock[%d]: out of DMA memory", i); 80173139Srwatson return XGE_HAL_ERR_OUT_OF_MEMORY; 81173139Srwatson } 82173139Srwatson xge_os_memzero(mempool->memblocks_arr[i], 83173139Srwatson mempool->memblock_size); 84173139Srwatson the_memblock = mempool->memblocks_arr[i]; 85171095Ssam 86173139Srwatson /* allocate memblock's private part. Each DMA memblock 87173139Srwatson * has a space allocated for item's private usage upon 88173139Srwatson * mempool's user request. Each time mempool grows, it will 89173139Srwatson * allocate new memblock and its private part at once. 90173139Srwatson * This helps to minimize memory usage a lot. */ 91173139Srwatson mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev, 92173139Srwatson mempool->items_priv_size * n_items); 93173139Srwatson if (mempool->memblocks_priv_arr[i] == NULL) { 94173139Srwatson xge_os_dma_free(mempool->pdev, 95173139Srwatson the_memblock, 96173139Srwatson mempool->memblock_size, 97173139Srwatson &dma_object->acc_handle, 98173139Srwatson &dma_object->handle); 99173139Srwatson xge_debug_mm(XGE_ERR, 100173139Srwatson "memblock_priv[%d]: out of virtual memory, " 101173139Srwatson "requested %d(%d:%d) bytes", i, 102173139Srwatson mempool->items_priv_size * n_items, 103173139Srwatson mempool->items_priv_size, n_items); 104173139Srwatson return XGE_HAL_ERR_OUT_OF_MEMORY; 105173139Srwatson } 106173139Srwatson xge_os_memzero(mempool->memblocks_priv_arr[i], 107173139Srwatson mempool->items_priv_size * n_items); 108171095Ssam 109173139Srwatson /* map memblock to physical memory */ 110173139Srwatson dma_object->addr = xge_os_dma_map(mempool->pdev, 111173139Srwatson dma_object->handle, 112173139Srwatson the_memblock, 113173139Srwatson mempool->memblock_size, 114173139Srwatson XGE_OS_DMA_DIR_BIDIRECTIONAL, 115171095Ssam#ifdef XGE_HAL_DMA_DTR_CONSISTENT 116173139Srwatson XGE_OS_DMA_CONSISTENT 117171095Ssam#else 118173139Srwatson XGE_OS_DMA_STREAMING 119171095Ssam#endif 120173139Srwatson ); 121173139Srwatson if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) { 122173139Srwatson xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 123173139Srwatson mempool->items_priv_size * 124173139Srwatson n_items); 125173139Srwatson xge_os_dma_free(mempool->pdev, 126173139Srwatson the_memblock, 127173139Srwatson mempool->memblock_size, 128173139Srwatson &dma_object->acc_handle, 129173139Srwatson &dma_object->handle); 130173139Srwatson return XGE_HAL_ERR_OUT_OF_MAPPING; 131173139Srwatson } 132171095Ssam 133173139Srwatson /* fill the items hash array */ 134173139Srwatson for (j=0; j<n_items; j++) { 135173139Srwatson int index = i*n_items + j; 136171095Ssam 137173139Srwatson if (first_time && index >= mempool->items_initial) { 138173139Srwatson break; 139173139Srwatson } 140171095Ssam 141173139Srwatson mempool->items_arr[index] = 142173139Srwatson ((char *)the_memblock + j*mempool->item_size); 143171095Ssam 144173139Srwatson /* let caller to do more job on each item */ 145173139Srwatson if (mempool->item_func_alloc != NULL) { 146173139Srwatson xge_hal_status_e status; 147171095Ssam 148173139Srwatson if ((status = mempool->item_func_alloc( 149173139Srwatson mempool, 150173139Srwatson the_memblock, 151173139Srwatson i, 152173139Srwatson dma_object, 153173139Srwatson mempool->items_arr[index], 154173139Srwatson index, 155173139Srwatson is_last, 156173139Srwatson mempool->userdata)) != XGE_HAL_OK) { 157171095Ssam 158173139Srwatson if (mempool->item_func_free != NULL) { 159173139Srwatson int k; 160171095Ssam 161173139Srwatson for (k=0; k<j; k++) { 162171095Ssam 163173139Srwatson index =i*n_items + k; 164171095Ssam 165173139Srwatson (void)mempool->item_func_free( 166173139Srwatson mempool, the_memblock, 167173139Srwatson i, dma_object, 168173139Srwatson mempool->items_arr[index], 169173139Srwatson index, is_last, 170173139Srwatson mempool->userdata); 171173139Srwatson } 172173139Srwatson } 173171095Ssam 174173139Srwatson xge_os_free(mempool->pdev, 175173139Srwatson mempool->memblocks_priv_arr[i], 176173139Srwatson mempool->items_priv_size * 177173139Srwatson n_items); 178173139Srwatson xge_os_dma_unmap(mempool->pdev, 179173139Srwatson dma_object->handle, 180173139Srwatson dma_object->addr, 181173139Srwatson mempool->memblock_size, 182173139Srwatson XGE_OS_DMA_DIR_BIDIRECTIONAL); 183173139Srwatson xge_os_dma_free(mempool->pdev, 184173139Srwatson the_memblock, 185173139Srwatson mempool->memblock_size, 186173139Srwatson &dma_object->acc_handle, 187173139Srwatson &dma_object->handle); 188173139Srwatson return status; 189173139Srwatson } 190173139Srwatson } 191171095Ssam 192173139Srwatson mempool->items_current = index + 1; 193173139Srwatson } 194171095Ssam 195173139Srwatson xge_debug_mm(XGE_TRACE, 196173139Srwatson "memblock%d: allocated %dk, vaddr 0x"XGE_OS_LLXFMT", " 197173139Srwatson "dma_addr 0x"XGE_OS_LLXFMT, i, mempool->memblock_size / 1024, 198173139Srwatson (unsigned long long)(ulong_t)mempool->memblocks_arr[i], 199173139Srwatson (unsigned long long)dma_object->addr); 200171095Ssam 201173139Srwatson (*num_allocated)++; 202171095Ssam 203173139Srwatson if (first_time && mempool->items_current == 204173139Srwatson mempool->items_initial) { 205173139Srwatson break; 206173139Srwatson } 207171095Ssam } 208171095Ssam 209171095Ssam /* increment actual number of allocated memblocks */ 210171095Ssam mempool->memblocks_allocated += *num_allocated; 211171095Ssam 212171095Ssam return XGE_HAL_OK; 213171095Ssam} 214171095Ssam 215171095Ssam/* 216171095Ssam * xge_hal_mempool_create 217171095Ssam * @memblock_size: 218171095Ssam * @items_initial: 219171095Ssam * @items_max: 220171095Ssam * @item_size: 221171095Ssam * @item_func: 222171095Ssam * 223171095Ssam * This function will create memory pool object. Pool may grow but will 224171095Ssam * never shrink. Pool consists of number of dynamically allocated blocks 225171095Ssam * with size enough to hold %items_initial number of items. Memory is 226171095Ssam * DMA-able but client must map/unmap before interoperating with the device. 227171095Ssam * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}. 228171095Ssam */ 229171095Ssamxge_hal_mempool_t* 230171095Ssam__hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size, 231173139Srwatson int items_priv_size, int items_initial, int items_max, 232173139Srwatson xge_hal_mempool_item_f item_func_alloc, 233173139Srwatson xge_hal_mempool_item_f item_func_free, void *userdata) 234171095Ssam{ 235171095Ssam xge_hal_status_e status; 236171095Ssam int memblocks_to_allocate; 237171095Ssam xge_hal_mempool_t *mempool; 238171095Ssam int allocated; 239171095Ssam 240171095Ssam if (memblock_size < item_size) { 241173139Srwatson xge_debug_mm(XGE_ERR, 242173139Srwatson "memblock_size %d < item_size %d: misconfiguration", 243173139Srwatson memblock_size, item_size); 244173139Srwatson return NULL; 245171095Ssam } 246171095Ssam 247171095Ssam mempool = (xge_hal_mempool_t *) \ 248173139Srwatson xge_os_malloc(pdev, sizeof(xge_hal_mempool_t)); 249171095Ssam if (mempool == NULL) { 250173139Srwatson xge_debug_mm(XGE_ERR, "mempool allocation failure"); 251173139Srwatson return NULL; 252171095Ssam } 253171095Ssam xge_os_memzero(mempool, sizeof(xge_hal_mempool_t)); 254171095Ssam 255173139Srwatson mempool->pdev = pdev; 256173139Srwatson mempool->memblock_size = memblock_size; 257173139Srwatson mempool->items_max = items_max; 258173139Srwatson mempool->items_initial = items_initial; 259173139Srwatson mempool->item_size = item_size; 260173139Srwatson mempool->items_priv_size = items_priv_size; 261173139Srwatson mempool->item_func_alloc = item_func_alloc; 262173139Srwatson mempool->item_func_free = item_func_free; 263173139Srwatson mempool->userdata = userdata; 264171095Ssam 265171095Ssam mempool->memblocks_allocated = 0; 266171095Ssam 267171095Ssam mempool->items_per_memblock = memblock_size / item_size; 268171095Ssam 269171095Ssam mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) / 270173139Srwatson mempool->items_per_memblock; 271171095Ssam 272171095Ssam /* allocate array of memblocks */ 273171095Ssam mempool->memblocks_arr = (void ** ) xge_os_malloc(mempool->pdev, 274173139Srwatson sizeof(void*) * mempool->memblocks_max); 275171095Ssam if (mempool->memblocks_arr == NULL) { 276173139Srwatson xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure"); 277173139Srwatson __hal_mempool_destroy(mempool); 278173139Srwatson return NULL; 279171095Ssam } 280171095Ssam xge_os_memzero(mempool->memblocks_arr, 281173139Srwatson sizeof(void*) * mempool->memblocks_max); 282171095Ssam 283171095Ssam /* allocate array of private parts of items per memblocks */ 284171095Ssam mempool->memblocks_priv_arr = (void **) xge_os_malloc(mempool->pdev, 285173139Srwatson sizeof(void*) * mempool->memblocks_max); 286171095Ssam if (mempool->memblocks_priv_arr == NULL) { 287173139Srwatson xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure"); 288173139Srwatson __hal_mempool_destroy(mempool); 289173139Srwatson return NULL; 290171095Ssam } 291171095Ssam xge_os_memzero(mempool->memblocks_priv_arr, 292173139Srwatson sizeof(void*) * mempool->memblocks_max); 293171095Ssam 294171095Ssam /* allocate array of memblocks DMA objects */ 295171095Ssam mempool->memblocks_dma_arr = 296173139Srwatson (xge_hal_mempool_dma_t *) xge_os_malloc(mempool->pdev, 297173139Srwatson sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 298171095Ssam 299171095Ssam if (mempool->memblocks_dma_arr == NULL) { 300173139Srwatson xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure"); 301173139Srwatson __hal_mempool_destroy(mempool); 302173139Srwatson return NULL; 303171095Ssam } 304171095Ssam xge_os_memzero(mempool->memblocks_dma_arr, 305173139Srwatson sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 306171095Ssam 307171095Ssam /* allocate hash array of items */ 308171095Ssam mempool->items_arr = (void **) xge_os_malloc(mempool->pdev, 309173139Srwatson sizeof(void*) * mempool->items_max); 310171095Ssam if (mempool->items_arr == NULL) { 311173139Srwatson xge_debug_mm(XGE_ERR, "items_arr allocation failure"); 312173139Srwatson __hal_mempool_destroy(mempool); 313173139Srwatson return NULL; 314171095Ssam } 315171095Ssam xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max); 316171095Ssam 317171095Ssam mempool->shadow_items_arr = (void **) xge_os_malloc(mempool->pdev, 318173139Srwatson sizeof(void*) * mempool->items_max); 319171095Ssam if (mempool->shadow_items_arr == NULL) { 320173139Srwatson xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure"); 321173139Srwatson __hal_mempool_destroy(mempool); 322173139Srwatson return NULL; 323171095Ssam } 324171095Ssam xge_os_memzero(mempool->shadow_items_arr, 325173139Srwatson sizeof(void *) * mempool->items_max); 326171095Ssam 327171095Ssam /* calculate initial number of memblocks */ 328171095Ssam memblocks_to_allocate = (mempool->items_initial + 329173139Srwatson mempool->items_per_memblock - 1) / 330173139Srwatson mempool->items_per_memblock; 331171095Ssam 332171095Ssam xge_debug_mm(XGE_TRACE, "allocating %d memblocks, " 333173139Srwatson "%d items per memblock", memblocks_to_allocate, 334173139Srwatson mempool->items_per_memblock); 335171095Ssam 336171095Ssam /* pre-allocate the mempool */ 337171095Ssam status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated); 338171095Ssam xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr, 339173139Srwatson sizeof(void*) * mempool->items_max); 340171095Ssam if (status != XGE_HAL_OK) { 341173139Srwatson xge_debug_mm(XGE_ERR, "mempool_grow failure"); 342173139Srwatson __hal_mempool_destroy(mempool); 343173139Srwatson return NULL; 344171095Ssam } 345171095Ssam 346171095Ssam xge_debug_mm(XGE_TRACE, 347173139Srwatson "total: allocated %dk of DMA-capable memory", 348173139Srwatson mempool->memblock_size * allocated / 1024); 349171095Ssam 350171095Ssam return mempool; 351171095Ssam} 352171095Ssam 353171095Ssam/* 354171095Ssam * xge_hal_mempool_destroy 355171095Ssam */ 356171095Ssamvoid 357171095Ssam__hal_mempool_destroy(xge_hal_mempool_t *mempool) 358171095Ssam{ 359171095Ssam int i, j; 360171095Ssam 361171095Ssam for (i=0; i<mempool->memblocks_allocated; i++) { 362173139Srwatson xge_hal_mempool_dma_t *dma_object; 363171095Ssam 364173139Srwatson xge_assert(mempool->memblocks_arr[i]); 365173139Srwatson xge_assert(mempool->memblocks_dma_arr + i); 366171095Ssam 367173139Srwatson dma_object = mempool->memblocks_dma_arr + i; 368171095Ssam 369173139Srwatson for (j=0; j<mempool->items_per_memblock; j++) { 370173139Srwatson int index = i*mempool->items_per_memblock + j; 371171095Ssam 372173139Srwatson /* to skip last partially filled(if any) memblock */ 373173139Srwatson if (index >= mempool->items_current) { 374173139Srwatson break; 375173139Srwatson } 376171095Ssam 377173139Srwatson /* let caller to do more job on each item */ 378173139Srwatson if (mempool->item_func_free != NULL) { 379171095Ssam 380173139Srwatson mempool->item_func_free(mempool, 381173139Srwatson mempool->memblocks_arr[i], 382173139Srwatson i, dma_object, 383173139Srwatson mempool->shadow_items_arr[index], 384173139Srwatson index, /* unused */ -1, 385173139Srwatson mempool->userdata); 386173139Srwatson } 387173139Srwatson } 388171095Ssam 389173139Srwatson xge_os_dma_unmap(mempool->pdev, 390171095Ssam dma_object->handle, dma_object->addr, 391173139Srwatson mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL); 392171095Ssam 393173139Srwatson xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 394173139Srwatson mempool->items_priv_size * mempool->items_per_memblock); 395171095Ssam 396173139Srwatson xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i], 397173139Srwatson mempool->memblock_size, &dma_object->acc_handle, 398173139Srwatson &dma_object->handle); 399171095Ssam } 400171095Ssam 401171095Ssam if (mempool->items_arr) { 402173139Srwatson xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) * 403173139Srwatson mempool->items_max); 404171095Ssam } 405171095Ssam 406171095Ssam if (mempool->shadow_items_arr) { 407173139Srwatson xge_os_free(mempool->pdev, mempool->shadow_items_arr, 408173139Srwatson sizeof(void*) * mempool->items_max); 409171095Ssam } 410171095Ssam 411171095Ssam if (mempool->memblocks_dma_arr) { 412173139Srwatson xge_os_free(mempool->pdev, mempool->memblocks_dma_arr, 413173139Srwatson sizeof(xge_hal_mempool_dma_t) * 414173139Srwatson mempool->memblocks_max); 415171095Ssam } 416171095Ssam 417171095Ssam if (mempool->memblocks_priv_arr) { 418173139Srwatson xge_os_free(mempool->pdev, mempool->memblocks_priv_arr, 419173139Srwatson sizeof(void*) * mempool->memblocks_max); 420171095Ssam } 421171095Ssam 422171095Ssam if (mempool->memblocks_arr) { 423173139Srwatson xge_os_free(mempool->pdev, mempool->memblocks_arr, 424173139Srwatson sizeof(void*) * mempool->memblocks_max); 425171095Ssam } 426171095Ssam 427171095Ssam xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t)); 428171095Ssam} 429