xgehal-mm.c revision 171095
1/*- 2 * Copyright (c) 2002-2007 Neterion, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: head/sys/dev/nxge/xgehal/xgehal-mm.c 171095 2007-06-29 22:47:18Z sam $ 27 */ 28 29/* 30 * FileName : hal-mm.c 31 * 32 * Description: chipset memory pool object implementation 33 * 34 * Created: 10 May 2004 35 */ 36 37#include <dev/nxge/include/xge-os-pal.h> 38#include <dev/nxge/include/xgehal-mm.h> 39#include <dev/nxge/include/xge-debug.h> 40 41/* 42 * __hal_mempool_grow 43 * 44 * Will resize mempool up to %num_allocate value. 45 */ 46xge_hal_status_e 47__hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate, 48 int *num_allocated) 49{ 50 int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0; 51 int n_items = mempool->items_per_memblock; 52 53 *num_allocated = 0; 54 55 if ((mempool->memblocks_allocated + num_allocate) > 56 mempool->memblocks_max) { 57 xge_debug_mm(XGE_ERR, "%s", 58 "__hal_mempool_grow: can grow anymore"); 59 return XGE_HAL_ERR_OUT_OF_MEMORY; 60 } 61 62 for (i = mempool->memblocks_allocated; 63 i < mempool->memblocks_allocated + num_allocate; i++) { 64 int j; 65 int is_last = 66 ((mempool->memblocks_allocated+num_allocate-1) == i); 67 xge_hal_mempool_dma_t *dma_object = 68 mempool->memblocks_dma_arr + i; 69 void *the_memblock; 70 int dma_flags; 71 72 dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED; 73#ifdef XGE_HAL_DMA_DTR_CONSISTENT 74 dma_flags |= XGE_OS_DMA_CONSISTENT; 75#else 76 dma_flags |= XGE_OS_DMA_STREAMING; 77#endif 78 79 /* allocate DMA-capable memblock */ 80 mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev, 81 mempool->memblock_size, 82 dma_flags, 83 &dma_object->handle, 84 &dma_object->acc_handle); 85 if (mempool->memblocks_arr[i] == NULL) { 86 xge_debug_mm(XGE_ERR, 87 "memblock[%d]: out of DMA memory", i); 88 return XGE_HAL_ERR_OUT_OF_MEMORY; 89 } 90 xge_os_memzero(mempool->memblocks_arr[i], 91 mempool->memblock_size); 92 the_memblock = mempool->memblocks_arr[i]; 93 94 /* allocate memblock's private part. Each DMA memblock 95 * has a space allocated for item's private usage upon 96 * mempool's user request. Each time mempool grows, it will 97 * allocate new memblock and its private part at once. 98 * This helps to minimize memory usage a lot. */ 99 mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev, 100 mempool->items_priv_size * n_items); 101 if (mempool->memblocks_priv_arr[i] == NULL) { 102 xge_os_dma_free(mempool->pdev, 103 the_memblock, 104 mempool->memblock_size, 105 &dma_object->acc_handle, 106 &dma_object->handle); 107 xge_debug_mm(XGE_ERR, 108 "memblock_priv[%d]: out of virtual memory, " 109 "requested %d(%d:%d) bytes", i, 110 mempool->items_priv_size * n_items, 111 mempool->items_priv_size, n_items); 112 return XGE_HAL_ERR_OUT_OF_MEMORY; 113 } 114 xge_os_memzero(mempool->memblocks_priv_arr[i], 115 mempool->items_priv_size * n_items); 116 117 /* map memblock to physical memory */ 118 dma_object->addr = xge_os_dma_map(mempool->pdev, 119 dma_object->handle, 120 the_memblock, 121 mempool->memblock_size, 122 XGE_OS_DMA_DIR_BIDIRECTIONAL, 123#ifdef XGE_HAL_DMA_DTR_CONSISTENT 124 XGE_OS_DMA_CONSISTENT 125#else 126 XGE_OS_DMA_STREAMING 127#endif 128 ); 129 if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) { 130 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 131 mempool->items_priv_size * 132 n_items); 133 xge_os_dma_free(mempool->pdev, 134 the_memblock, 135 mempool->memblock_size, 136 &dma_object->acc_handle, 137 &dma_object->handle); 138 return XGE_HAL_ERR_OUT_OF_MAPPING; 139 } 140 141 /* fill the items hash array */ 142 for (j=0; j<n_items; j++) { 143 int index = i*n_items + j; 144 145 if (first_time && index >= mempool->items_initial) { 146 break; 147 } 148 149 mempool->items_arr[index] = 150 ((char *)the_memblock + j*mempool->item_size); 151 152 /* let caller to do more job on each item */ 153 if (mempool->item_func_alloc != NULL) { 154 xge_hal_status_e status; 155 156 if ((status = mempool->item_func_alloc( 157 mempool, 158 the_memblock, 159 i, 160 dma_object, 161 mempool->items_arr[index], 162 index, 163 is_last, 164 mempool->userdata)) != XGE_HAL_OK) { 165 166 if (mempool->item_func_free != NULL) { 167 int k; 168 169 for (k=0; k<j; k++) { 170 171 index =i*n_items + k; 172 173 (void)mempool->item_func_free( 174 mempool, the_memblock, 175 i, dma_object, 176 mempool->items_arr[index], 177 index, is_last, 178 mempool->userdata); 179 } 180 } 181 182 xge_os_free(mempool->pdev, 183 mempool->memblocks_priv_arr[i], 184 mempool->items_priv_size * 185 n_items); 186 xge_os_dma_unmap(mempool->pdev, 187 dma_object->handle, 188 dma_object->addr, 189 mempool->memblock_size, 190 XGE_OS_DMA_DIR_BIDIRECTIONAL); 191 xge_os_dma_free(mempool->pdev, 192 the_memblock, 193 mempool->memblock_size, 194 &dma_object->acc_handle, 195 &dma_object->handle); 196 return status; 197 } 198 } 199 200 mempool->items_current = index + 1; 201 } 202 203 xge_debug_mm(XGE_TRACE, 204 "memblock%d: allocated %dk, vaddr 0x"XGE_OS_LLXFMT", " 205 "dma_addr 0x"XGE_OS_LLXFMT, i, mempool->memblock_size / 1024, 206 (unsigned long long)(ulong_t)mempool->memblocks_arr[i], 207 (unsigned long long)dma_object->addr); 208 209 (*num_allocated)++; 210 211 if (first_time && mempool->items_current == 212 mempool->items_initial) { 213 break; 214 } 215 } 216 217 /* increment actual number of allocated memblocks */ 218 mempool->memblocks_allocated += *num_allocated; 219 220 return XGE_HAL_OK; 221} 222 223/* 224 * xge_hal_mempool_create 225 * @memblock_size: 226 * @items_initial: 227 * @items_max: 228 * @item_size: 229 * @item_func: 230 * 231 * This function will create memory pool object. Pool may grow but will 232 * never shrink. Pool consists of number of dynamically allocated blocks 233 * with size enough to hold %items_initial number of items. Memory is 234 * DMA-able but client must map/unmap before interoperating with the device. 235 * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}. 236 */ 237xge_hal_mempool_t* 238__hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size, 239 int items_priv_size, int items_initial, int items_max, 240 xge_hal_mempool_item_f item_func_alloc, 241 xge_hal_mempool_item_f item_func_free, void *userdata) 242{ 243 xge_hal_status_e status; 244 int memblocks_to_allocate; 245 xge_hal_mempool_t *mempool; 246 int allocated; 247 248 if (memblock_size < item_size) { 249 xge_debug_mm(XGE_ERR, 250 "memblock_size %d < item_size %d: misconfiguration", 251 memblock_size, item_size); 252 return NULL; 253 } 254 255 mempool = (xge_hal_mempool_t *) \ 256 xge_os_malloc(pdev, sizeof(xge_hal_mempool_t)); 257 if (mempool == NULL) { 258 xge_debug_mm(XGE_ERR, "mempool allocation failure"); 259 return NULL; 260 } 261 xge_os_memzero(mempool, sizeof(xge_hal_mempool_t)); 262 263 mempool->pdev = pdev; 264 mempool->memblock_size = memblock_size; 265 mempool->items_max = items_max; 266 mempool->items_initial = items_initial; 267 mempool->item_size = item_size; 268 mempool->items_priv_size = items_priv_size; 269 mempool->item_func_alloc = item_func_alloc; 270 mempool->item_func_free = item_func_free; 271 mempool->userdata = userdata; 272 273 mempool->memblocks_allocated = 0; 274 275 mempool->items_per_memblock = memblock_size / item_size; 276 277 mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) / 278 mempool->items_per_memblock; 279 280 /* allocate array of memblocks */ 281 mempool->memblocks_arr = (void ** ) xge_os_malloc(mempool->pdev, 282 sizeof(void*) * mempool->memblocks_max); 283 if (mempool->memblocks_arr == NULL) { 284 xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure"); 285 __hal_mempool_destroy(mempool); 286 return NULL; 287 } 288 xge_os_memzero(mempool->memblocks_arr, 289 sizeof(void*) * mempool->memblocks_max); 290 291 /* allocate array of private parts of items per memblocks */ 292 mempool->memblocks_priv_arr = (void **) xge_os_malloc(mempool->pdev, 293 sizeof(void*) * mempool->memblocks_max); 294 if (mempool->memblocks_priv_arr == NULL) { 295 xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure"); 296 __hal_mempool_destroy(mempool); 297 return NULL; 298 } 299 xge_os_memzero(mempool->memblocks_priv_arr, 300 sizeof(void*) * mempool->memblocks_max); 301 302 /* allocate array of memblocks DMA objects */ 303 mempool->memblocks_dma_arr = 304 (xge_hal_mempool_dma_t *) xge_os_malloc(mempool->pdev, 305 sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 306 307 if (mempool->memblocks_dma_arr == NULL) { 308 xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure"); 309 __hal_mempool_destroy(mempool); 310 return NULL; 311 } 312 xge_os_memzero(mempool->memblocks_dma_arr, 313 sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 314 315 /* allocate hash array of items */ 316 mempool->items_arr = (void **) xge_os_malloc(mempool->pdev, 317 sizeof(void*) * mempool->items_max); 318 if (mempool->items_arr == NULL) { 319 xge_debug_mm(XGE_ERR, "items_arr allocation failure"); 320 __hal_mempool_destroy(mempool); 321 return NULL; 322 } 323 xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max); 324 325 mempool->shadow_items_arr = (void **) xge_os_malloc(mempool->pdev, 326 sizeof(void*) * mempool->items_max); 327 if (mempool->shadow_items_arr == NULL) { 328 xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure"); 329 __hal_mempool_destroy(mempool); 330 return NULL; 331 } 332 xge_os_memzero(mempool->shadow_items_arr, 333 sizeof(void *) * mempool->items_max); 334 335 /* calculate initial number of memblocks */ 336 memblocks_to_allocate = (mempool->items_initial + 337 mempool->items_per_memblock - 1) / 338 mempool->items_per_memblock; 339 340 xge_debug_mm(XGE_TRACE, "allocating %d memblocks, " 341 "%d items per memblock", memblocks_to_allocate, 342 mempool->items_per_memblock); 343 344 /* pre-allocate the mempool */ 345 status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated); 346 xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr, 347 sizeof(void*) * mempool->items_max); 348 if (status != XGE_HAL_OK) { 349 xge_debug_mm(XGE_ERR, "mempool_grow failure"); 350 __hal_mempool_destroy(mempool); 351 return NULL; 352 } 353 354 xge_debug_mm(XGE_TRACE, 355 "total: allocated %dk of DMA-capable memory", 356 mempool->memblock_size * allocated / 1024); 357 358 return mempool; 359} 360 361/* 362 * xge_hal_mempool_destroy 363 */ 364void 365__hal_mempool_destroy(xge_hal_mempool_t *mempool) 366{ 367 int i, j; 368 369 for (i=0; i<mempool->memblocks_allocated; i++) { 370 xge_hal_mempool_dma_t *dma_object; 371 372 xge_assert(mempool->memblocks_arr[i]); 373 xge_assert(mempool->memblocks_dma_arr + i); 374 375 dma_object = mempool->memblocks_dma_arr + i; 376 377 for (j=0; j<mempool->items_per_memblock; j++) { 378 int index = i*mempool->items_per_memblock + j; 379 380 /* to skip last partially filled(if any) memblock */ 381 if (index >= mempool->items_current) { 382 break; 383 } 384 385 /* let caller to do more job on each item */ 386 if (mempool->item_func_free != NULL) { 387 388 mempool->item_func_free(mempool, 389 mempool->memblocks_arr[i], 390 i, dma_object, 391 mempool->shadow_items_arr[index], 392 index, /* unused */ -1, 393 mempool->userdata); 394 } 395 } 396 397 xge_os_dma_unmap(mempool->pdev, 398 dma_object->handle, dma_object->addr, 399 mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL); 400 401 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 402 mempool->items_priv_size * mempool->items_per_memblock); 403 404 xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i], 405 mempool->memblock_size, &dma_object->acc_handle, 406 &dma_object->handle); 407 } 408 409 if (mempool->items_arr) { 410 xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) * 411 mempool->items_max); 412 } 413 414 if (mempool->shadow_items_arr) { 415 xge_os_free(mempool->pdev, mempool->shadow_items_arr, 416 sizeof(void*) * mempool->items_max); 417 } 418 419 if (mempool->memblocks_dma_arr) { 420 xge_os_free(mempool->pdev, mempool->memblocks_dma_arr, 421 sizeof(xge_hal_mempool_dma_t) * 422 mempool->memblocks_max); 423 } 424 425 if (mempool->memblocks_priv_arr) { 426 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr, 427 sizeof(void*) * mempool->memblocks_max); 428 } 429 430 if (mempool->memblocks_arr) { 431 xge_os_free(mempool->pdev, mempool->memblocks_arr, 432 sizeof(void*) * mempool->memblocks_max); 433 } 434 435 xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t)); 436} 437