1/*- 2 * Copyright(c) 2002-2011 Exar Corp. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification are permitted provided the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Exar Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31/*$FreeBSD$*/ 32 33#include <dev/vxge/vxgehal/vxgehal.h> 34 35/* 36 * __hal_mempool_grow 37 * 38 * Will resize mempool up to %num_allocate value. 39 */ 40static vxge_hal_status_e 41__hal_mempool_grow( 42 vxge_hal_mempool_t *mempool, 43 u32 num_allocate, 44 u32 *num_allocated) 45{ 46 u32 i, j, k, item_index, is_last; 47 u32 first_time = mempool->memblocks_allocated == 0 ? 1 : 0; 48 u32 n_items = mempool->items_per_memblock; 49 u32 start_block_idx = mempool->memblocks_allocated; 50 u32 end_block_idx = mempool->memblocks_allocated + num_allocate; 51 __hal_device_t *hldev; 52 53 vxge_assert(mempool != NULL); 54 55 hldev = (__hal_device_t *) mempool->devh; 56 57 vxge_hal_trace_log_mm("==> %s:%s:%d", 58 __FILE__, __func__, __LINE__); 59 60 vxge_hal_trace_log_mm( 61 "mempool = 0x"VXGE_OS_STXFMT", num_allocate = %d, " 62 "num_allocated = 0x"VXGE_OS_STXFMT, (ptr_t) mempool, 63 num_allocate, (ptr_t) num_allocated); 64 65 *num_allocated = 0; 66 67 if (end_block_idx > mempool->memblocks_max) { 68 vxge_hal_err_log_mm("%s", 69 "__hal_mempool_grow: can grow anymore"); 70 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 71 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 72 return (VXGE_HAL_ERR_OUT_OF_MEMORY); 73 } 74 75 for (i = start_block_idx; i < end_block_idx; i++) { 76 77 void *the_memblock; 78 vxge_hal_mempool_dma_t *dma_object; 79 80 is_last = ((end_block_idx - 1) == i); 81 dma_object = mempool->memblocks_dma_arr + i; 82 83 /* 84 * allocate memblock's private part. Each DMA memblock 85 * has a space allocated for item's private usage upon 86 * mempool's user request. Each time mempool grows, it will 87 * allocate new memblock and its private part at once. 88 * This helps to minimize memory usage a lot. 89 */ 90 mempool->memblocks_priv_arr[i] = vxge_os_malloc( 91 ((__hal_device_t *) mempool->devh)->header.pdev, 92 mempool->items_priv_size * n_items); 93 if (mempool->memblocks_priv_arr[i] == NULL) { 94 95 vxge_hal_err_log_mm("memblock_priv[%d]: \ 96 out of virtual memory, " 97 "requested %d(%d:%d) bytes", i, 98 mempool->items_priv_size * n_items, 99 mempool->items_priv_size, n_items); 100 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 101 __FILE__, __func__, __LINE__, 102 VXGE_HAL_ERR_OUT_OF_MEMORY); 103 return (VXGE_HAL_ERR_OUT_OF_MEMORY); 104 105 } 106 107 vxge_os_memzero(mempool->memblocks_priv_arr[i], 108 mempool->items_priv_size * n_items); 109 110 /* allocate DMA-capable memblock */ 111 mempool->memblocks_arr[i] = 112 __hal_blockpool_malloc(mempool->devh, 113 mempool->memblock_size, 114 &dma_object->addr, 115 &dma_object->handle, 116 &dma_object->acc_handle); 117 if (mempool->memblocks_arr[i] == NULL) { 118 vxge_os_free( 119 ((__hal_device_t *) mempool->devh)->header.pdev, 120 mempool->memblocks_priv_arr[i], 121 mempool->items_priv_size * n_items); 122 vxge_hal_err_log_mm("memblock[%d]: \ 123 out of DMA memory", i); 124 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 125 __FILE__, __func__, __LINE__, 126 VXGE_HAL_ERR_OUT_OF_MEMORY); 127 return (VXGE_HAL_ERR_OUT_OF_MEMORY); 128 } 129 130 (*num_allocated)++; 131 mempool->memblocks_allocated++; 132 133 vxge_os_memzero(mempool->memblocks_arr[i], 134 mempool->memblock_size); 135 136 the_memblock = mempool->memblocks_arr[i]; 137 138 /* fill the items hash array */ 139 for (j = 0; j < n_items; j++) { 140 item_index = i * n_items + j; 141 142 if (first_time && (item_index >= mempool->items_initial)) 143 break; 144 145 mempool->items_arr[item_index] = 146 ((char *) the_memblock + j *mempool->item_size); 147 148 /* let caller to do more job on each item */ 149 if (mempool->item_func_alloc != NULL) { 150 vxge_hal_status_e status; 151 152 if ((status = mempool->item_func_alloc( 153 mempool, 154 the_memblock, 155 i, 156 dma_object, 157 mempool->items_arr[item_index], 158 item_index, 159 is_last, 160 mempool->userdata)) != VXGE_HAL_OK) { 161 162 if (mempool->item_func_free != NULL) { 163 164 for (k = 0; k < j; k++) { 165 166 item_index = i * n_items + k; 167 168 (void) mempool->item_func_free( 169 mempool, 170 the_memblock, 171 i, dma_object, 172 mempool->items_arr[item_index], 173 item_index, is_last, 174 mempool->userdata); 175 } 176 } 177 178 vxge_os_free(((__hal_device_t *) 179 mempool->devh)->header.pdev, 180 mempool->memblocks_priv_arr[i], 181 mempool->items_priv_size * 182 n_items); 183 184 __hal_blockpool_free(mempool->devh, 185 the_memblock, 186 mempool->memblock_size, 187 &dma_object->addr, 188 &dma_object->handle, 189 &dma_object->acc_handle); 190 191 (*num_allocated)--; 192 mempool->memblocks_allocated--; 193 return (status); 194 } 195 } 196 197 mempool->items_current = item_index + 1; 198 } 199 200 vxge_hal_info_log_mm( 201 "memblock%d: allocated %dk, vaddr 0x"VXGE_OS_STXFMT", " 202 "dma_addr 0x"VXGE_OS_STXFMT, 203 i, mempool->memblock_size / 1024, 204 (ptr_t) mempool->memblocks_arr[i], dma_object->addr); 205 206 if (first_time && mempool->items_current == 207 mempool->items_initial) { 208 break; 209 } 210 } 211 212 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0", 213 __FILE__, __func__, __LINE__); 214 215 return (VXGE_HAL_OK); 216} 217 218/* 219 * vxge_hal_mempool_create 220 * @memblock_size: 221 * @items_initial: 222 * @items_max: 223 * @item_size: 224 * @item_func: 225 * 226 * This function will create memory pool object. Pool may grow but will 227 * never shrink. Pool consists of number of dynamically allocated blocks 228 * with size enough to hold %items_initial number of items. Memory is 229 * DMA-able but client must map/unmap before interoperating with the device. 230 * See also: vxge_os_dma_map(), vxge_hal_dma_unmap(), vxge_hal_status_e {}. 231 */ 232vxge_hal_mempool_t * 233vxge_hal_mempool_create( 234 vxge_hal_device_h devh, 235 u32 memblock_size, 236 u32 item_size, 237 u32 items_priv_size, 238 u32 items_initial, 239 u32 items_max, 240 vxge_hal_mempool_item_f item_func_alloc, 241 vxge_hal_mempool_item_f item_func_free, 242 void *userdata) 243{ 244 vxge_hal_status_e status; 245 u32 memblocks_to_allocate; 246 vxge_hal_mempool_t *mempool; 247 __hal_device_t *hldev; 248 u32 allocated; 249 250 vxge_assert(devh != NULL); 251 252 hldev = (__hal_device_t *) devh; 253 254 vxge_hal_trace_log_mm("==> %s:%s:%d", 255 __FILE__, __func__, __LINE__); 256 257 vxge_hal_trace_log_mm( 258 "devh = 0x"VXGE_OS_STXFMT", memblock_size = %d, item_size = %d, " 259 "items_priv_size = %d, items_initial = %d, items_max = %d, " 260 "item_func_alloc = 0x"VXGE_OS_STXFMT", " 261 "item_func_free = 0x"VXGE_OS_STXFMT", " 262 "userdata = 0x"VXGE_OS_STXFMT, (ptr_t) devh, 263 memblock_size, item_size, items_priv_size, 264 items_initial, items_max, (ptr_t) item_func_alloc, 265 (ptr_t) item_func_free, (ptr_t) userdata); 266 267 if (memblock_size < item_size) { 268 vxge_hal_err_log_mm( 269 "memblock_size %d < item_size %d: misconfiguration", 270 memblock_size, item_size); 271 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 272 __FILE__, __func__, __LINE__, VXGE_HAL_FAIL); 273 return (NULL); 274 } 275 276 mempool = (vxge_hal_mempool_t *) vxge_os_malloc( 277 ((__hal_device_t *) devh)->header.pdev, sizeof(vxge_hal_mempool_t)); 278 if (mempool == NULL) { 279 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 280 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 281 return (NULL); 282 } 283 vxge_os_memzero(mempool, sizeof(vxge_hal_mempool_t)); 284 285 mempool->devh = devh; 286 mempool->memblock_size = memblock_size; 287 mempool->items_max = items_max; 288 mempool->items_initial = items_initial; 289 mempool->item_size = item_size; 290 mempool->items_priv_size = items_priv_size; 291 mempool->item_func_alloc = item_func_alloc; 292 mempool->item_func_free = item_func_free; 293 mempool->userdata = userdata; 294 295 mempool->memblocks_allocated = 0; 296 297 if (memblock_size != VXGE_OS_HOST_PAGE_SIZE) 298 mempool->dma_flags = VXGE_OS_DMA_CACHELINE_ALIGNED; 299 300#if defined(VXGE_HAL_DMA_CONSISTENT) 301 mempool->dma_flags |= VXGE_OS_DMA_CONSISTENT; 302#else 303 mempool->dma_flags |= VXGE_OS_DMA_STREAMING; 304#endif 305 306 mempool->items_per_memblock = memblock_size / item_size; 307 308 mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) / 309 mempool->items_per_memblock; 310 311 /* allocate array of memblocks */ 312 mempool->memblocks_arr = (void **)vxge_os_malloc( 313 ((__hal_device_t *) mempool->devh)->header.pdev, 314 sizeof(void *) * mempool->memblocks_max); 315 if (mempool->memblocks_arr == NULL) { 316 vxge_hal_mempool_destroy(mempool); 317 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 318 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 319 return (NULL); 320 } 321 vxge_os_memzero(mempool->memblocks_arr, 322 sizeof(void *) * mempool->memblocks_max); 323 324 /* allocate array of private parts of items per memblocks */ 325 mempool->memblocks_priv_arr = (void **)vxge_os_malloc( 326 ((__hal_device_t *) mempool->devh)->header.pdev, 327 sizeof(void *) * mempool->memblocks_max); 328 if (mempool->memblocks_priv_arr == NULL) { 329 vxge_hal_mempool_destroy(mempool); 330 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 331 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 332 return (NULL); 333 } 334 vxge_os_memzero(mempool->memblocks_priv_arr, 335 sizeof(void *) * mempool->memblocks_max); 336 337 /* allocate array of memblocks DMA objects */ 338 mempool->memblocks_dma_arr = 339 (vxge_hal_mempool_dma_t *) vxge_os_malloc( 340 ((__hal_device_t *) mempool->devh)->header.pdev, 341 sizeof(vxge_hal_mempool_dma_t) * mempool->memblocks_max); 342 343 if (mempool->memblocks_dma_arr == NULL) { 344 vxge_hal_mempool_destroy(mempool); 345 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 346 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 347 return (NULL); 348 } 349 vxge_os_memzero(mempool->memblocks_dma_arr, 350 sizeof(vxge_hal_mempool_dma_t) * mempool->memblocks_max); 351 352 /* allocate hash array of items */ 353 mempool->items_arr = (void **)vxge_os_malloc( 354 ((__hal_device_t *) mempool->devh)->header.pdev, 355 sizeof(void *) * mempool->items_max); 356 if (mempool->items_arr == NULL) { 357 vxge_hal_mempool_destroy(mempool); 358 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 359 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 360 return (NULL); 361 } 362 vxge_os_memzero(mempool->items_arr, 363 sizeof(void *) * mempool->items_max); 364 365 mempool->shadow_items_arr = (void **)vxge_os_malloc( 366 ((__hal_device_t *) mempool->devh)->header.pdev, 367 sizeof(void *) * mempool->items_max); 368 if (mempool->shadow_items_arr == NULL) { 369 vxge_hal_mempool_destroy(mempool); 370 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 371 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 372 return (NULL); 373 } 374 vxge_os_memzero(mempool->shadow_items_arr, 375 sizeof(void *) * mempool->items_max); 376 377 /* calculate initial number of memblocks */ 378 memblocks_to_allocate = (mempool->items_initial + 379 mempool->items_per_memblock - 1) / 380 mempool->items_per_memblock; 381 382 vxge_hal_info_log_mm("allocating %d memblocks, " 383 "%d items per memblock", memblocks_to_allocate, 384 mempool->items_per_memblock); 385 386 /* pre-allocate the mempool */ 387 status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated); 388 vxge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr, 389 sizeof(void *) * mempool->items_max); 390 if (status != VXGE_HAL_OK) { 391 vxge_hal_mempool_destroy(mempool); 392 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d", 393 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY); 394 return (NULL); 395 } 396 397 vxge_hal_info_log_mm( 398 "total: allocated %dk of DMA-capable memory", 399 mempool->memblock_size * allocated / 1024); 400 401 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0", 402 __FILE__, __func__, __LINE__); 403 404 return (mempool); 405} 406 407/* 408 * vxge_hal_mempool_destroy 409 */ 410void 411vxge_hal_mempool_destroy( 412 vxge_hal_mempool_t *mempool) 413{ 414 u32 i, j, item_index; 415 __hal_device_t *hldev; 416 417 vxge_assert(mempool != NULL); 418 419 hldev = (__hal_device_t *) mempool->devh; 420 421 vxge_hal_trace_log_mm("==> %s:%s:%d", 422 __FILE__, __func__, __LINE__); 423 424 vxge_hal_trace_log_mm("mempool = 0x"VXGE_OS_STXFMT, 425 (ptr_t) mempool); 426 427 for (i = 0; i < mempool->memblocks_allocated; i++) { 428 vxge_hal_mempool_dma_t *dma_object; 429 430 vxge_assert(mempool->memblocks_arr[i]); 431 vxge_assert(mempool->memblocks_dma_arr + i); 432 433 dma_object = mempool->memblocks_dma_arr + i; 434 435 for (j = 0; j < mempool->items_per_memblock; j++) { 436 item_index = i * mempool->items_per_memblock + j; 437 438 /* to skip last partially filled(if any) memblock */ 439 if (item_index >= mempool->items_current) 440 break; 441 442 /* let caller to do more job on each item */ 443 if (mempool->item_func_free != NULL) { 444 445 mempool->item_func_free(mempool, 446 mempool->memblocks_arr[i], 447 i, dma_object, 448 mempool->shadow_items_arr[item_index], 449 item_index, /* unused */ -1, 450 mempool->userdata); 451 } 452 } 453 454 vxge_os_free(hldev->header.pdev, 455 mempool->memblocks_priv_arr[i], 456 mempool->items_priv_size * mempool->items_per_memblock); 457 458 __hal_blockpool_free(hldev, 459 mempool->memblocks_arr[i], 460 mempool->memblock_size, 461 &dma_object->addr, 462 &dma_object->handle, 463 &dma_object->acc_handle); 464 } 465 466 if (mempool->items_arr) { 467 vxge_os_free(hldev->header.pdev, 468 mempool->items_arr, sizeof(void *) * mempool->items_max); 469 } 470 471 if (mempool->shadow_items_arr) { 472 vxge_os_free(hldev->header.pdev, 473 mempool->shadow_items_arr, 474 sizeof(void *) * mempool->items_max); 475 } 476 477 if (mempool->memblocks_dma_arr) { 478 vxge_os_free(hldev->header.pdev, 479 mempool->memblocks_dma_arr, 480 sizeof(vxge_hal_mempool_dma_t) * 481 mempool->memblocks_max); 482 } 483 484 if (mempool->memblocks_priv_arr) { 485 vxge_os_free(hldev->header.pdev, 486 mempool->memblocks_priv_arr, 487 sizeof(void *) * mempool->memblocks_max); 488 } 489 490 if (mempool->memblocks_arr) { 491 vxge_os_free(hldev->header.pdev, 492 mempool->memblocks_arr, 493 sizeof(void *) * mempool->memblocks_max); 494 } 495 496 vxge_os_free(hldev->header.pdev, 497 mempool, sizeof(vxge_hal_mempool_t)); 498 499 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0", 500 __FILE__, __func__, __LINE__); 501} 502 503/* 504 * vxge_hal_check_alignment - Check buffer alignment and calculate the 505 * "misaligned" portion. 506 * @dma_pointer: DMA address of the buffer. 507 * @size: Buffer size, in bytes. 508 * @alignment: Alignment "granularity" (see below), in bytes. 509 * @copy_size: Maximum number of bytes to "extract" from the buffer 510 * (in order to spost it as a separate scatter-gather entry). See below. 511 * 512 * Check buffer alignment and calculate "misaligned" portion, if exists. 513 * The buffer is considered aligned if its address is multiple of 514 * the specified @alignment. If this is the case, 515 * vxge_hal_check_alignment() returns zero. 516 * Otherwise, vxge_hal_check_alignment() uses the last argument, 517 * @copy_size, 518 * to calculate the size to "extract" from the buffer. The @copy_size 519 * may or may not be equal @alignment. The difference between these two 520 * arguments is that the @alignment is used to make the decision: aligned 521 * or not aligned. While the @copy_size is used to calculate the portion 522 * of the buffer to "extract", i.e. to post as a separate entry in the 523 * transmit descriptor. For example, the combination 524 * @alignment = 8 and @copy_size = 64 will work okay on AMD Opteron boxes. 525 * 526 * Note: @copy_size should be a multiple of @alignment. In many practical 527 * cases @copy_size and @alignment will probably be equal. 528 * 529 * See also: vxge_hal_fifo_txdl_buffer_set_aligned(). 530 */ 531u32 532vxge_hal_check_alignment( 533 dma_addr_t dma_pointer, 534 u32 size, 535 u32 alignment, 536 u32 copy_size) 537{ 538 u32 misaligned_size; 539 540 misaligned_size = (int)(dma_pointer & (alignment - 1)); 541 if (!misaligned_size) { 542 return (0); 543 } 544 545 if (size > copy_size) { 546 misaligned_size = (int)(dma_pointer & (copy_size - 1)); 547 misaligned_size = copy_size - misaligned_size; 548 } else { 549 misaligned_size = size; 550 } 551 552 return (misaligned_size); 553} 554