mlx4_alloc.c revision 329159
1/* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <linux/errno.h> 35#include <linux/slab.h> 36#include <linux/mm.h> 37#include <linux/module.h> 38#include <linux/dma-mapping.h> 39#include <linux/vmalloc.h> 40 41#include "mlx4.h" 42 43u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 44{ 45 u32 obj; 46 47 spin_lock(&bitmap->lock); 48 49 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 50 if (obj >= bitmap->max) { 51 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 52 & bitmap->mask; 53 obj = find_first_zero_bit(bitmap->table, bitmap->max); 54 } 55 56 if (obj < bitmap->max) { 57 set_bit(obj, bitmap->table); 58 bitmap->last = (obj + 1); 59 if (bitmap->last == bitmap->max) 60 bitmap->last = 0; 61 obj |= bitmap->top; 62 } else 63 obj = -1; 64 65 if (obj != -1) 66 --bitmap->avail; 67 68 spin_unlock(&bitmap->lock); 69 70 return obj; 71} 72 73void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 74{ 75 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 76} 77 78static unsigned long find_aligned_range(unsigned long *bitmap, 79 u32 start, u32 nbits, 80 int len, int align, u32 skip_mask) 81{ 82 unsigned long end, i; 83 84again: 85 start = ALIGN(start, align); 86 87 while ((start < nbits) && (test_bit(start, bitmap) || 88 (start & skip_mask))) 89 start += align; 90 91 if (start >= nbits) 92 return -1; 93 94 end = start+len; 95 if (end > nbits) 96 return -1; 97 98 for (i = start + 1; i < end; i++) { 99 if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { 100 start = i + 1; 101 goto again; 102 } 103 } 104 105 return start; 106} 107 108u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, 109 int align, u32 skip_mask) 110{ 111 u32 obj; 112 113 if (likely(cnt == 1 && align == 1 && !skip_mask)) 114 return mlx4_bitmap_alloc(bitmap); 115 116 spin_lock(&bitmap->lock); 117 118 obj = find_aligned_range(bitmap->table, bitmap->last, 119 bitmap->max, cnt, align, skip_mask); 120 if (obj >= bitmap->max) { 121 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 122 & bitmap->mask; 123 obj = find_aligned_range(bitmap->table, 0, bitmap->max, 124 cnt, align, skip_mask); 125 } 126 127 if (obj < bitmap->max) { 128 bitmap_set(bitmap->table, obj, cnt); 129 if (obj == bitmap->last) { 130 bitmap->last = (obj + cnt); 131 if (bitmap->last >= bitmap->max) 132 bitmap->last = 0; 133 } 134 obj |= bitmap->top; 135 } else 136 obj = -1; 137 138 if (obj != -1) 139 bitmap->avail -= cnt; 140 141 spin_unlock(&bitmap->lock); 142 143 return obj; 144} 145 146u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 147{ 148 return bitmap->avail; 149} 150 151static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj) 152{ 153 return obj & (bitmap->max + bitmap->reserved_top - 1); 154} 155 156void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, 157 int use_rr) 158{ 159 obj &= bitmap->max + bitmap->reserved_top - 1; 160 161 spin_lock(&bitmap->lock); 162 if (!use_rr) { 163 bitmap->last = min(bitmap->last, obj); 164 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 165 & bitmap->mask; 166 } 167 bitmap_clear(bitmap->table, obj, cnt); 168 bitmap->avail += cnt; 169 spin_unlock(&bitmap->lock); 170} 171 172int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 173 u32 reserved_bot, u32 reserved_top) 174{ 175 /* num must be a power of 2 */ 176 if (num != roundup_pow_of_two(num)) 177 return -EINVAL; 178 179 bitmap->last = 0; 180 bitmap->top = 0; 181 bitmap->max = num - reserved_top; 182 bitmap->mask = mask; 183 bitmap->reserved_top = reserved_top; 184 bitmap->avail = num - reserved_top - reserved_bot; 185 bitmap->effective_len = bitmap->avail; 186 spin_lock_init(&bitmap->lock); 187 bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * 188 sizeof (long), GFP_KERNEL); 189 if (!bitmap->table) 190 return -ENOMEM; 191 192 bitmap_set(bitmap->table, 0, reserved_bot); 193 194 return 0; 195} 196 197void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 198{ 199 kfree(bitmap->table); 200} 201 202struct mlx4_zone_allocator { 203 struct list_head entries; 204 struct list_head prios; 205 u32 last_uid; 206 u32 mask; 207 /* protect the zone_allocator from concurrent accesses */ 208 spinlock_t lock; 209 enum mlx4_zone_alloc_flags flags; 210}; 211 212struct mlx4_zone_entry { 213 struct list_head list; 214 struct list_head prio_list; 215 u32 uid; 216 struct mlx4_zone_allocator *allocator; 217 struct mlx4_bitmap *bitmap; 218 int use_rr; 219 int priority; 220 int offset; 221 enum mlx4_zone_flags flags; 222}; 223 224struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags) 225{ 226 struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL); 227 228 if (NULL == zones) 229 return NULL; 230 231 INIT_LIST_HEAD(&zones->entries); 232 INIT_LIST_HEAD(&zones->prios); 233 spin_lock_init(&zones->lock); 234 zones->last_uid = 0; 235 zones->mask = 0; 236 zones->flags = flags; 237 238 return zones; 239} 240 241int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, 242 struct mlx4_bitmap *bitmap, 243 u32 flags, 244 int priority, 245 int offset, 246 u32 *puid) 247{ 248 u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1); 249 struct mlx4_zone_entry *it; 250 struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL); 251 252 if (NULL == zone) 253 return -ENOMEM; 254 255 zone->flags = flags; 256 zone->bitmap = bitmap; 257 zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0; 258 zone->priority = priority; 259 zone->offset = offset; 260 261 spin_lock(&zone_alloc->lock); 262 263 zone->uid = zone_alloc->last_uid++; 264 zone->allocator = zone_alloc; 265 266 if (zone_alloc->mask < mask) 267 zone_alloc->mask = mask; 268 269 list_for_each_entry(it, &zone_alloc->prios, prio_list) 270 if (it->priority >= priority) 271 break; 272 273 if (&it->prio_list == &zone_alloc->prios || it->priority > priority) 274 list_add_tail(&zone->prio_list, &it->prio_list); 275 list_add_tail(&zone->list, &it->list); 276 277 spin_unlock(&zone_alloc->lock); 278 279 *puid = zone->uid; 280 281 return 0; 282} 283 284/* Should be called under a lock */ 285static int __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry) 286{ 287 struct mlx4_zone_allocator *zone_alloc = entry->allocator; 288 289 if (!list_empty(&entry->prio_list)) { 290 /* Check if we need to add an alternative node to the prio list */ 291 if (!list_is_last(&entry->list, &zone_alloc->entries)) { 292 struct mlx4_zone_entry *next = list_first_entry(&entry->list, 293 typeof(*next), 294 list); 295 296 if (next->priority == entry->priority) 297 list_add_tail(&next->prio_list, &entry->prio_list); 298 } 299 300 list_del(&entry->prio_list); 301 } 302 303 list_del(&entry->list); 304 305 if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) { 306 u32 mask = 0; 307 struct mlx4_zone_entry *it; 308 309 list_for_each_entry(it, &zone_alloc->prios, prio_list) { 310 u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1); 311 312 if (mask < cur_mask) 313 mask = cur_mask; 314 } 315 zone_alloc->mask = mask; 316 } 317 318 return 0; 319} 320 321void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc) 322{ 323 struct mlx4_zone_entry *zone, *tmp; 324 325 spin_lock(&zone_alloc->lock); 326 327 list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) { 328 list_del(&zone->list); 329 list_del(&zone->prio_list); 330 kfree(zone); 331 } 332 333 spin_unlock(&zone_alloc->lock); 334 kfree(zone_alloc); 335} 336 337/* Should be called under a lock */ 338static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count, 339 int align, u32 skip_mask, u32 *puid) 340{ 341 u32 uid = 0; 342 u32 res; 343 struct mlx4_zone_allocator *zone_alloc = zone->allocator; 344 struct mlx4_zone_entry *curr_node; 345 346 res = mlx4_bitmap_alloc_range(zone->bitmap, count, 347 align, skip_mask); 348 349 if (res != (u32)-1) { 350 res += zone->offset; 351 uid = zone->uid; 352 goto out; 353 } 354 355 list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) { 356 if (unlikely(curr_node->priority == zone->priority)) 357 break; 358 } 359 360 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) { 361 struct mlx4_zone_entry *it = curr_node; 362 363 list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) { 364 res = mlx4_bitmap_alloc_range(it->bitmap, count, 365 align, skip_mask); 366 if (res != (u32)-1) { 367 res += it->offset; 368 uid = it->uid; 369 goto out; 370 } 371 } 372 } 373 374 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) { 375 struct mlx4_zone_entry *it = curr_node; 376 377 list_for_each_entry_from(it, &zone_alloc->entries, list) { 378 if (unlikely(it == zone)) 379 continue; 380 381 if (unlikely(it->priority != curr_node->priority)) 382 break; 383 384 res = mlx4_bitmap_alloc_range(it->bitmap, count, 385 align, skip_mask); 386 if (res != (u32)-1) { 387 res += it->offset; 388 uid = it->uid; 389 goto out; 390 } 391 } 392 } 393 394 if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) { 395 if (list_is_last(&curr_node->prio_list, &zone_alloc->prios)) 396 goto out; 397 398 curr_node = list_first_entry(&curr_node->prio_list, 399 typeof(*curr_node), 400 prio_list); 401 402 list_for_each_entry_from(curr_node, &zone_alloc->entries, list) { 403 res = mlx4_bitmap_alloc_range(curr_node->bitmap, count, 404 align, skip_mask); 405 if (res != (u32)-1) { 406 res += curr_node->offset; 407 uid = curr_node->uid; 408 goto out; 409 } 410 } 411 } 412 413out: 414 if (NULL != puid && res != (u32)-1) 415 *puid = uid; 416 return res; 417} 418 419/* Should be called under a lock */ 420static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj, 421 u32 count) 422{ 423 mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr); 424} 425 426/* Should be called under a lock */ 427static struct mlx4_zone_entry *__mlx4_find_zone_by_uid( 428 struct mlx4_zone_allocator *zones, u32 uid) 429{ 430 struct mlx4_zone_entry *zone; 431 432 list_for_each_entry(zone, &zones->entries, list) { 433 if (zone->uid == uid) 434 return zone; 435 } 436 437 return NULL; 438} 439 440struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid) 441{ 442 struct mlx4_zone_entry *zone; 443 struct mlx4_bitmap *bitmap; 444 445 spin_lock(&zones->lock); 446 447 zone = __mlx4_find_zone_by_uid(zones, uid); 448 449 bitmap = zone == NULL ? NULL : zone->bitmap; 450 451 spin_unlock(&zones->lock); 452 453 return bitmap; 454} 455 456int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid) 457{ 458 struct mlx4_zone_entry *zone; 459 int res; 460 461 spin_lock(&zones->lock); 462 463 zone = __mlx4_find_zone_by_uid(zones, uid); 464 465 if (NULL == zone) { 466 res = -1; 467 goto out; 468 } 469 470 res = __mlx4_zone_remove_one_entry(zone); 471 472out: 473 spin_unlock(&zones->lock); 474 kfree(zone); 475 476 return res; 477} 478 479/* Should be called under a lock */ 480static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique( 481 struct mlx4_zone_allocator *zones, u32 obj) 482{ 483 struct mlx4_zone_entry *zone, *zone_candidate = NULL; 484 u32 dist = (u32)-1; 485 486 /* Search for the smallest zone that this obj could be 487 * allocated from. This is done in order to handle 488 * situations when small bitmaps are allocated from bigger 489 * bitmaps (and the allocated space is marked as reserved in 490 * the bigger bitmap. 491 */ 492 list_for_each_entry(zone, &zones->entries, list) { 493 if (obj >= zone->offset) { 494 u32 mobj = (obj - zone->offset) & zones->mask; 495 496 if (mobj < zone->bitmap->max) { 497 u32 curr_dist = zone->bitmap->effective_len; 498 499 if (curr_dist < dist) { 500 dist = curr_dist; 501 zone_candidate = zone; 502 } 503 } 504 } 505 } 506 507 return zone_candidate; 508} 509 510u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, 511 int align, u32 skip_mask, u32 *puid) 512{ 513 struct mlx4_zone_entry *zone; 514 int res = -1; 515 516 spin_lock(&zones->lock); 517 518 zone = __mlx4_find_zone_by_uid(zones, uid); 519 520 if (NULL == zone) 521 goto out; 522 523 res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid); 524 525out: 526 spin_unlock(&zones->lock); 527 528 return res; 529} 530 531u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count) 532{ 533 struct mlx4_zone_entry *zone; 534 int res = 0; 535 536 spin_lock(&zones->lock); 537 538 zone = __mlx4_find_zone_by_uid(zones, uid); 539 540 if (NULL == zone) { 541 res = -1; 542 goto out; 543 } 544 545 __mlx4_free_from_zone(zone, obj, count); 546 547out: 548 spin_unlock(&zones->lock); 549 550 return res; 551} 552 553u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count) 554{ 555 struct mlx4_zone_entry *zone; 556 int res; 557 558 if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP)) 559 return -EFAULT; 560 561 spin_lock(&zones->lock); 562 563 zone = __mlx4_find_zone_by_uid_unique(zones, obj); 564 565 if (NULL == zone) { 566 res = -1; 567 goto out; 568 } 569 570 __mlx4_free_from_zone(zone, obj, count); 571 res = 0; 572 573out: 574 spin_unlock(&zones->lock); 575 576 return res; 577} 578/* 579 * Handling for queue buffers -- we allocate a bunch of memory and 580 * register it in a memory region at HCA virtual address 0. If the 581 * requested size is > max_direct, we split the allocation into 582 * multiple pages, so we don't require too much contiguous memory. 583 */ 584 585int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 586 struct mlx4_buf *buf, gfp_t gfp) 587{ 588 dma_addr_t t; 589 590 if (size <= max_direct) { 591 buf->nbufs = 1; 592 buf->npages = 1; 593 buf->page_shift = get_order(size) + PAGE_SHIFT; 594 buf->direct.buf = dma_alloc_coherent(&dev->persist->pdev->dev, 595 size, &t, gfp); 596 if (!buf->direct.buf) 597 return -ENOMEM; 598 599 buf->direct.map = t; 600 601 while (t & ((1 << buf->page_shift) - 1)) { 602 --buf->page_shift; 603 buf->npages *= 2; 604 } 605 606 memset(buf->direct.buf, 0, size); 607 } else { 608 int i; 609 610 buf->direct.buf = NULL; 611 buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 612 buf->npages = buf->nbufs; 613 buf->page_shift = PAGE_SHIFT; 614 buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 615 gfp); 616 if (!buf->page_list) 617 return -ENOMEM; 618 619 for (i = 0; i < buf->nbufs; ++i) { 620 buf->page_list[i].buf = 621 dma_alloc_coherent(&dev->persist->pdev->dev, 622 PAGE_SIZE, 623 &t, gfp); 624 if (!buf->page_list[i].buf) 625 goto err_free; 626 627 buf->page_list[i].map = t; 628 629 memset(buf->page_list[i].buf, 0, PAGE_SIZE); 630 } 631 632 if (BITS_PER_LONG == 64) { 633 struct page **pages; 634 pages = kmalloc(sizeof *pages * buf->nbufs, gfp); 635 if (!pages) 636 goto err_free; 637 for (i = 0; i < buf->nbufs; ++i) 638 pages[i] = virt_to_page(buf->page_list[i].buf); 639 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 640 kfree(pages); 641 if (!buf->direct.buf) 642 goto err_free; 643 } 644 } 645 646 return 0; 647 648err_free: 649 mlx4_buf_free(dev, size, buf); 650 651 return -ENOMEM; 652} 653EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 654 655void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 656{ 657 int i; 658 659 if (buf->nbufs == 1) 660 dma_free_coherent(&dev->persist->pdev->dev, size, 661 buf->direct.buf, 662 buf->direct.map); 663 else { 664 if (BITS_PER_LONG == 64) 665 vunmap(buf->direct.buf); 666 667 for (i = 0; i < buf->nbufs; ++i) 668 if (buf->page_list[i].buf) 669 dma_free_coherent(&dev->persist->pdev->dev, 670 PAGE_SIZE, 671 buf->page_list[i].buf, 672 buf->page_list[i].map); 673 kfree(buf->page_list); 674 } 675} 676EXPORT_SYMBOL_GPL(mlx4_buf_free); 677 678static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device, 679 gfp_t gfp) 680{ 681 struct mlx4_db_pgdir *pgdir; 682 683 pgdir = kzalloc(sizeof *pgdir, gfp); 684 if (!pgdir) 685 return NULL; 686 687 bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 688 pgdir->bits[0] = pgdir->order0; 689 pgdir->bits[1] = pgdir->order1; 690 pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 691 &pgdir->db_dma, gfp); 692 if (!pgdir->db_page) { 693 kfree(pgdir); 694 return NULL; 695 } 696 697 return pgdir; 698} 699 700static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 701 struct mlx4_db *db, int order) 702{ 703 int o; 704 int i; 705 706 for (o = order; o <= 1; ++o) { 707 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 708 if (i < MLX4_DB_PER_PAGE >> o) 709 goto found; 710 } 711 712 return -ENOMEM; 713 714found: 715 clear_bit(i, pgdir->bits[o]); 716 717 i <<= o; 718 719 if (o > order) 720 set_bit(i ^ 1, pgdir->bits[order]); 721 722 db->u.pgdir = pgdir; 723 db->index = i; 724 db->db = pgdir->db_page + db->index; 725 db->dma = pgdir->db_dma + db->index * 4; 726 db->order = order; 727 728 return 0; 729} 730 731int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp) 732{ 733 struct mlx4_priv *priv = mlx4_priv(dev); 734 struct mlx4_db_pgdir *pgdir; 735 int ret = 0; 736 737 mutex_lock(&priv->pgdir_mutex); 738 739 list_for_each_entry(pgdir, &priv->pgdir_list, list) 740 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 741 goto out; 742 743 pgdir = mlx4_alloc_db_pgdir(&dev->persist->pdev->dev, gfp); 744 if (!pgdir) { 745 ret = -ENOMEM; 746 goto out; 747 } 748 749 list_add(&pgdir->list, &priv->pgdir_list); 750 751 /* This should never fail -- we just allocated an empty page: */ 752 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 753 754out: 755 mutex_unlock(&priv->pgdir_mutex); 756 757 return ret; 758} 759EXPORT_SYMBOL_GPL(mlx4_db_alloc); 760 761void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 762{ 763 struct mlx4_priv *priv = mlx4_priv(dev); 764 int o; 765 int i; 766 767 mutex_lock(&priv->pgdir_mutex); 768 769 o = db->order; 770 i = db->index; 771 772 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 773 clear_bit(i ^ 1, db->u.pgdir->order0); 774 ++o; 775 } 776 i >>= o; 777 set_bit(i, db->u.pgdir->bits[o]); 778 779 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 780 dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, 781 db->u.pgdir->db_page, db->u.pgdir->db_dma); 782 list_del(&db->u.pgdir->list); 783 kfree(db->u.pgdir); 784 } 785 786 mutex_unlock(&priv->pgdir_mutex); 787} 788EXPORT_SYMBOL_GPL(mlx4_db_free); 789 790int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 791 int size, int max_direct) 792{ 793 int err; 794 795 err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL); 796 if (err) 797 return err; 798 799 *wqres->db.db = 0; 800 801 err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL); 802 if (err) 803 goto err_db; 804 805 err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 806 &wqres->mtt); 807 if (err) 808 goto err_buf; 809 810 err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL); 811 if (err) 812 goto err_mtt; 813 814 return 0; 815 816err_mtt: 817 mlx4_mtt_cleanup(dev, &wqres->mtt); 818err_buf: 819 mlx4_buf_free(dev, size, &wqres->buf); 820err_db: 821 mlx4_db_free(dev, &wqres->db); 822 823 return err; 824} 825EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 826 827void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 828 int size) 829{ 830 mlx4_mtt_cleanup(dev, &wqres->mtt); 831 mlx4_buf_free(dev, size, &wqres->buf); 832 mlx4_db_free(dev, &wqres->db); 833} 834EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 835