1/* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34/*#include <linux/errno.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/module.h> 38 #include <linux/dma-mapping.h> 39 #include <linux/vmalloc.h> 40 */ 41#include <linux/log2.h> 42#include <linux/bitops.h> 43#include <linux/mm.h> 44#include <linux/gfp.h> 45#include "mlx4.h" 46 47uint32_t mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) { 48 uint32_t obj; 49 50 /*spin_lock(&bitmap->lock);*/ 51 52 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 53 if (obj >= bitmap->max) { 54 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 55 & bitmap->mask; 56 obj = find_first_zero_bit(bitmap->table, bitmap->max); 57 } 58 59 if (obj < bitmap->max) { 60 set_bit(obj, bitmap->table); 61 bitmap->last = (obj + 1); 62 if (bitmap->last == bitmap->max) 63 bitmap->last = 0; 64 obj |= bitmap->top; 65 } else 66 obj = -1; 67 68 if (obj != -1) 69 --bitmap->avail; 70 71 /*spin_unlock(&bitmap->lock);*/ 72 73 return obj; 74} 75/* 76 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, uint32_t obj, int use_rr) 77 { 78 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 79 } 80 */ 81static unsigned long find_aligned_range(unsigned long *bitmap, uint32_t start, 82 uint32_t nbits, int len, int align, uint32_t skip_mask) { 83 unsigned long end, i; 84 85 again: start = ALIGN(start, align); 86 87 while ((start < nbits) && (test_bit(start, bitmap) || (start & skip_mask))) 88 start += align; 89 90 if (start >= nbits) 91 return -1; 92 93 end = start + len; 94 if (end > nbits) 95 return -1; 96 97 for (i = start + 1; i < end; i++) { 98 if (test_bit(i, bitmap) || ((uint32_t) i & skip_mask)) { 99 start = i + 1; 100 goto again; 101 } 102 } 103 104 return start; 105} 106 107uint32_t mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align, 108 uint32_t skip_mask) { 109 uint32_t obj; 110 111 if (/*likely(*/cnt == 1 && align == 1 && !skip_mask/*)*/) 112 return mlx4_bitmap_alloc(bitmap); 113 114 /*spin_lock(&bitmap->lock);*/ 115 116 obj = find_aligned_range(bitmap->table, bitmap->last, bitmap->max, cnt, 117 align, skip_mask); 118 if (obj >= bitmap->max) { 119 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 120 & bitmap->mask; 121 obj = find_aligned_range(bitmap->table, 0, bitmap->max, cnt, align, 122 skip_mask); 123 } 124 125 if (obj < bitmap->max) { 126 bitmap_set(bitmap->table, obj, cnt); 127 if (obj == bitmap->last) { 128 bitmap->last = (obj + cnt); 129 if (bitmap->last >= bitmap->max) 130 bitmap->last = 0; 131 } 132 obj |= bitmap->top; 133 } else 134 obj = -1; 135 136 if (obj != -1) 137 bitmap->avail -= cnt; 138 139 /*spin_unlock(&bitmap->lock);*/ 140 141 return obj; 142} 143 144uint32_t mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) { 145 return bitmap->avail; 146} 147 148void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, uint32_t obj, int cnt, 149 int use_rr) { 150 obj &= bitmap->max + bitmap->reserved_top - 1; 151 152 /*spin_lock(&bitmap->lock);*/ 153 if (!use_rr) { 154 bitmap->last = min(bitmap->last, obj); 155 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 156 & bitmap->mask; 157 } 158 bitmap_clear(bitmap->table, obj, cnt); 159 bitmap->avail += cnt; 160 /*spin_unlock(&bitmap->lock);*/ 161} 162 163int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, uint32_t num, uint32_t mask, 164 uint32_t reserved_bot, uint32_t reserved_top) { 165 /*sanity check*/ 166 if (num <= (u64) reserved_top + reserved_bot) 167 return -EINVAL; 168 169 /*num must be a power of 2*/ 170 if (num != roundup_pow_of_two(num)) 171 return -EINVAL; 172 173 if (reserved_bot + reserved_top >= num) 174 return -EINVAL; 175 176 bitmap->last = 0; 177 bitmap->top = 0; 178 bitmap->max = num - reserved_top; 179 bitmap->mask = mask; 180 bitmap->reserved_top = reserved_top; 181 bitmap->avail = num - reserved_top - reserved_bot; 182 /*spin_lock_init(&bitmap->lock);*/ 183 bitmap->table = calloc(BITS_TO_LONGS(bitmap->max), sizeof(long)); 184 if (!bitmap->table) 185 return -ENOMEM; 186 187 bitmap_set(bitmap->table, 0, reserved_bot); 188 189 return 0; 190} 191 192void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) { 193 free(bitmap->table); 194} 195 196/* 197 * Handling for queue buffers -- we allocate a bunch of memory and 198 * register it in a memory region at HCA virtual address 0. If the 199 * requested size is > max_direct, we split the allocation into 200 * multiple pages, so we don't require too much contiguous memory. 201 202 */ 203int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 204 struct mlx4_buf *buf) { 205 genpaddr_t t; 206 207 /*if (size <= max_direct) {*/ 208 buf->nbufs = 1; 209 buf->npages = 1; 210 buf->page_shift = get_order(size) + PAGE_SHIFT; 211 buf->direct.buf = dma_alloc(size, &t); 212 if (!buf->direct.buf) 213 return -ENOMEM; 214 215 buf->direct.map = t; 216 217 /*printf("virt: %p; phys: %lx\n", buf->direct.buf, buf->direct.map);*/ 218 /*printf("buf->page_shift: %d\n", buf->page_shift);*/ 219 220 while (t & ((1 << buf->page_shift) - 1)) { 221 --buf->page_shift; 222 buf->npages *= 2; 223 } 224 225 /*printf("buf->page_shift: %d\n", buf->page_shift);*/ 226 227 memset(buf->direct.buf, 0, size); 228 229 /*TODO: implement scattered allocation*/ 230 /*} else { 231 int i; 232 233 buf->direct.buf = NULL; 234 buf->nbufs = (size + BASE_PAGE_SIZE - 1) / BASE_PAGE_SIZE; 235 buf->npages = buf->nbufs; 236 buf->page_shift = PAGE_SHIFT; 237 buf->page_list = calloc(buf->nbufs, sizeof(*buf->page_list)); 238 if (!buf->page_list) 239 return -ENOMEM; 240 241 for (i = 0; i < buf->nbufs; ++i) { 242 buf->page_list[i].buf = dma_alloc(BASE_PAGE_SIZE, &t); 243 if (!buf->page_list[i].buf) 244 goto err_free; 245 246 buf->page_list[i].map = t; 247 248 memset(buf->page_list[i].buf, 0, BASE_PAGE_SIZE); 249 }*/ 250 251 /*if (BITS_PER_LONG == 64) { 252 struct page **pages; 253 pages = malloc(sizeof *pages * buf->nbufs); 254 if (!pages) 255 goto err_free; 256 for (i = 0; i < buf->nbufs; ++i) 257 pages[i] = virt_to_page(buf->page_list[i].buf); 258 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 259 kfree(pages); 260 if (!buf->direct.buf) 261 goto err_free; 262 }*/ 263 264 /*}*/ 265 266 return 0; 267 268 /*err_free:*//*mlx4_buf_free(dev, size, buf);*/ 269 270 return -ENOMEM; 271} 272/* 273 EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 274 275 void mlx4_buf_free(struct mlx4_priv *priv, int size, struct mlx4_buf *buf) 276 { 277 int i; 278 279 if (buf->nbufs == 1) 280 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 281 buf->direct.map); 282 else { 283 if (BITS_PER_LONG == 64 && buf->direct.buf) 284 vunmap(buf->direct.buf); 285 286 for (i = 0; i < buf->nbufs; ++i) 287 if (buf->page_list[i].buf) 288 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 289 buf->page_list[i].buf, 290 buf->page_list[i].map); 291 kfree(buf->page_list); 292 } 293 } 294 EXPORT_SYMBOL_GPL(mlx4_buf_free); 295 */ 296static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(void) { 297 struct mlx4_db_pgdir *pgdir; 298 299 pgdir = calloc(1, sizeof *pgdir); 300 if (!pgdir) 301 return NULL; 302 303 bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 304 pgdir->bits[0] = pgdir->order0; 305 pgdir->bits[1] = pgdir->order1; 306 pgdir->db_page = dma_alloc(BASE_PAGE_SIZE, &pgdir->db_dma); 307 if (!pgdir->db_page) { 308 free(pgdir); 309 return NULL; 310 } 311 312 return pgdir; 313} 314 315static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 316 struct mlx4_db *db, int order) { 317 int o; 318 int i; 319 320 for (o = order; o <= 1; ++o) { 321 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 322 /*printf("ffb: %d\n", i);*/ 323 if (i < MLX4_DB_PER_PAGE >> o) 324 goto found; 325 } 326 327 return -ENOMEM; 328 329 found: clear_bit(i, pgdir->bits[o]); 330 331 i <<= o; 332 333 if (o > order) 334 set_bit(i ^ 1, pgdir->bits[order]); 335 336 db->u.pgdir = pgdir; 337 db->index = i; 338 db->db = pgdir->db_page + db->index; 339 db->dma = pgdir->db_dma + db->index * 4; 340 db->order = order; 341 342 return 0; 343} 344 345int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) { 346 struct mlx4_priv *priv = mlx4_priv(dev); 347 struct mlx4_db_pgdir *pgdir; 348 int ret = 0; 349 350 /*mutex_lock(&priv->pgdir_mutex);*/ 351 352 list_for_each_entry(pgdir, &priv->pgdir_list, list) 353 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 354 goto out; 355 356 pgdir = mlx4_alloc_db_pgdir(); 357 if (!pgdir) { 358 ret = -ENOMEM; 359 goto out; 360 } 361 362 list_add(&pgdir->list, &priv->pgdir_list); 363 364 /*This should never fail -- we just allocated an empty page:*/ 365 assert(!mlx4_alloc_db_from_pgdir(pgdir, db, order)); 366 367 out: /*mutex_unlock(&priv->pgdir_mutex);*/ 368 369 return ret; 370} 371/* 372 EXPORT_SYMBOL_GPL(mlx4_db_alloc); 373 374 void mlx4_db_free(struct mlx4_priv *priv, struct mlx4_db *db) 375 { 376 struct mlx4_priv *priv = mlx4_priv(dev); 377 int o; 378 int i; 379 380 mutex_lock(&priv->pgdir_mutex); 381 382 o = db->order; 383 i = db->index; 384 385 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 386 clear_bit(i ^ 1, db->u.pgdir->order0); 387 ++o; 388 } 389 i >>= o; 390 set_bit(i, db->u.pgdir->bits[o]); 391 392 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 393 dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 394 db->u.pgdir->db_page, db->u.pgdir->db_dma); 395 list_del(&db->u.pgdir->list); 396 kfree(db->u.pgdir); 397 } 398 399 mutex_unlock(&priv->pgdir_mutex); 400 } 401 EXPORT_SYMBOL_GPL(mlx4_db_free); 402 */ 403int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 404 int size, int max_direct) { 405 int err; 406 407 err = mlx4_db_alloc(dev, &wqres->db, 1); 408 if (err) 409 return err; 410 411 *wqres->db.db = 0; 412 413 err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf); 414 if (err) 415 goto err_db; 416 417 err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 418 &wqres->mtt); 419 if (err) 420 goto err_buf; 421 422 err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); 423 if (err) 424 goto err_mtt; 425 426 return 0; 427 428 err_mtt: /*mlx4_mtt_cleanup(dev, &wqres->mtt);*/ 429 err_buf: /*mlx4_buf_free(dev, size, &wqres->buf);*/ 430 err_db: /*mlx4_db_free(dev, &wqres->db);*/ 431 432 return err; 433} 434/* 435 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 436 437 void mlx4_free_hwq_res(struct mlx4_priv *priv, struct mlx4_hwq_resources *wqres, 438 int size) 439 { 440 mlx4_mtt_cleanup(dev, &wqres->mtt); 441 mlx4_buf_free(dev, size, &wqres->buf); 442 mlx4_db_free(dev, &wqres->db); 443 } 444 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res);*/ 445