1/* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <linux/errno.h> 35#include <linux/slab.h> 36#include <linux/mm.h> 37#include <linux/module.h> 38#include <linux/dma-mapping.h> 39#include <linux/vmalloc.h> 40 41#include "mlx4.h" 42 43u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 44{ 45 u32 obj; 46 47 spin_lock(&bitmap->lock); 48 49 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 50 if (obj >= bitmap->max) { 51 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 52 & bitmap->mask; 53 obj = find_first_zero_bit(bitmap->table, bitmap->max); 54 } 55 56 if (obj < bitmap->max) { 57 set_bit(obj, bitmap->table); 58 bitmap->last = (obj + 1); 59 if (bitmap->last == bitmap->max) 60 bitmap->last = 0; 61 obj |= bitmap->top; 62 } else 63 obj = -1; 64 65 if (obj != -1) 66 --bitmap->avail; 67 68 spin_unlock(&bitmap->lock); 69 70 return obj; 71} 72 73void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 74{ 75 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 76} 77 78static unsigned long find_aligned_range(unsigned long *bitmap, 79 u32 start, u32 nbits, 80 int len, int align, u32 skip_mask) 81{ 82 unsigned long end, i; 83 84again: 85 start = ALIGN(start, align); 86 87 while ((start < nbits) && (test_bit(start, bitmap) || 88 (start & skip_mask))) 89 start += align; 90 91 if (start >= nbits) 92 return -1; 93 94 end = start+len; 95 if (end > nbits) 96 return -1; 97 98 for (i = start + 1; i < end; i++) { 99 if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { 100 start = i + 1; 101 goto again; 102 } 103 } 104 105 return start; 106} 107 108u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, 109 int align, u32 skip_mask) 110{ 111 u32 obj; 112 113 if (likely(cnt == 1 && align == 1 && !skip_mask)) 114 return mlx4_bitmap_alloc(bitmap); 115 116 spin_lock(&bitmap->lock); 117 118 obj = find_aligned_range(bitmap->table, bitmap->last, 119 bitmap->max, cnt, align, skip_mask); 120 if (obj >= bitmap->max) { 121 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 122 & bitmap->mask; 123 obj = find_aligned_range(bitmap->table, 0, bitmap->max, 124 cnt, align, skip_mask); 125 } 126 127 if (obj < bitmap->max) { 128 bitmap_set(bitmap->table, obj, cnt); 129 if (obj == bitmap->last) { 130 bitmap->last = (obj + cnt); 131 if (bitmap->last >= bitmap->max) 132 bitmap->last = 0; 133 } 134 obj |= bitmap->top; 135 } else 136 obj = -1; 137 138 if (obj != -1) 139 bitmap->avail -= cnt; 140 141 spin_unlock(&bitmap->lock); 142 143 return obj; 144} 145 146u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 147{ 148 return bitmap->avail; 149} 150 151void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, 152 int use_rr) 153{ 154 obj &= bitmap->max + bitmap->reserved_top - 1; 155 156 spin_lock(&bitmap->lock); 157 if (!use_rr) { 158 bitmap->last = min(bitmap->last, obj); 159 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 160 & bitmap->mask; 161 } 162 bitmap_clear(bitmap->table, obj, cnt); 163 bitmap->avail += cnt; 164 spin_unlock(&bitmap->lock); 165} 166 167int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 168 u32 reserved_bot, u32 reserved_top) 169{ 170 /* sanity check */ 171 if (num <= (u64)reserved_top + reserved_bot) 172 return -EINVAL; 173 174 /* num must be a power of 2 */ 175 if (num != roundup_pow_of_two(num)) 176 return -EINVAL; 177 178 if (reserved_bot + reserved_top >= num) 179 return -EINVAL; 180 181 bitmap->last = 0; 182 bitmap->top = 0; 183 bitmap->max = num - reserved_top; 184 bitmap->mask = mask; 185 bitmap->reserved_top = reserved_top; 186 bitmap->avail = num - reserved_top - reserved_bot; 187 spin_lock_init(&bitmap->lock); 188 bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * 189 sizeof (long), GFP_KERNEL); 190 if (!bitmap->table) 191 return -ENOMEM; 192 193 bitmap_set(bitmap->table, 0, reserved_bot); 194 195 return 0; 196} 197 198void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 199{ 200 kfree(bitmap->table); 201} 202 203/* 204 * Handling for queue buffers -- we allocate a bunch of memory and 205 * register it in a memory region at HCA virtual address 0. If the 206 * requested size is > max_direct, we split the allocation into 207 * multiple pages, so we don't require too much contiguous memory. 208 */ 209 210int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 211 struct mlx4_buf *buf) 212{ 213 dma_addr_t t; 214 215 if (size <= max_direct) { 216 buf->nbufs = 1; 217 buf->npages = 1; 218 buf->page_shift = get_order(size) + PAGE_SHIFT; 219 buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 220 size, &t, GFP_KERNEL); 221 if (!buf->direct.buf) 222 return -ENOMEM; 223 224 buf->direct.map = t; 225 226 while (t & ((1 << buf->page_shift) - 1)) { 227 --buf->page_shift; 228 buf->npages *= 2; 229 } 230 231 memset(buf->direct.buf, 0, size); 232 } else { 233 int i; 234 235 buf->direct.buf = NULL; 236 buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 237 buf->npages = buf->nbufs; 238 buf->page_shift = PAGE_SHIFT; 239 buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 240 GFP_KERNEL); 241 if (!buf->page_list) 242 return -ENOMEM; 243 244 for (i = 0; i < buf->nbufs; ++i) { 245 buf->page_list[i].buf = 246 dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 247 &t, GFP_KERNEL); 248 if (!buf->page_list[i].buf) 249 goto err_free; 250 251 buf->page_list[i].map = t; 252 253 memset(buf->page_list[i].buf, 0, PAGE_SIZE); 254 } 255 256 if (BITS_PER_LONG == 64) { 257 struct page **pages; 258 pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); 259 if (!pages) 260 goto err_free; 261 for (i = 0; i < buf->nbufs; ++i) 262 pages[i] = virt_to_page(buf->page_list[i].buf); 263 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 264 kfree(pages); 265 if (!buf->direct.buf) 266 goto err_free; 267 } 268 } 269 270 return 0; 271 272err_free: 273 mlx4_buf_free(dev, size, buf); 274 275 return -ENOMEM; 276} 277EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 278 279void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 280{ 281 int i; 282 283 if (buf->nbufs == 1) 284 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 285 buf->direct.map); 286 else { 287 if (BITS_PER_LONG == 64 && buf->direct.buf) 288 vunmap(buf->direct.buf); 289 290 for (i = 0; i < buf->nbufs; ++i) 291 if (buf->page_list[i].buf) 292 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 293 buf->page_list[i].buf, 294 buf->page_list[i].map); 295 kfree(buf->page_list); 296 } 297} 298EXPORT_SYMBOL_GPL(mlx4_buf_free); 299 300static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) 301{ 302 struct mlx4_db_pgdir *pgdir; 303 304 pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL); 305 if (!pgdir) 306 return NULL; 307 308 bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 309 pgdir->bits[0] = pgdir->order0; 310 pgdir->bits[1] = pgdir->order1; 311 pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 312 &pgdir->db_dma, GFP_KERNEL); 313 if (!pgdir->db_page) { 314 kfree(pgdir); 315 return NULL; 316 } 317 318 return pgdir; 319} 320 321static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 322 struct mlx4_db *db, int order) 323{ 324 int o; 325 int i; 326 327 for (o = order; o <= 1; ++o) { 328 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 329 if (i < MLX4_DB_PER_PAGE >> o) 330 goto found; 331 } 332 333 return -ENOMEM; 334 335found: 336 clear_bit(i, pgdir->bits[o]); 337 338 i <<= o; 339 340 if (o > order) 341 set_bit(i ^ 1, pgdir->bits[order]); 342 343 db->u.pgdir = pgdir; 344 db->index = i; 345 db->db = pgdir->db_page + db->index; 346 db->dma = pgdir->db_dma + db->index * 4; 347 db->order = order; 348 349 return 0; 350} 351 352int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) 353{ 354 struct mlx4_priv *priv = mlx4_priv(dev); 355 struct mlx4_db_pgdir *pgdir; 356 int ret = 0; 357 358 mutex_lock(&priv->pgdir_mutex); 359 360 list_for_each_entry(pgdir, &priv->pgdir_list, list) 361 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 362 goto out; 363 364 pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev)); 365 if (!pgdir) { 366 ret = -ENOMEM; 367 goto out; 368 } 369 370 list_add(&pgdir->list, &priv->pgdir_list); 371 372 /* This should never fail -- we just allocated an empty page: */ 373 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 374 375out: 376 mutex_unlock(&priv->pgdir_mutex); 377 378 return ret; 379} 380EXPORT_SYMBOL_GPL(mlx4_db_alloc); 381 382void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 383{ 384 struct mlx4_priv *priv = mlx4_priv(dev); 385 int o; 386 int i; 387 388 mutex_lock(&priv->pgdir_mutex); 389 390 o = db->order; 391 i = db->index; 392 393 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 394 clear_bit(i ^ 1, db->u.pgdir->order0); 395 ++o; 396 } 397 i >>= o; 398 set_bit(i, db->u.pgdir->bits[o]); 399 400 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 401 dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 402 db->u.pgdir->db_page, db->u.pgdir->db_dma); 403 list_del(&db->u.pgdir->list); 404 kfree(db->u.pgdir); 405 } 406 407 mutex_unlock(&priv->pgdir_mutex); 408} 409EXPORT_SYMBOL_GPL(mlx4_db_free); 410 411int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 412 int size, int max_direct) 413{ 414 int err; 415 416 err = mlx4_db_alloc(dev, &wqres->db, 1); 417 if (err) 418 return err; 419 420 *wqres->db.db = 0; 421 422 err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf); 423 if (err) 424 goto err_db; 425 426 err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 427 &wqres->mtt); 428 if (err) 429 goto err_buf; 430 431 err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); 432 if (err) 433 goto err_mtt; 434 435 return 0; 436 437err_mtt: 438 mlx4_mtt_cleanup(dev, &wqres->mtt); 439err_buf: 440 mlx4_buf_free(dev, size, &wqres->buf); 441err_db: 442 mlx4_db_free(dev, &wqres->db); 443 444 return err; 445} 446EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 447 448void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 449 int size) 450{ 451 mlx4_mtt_cleanup(dev, &wqres->mtt); 452 mlx4_buf_free(dev, size, &wqres->buf); 453 mlx4_db_free(dev, &wqres->db); 454} 455EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 456