1219820Sjeff/* 2219820Sjeff * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3272407Shselasky * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved. 4219820Sjeff * 5219820Sjeff * This software is available to you under a choice of one of two 6219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 7219820Sjeff * General Public License (GPL) Version 2, available from the file 8219820Sjeff * COPYING in the main directory of this source tree, or the 9219820Sjeff * OpenIB.org BSD license below: 10219820Sjeff * 11219820Sjeff * Redistribution and use in source and binary forms, with or 12219820Sjeff * without modification, are permitted provided that the following 13219820Sjeff * conditions are met: 14219820Sjeff * 15219820Sjeff * - Redistributions of source code must retain the above 16219820Sjeff * copyright notice, this list of conditions and the following 17219820Sjeff * disclaimer. 18219820Sjeff * 19219820Sjeff * - Redistributions in binary form must reproduce the above 20219820Sjeff * copyright notice, this list of conditions and the following 21219820Sjeff * disclaimer in the documentation and/or other materials 22219820Sjeff * provided with the distribution. 23219820Sjeff * 24219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31219820Sjeff * SOFTWARE. 32219820Sjeff */ 33219820Sjeff 34219820Sjeff#include <linux/errno.h> 35219820Sjeff#include <linux/slab.h> 36219820Sjeff#include <linux/mm.h> 37272407Shselasky#include <linux/module.h> 38219820Sjeff#include <linux/dma-mapping.h> 39219820Sjeff#include <linux/vmalloc.h> 40219820Sjeff 41219820Sjeff#include "mlx4.h" 42219820Sjeff 43219820Sjeffu32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 44219820Sjeff{ 45219820Sjeff u32 obj; 46219820Sjeff 47219820Sjeff spin_lock(&bitmap->lock); 48219820Sjeff 49219820Sjeff obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 50219820Sjeff if (obj >= bitmap->max) { 51219820Sjeff bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 52219820Sjeff & bitmap->mask; 53219820Sjeff obj = find_first_zero_bit(bitmap->table, bitmap->max); 54219820Sjeff } 55219820Sjeff 56219820Sjeff if (obj < bitmap->max) { 57219820Sjeff set_bit(obj, bitmap->table); 58219820Sjeff bitmap->last = (obj + 1); 59219820Sjeff if (bitmap->last == bitmap->max) 60219820Sjeff bitmap->last = 0; 61219820Sjeff obj |= bitmap->top; 62219820Sjeff } else 63219820Sjeff obj = -1; 64219820Sjeff 65219820Sjeff if (obj != -1) 66219820Sjeff --bitmap->avail; 67219820Sjeff 68219820Sjeff spin_unlock(&bitmap->lock); 69219820Sjeff 70219820Sjeff return obj; 71219820Sjeff} 72219820Sjeff 73272407Shselaskyvoid mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 74219820Sjeff{ 75272407Shselasky mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 76219820Sjeff} 77219820Sjeff 78219820Sjeffstatic unsigned long find_aligned_range(unsigned long *bitmap, 79219820Sjeff u32 start, u32 nbits, 80255932Salfred int len, int align, u32 skip_mask) 81219820Sjeff{ 82219820Sjeff unsigned long end, i; 83219820Sjeff 84219820Sjeffagain: 85219820Sjeff start = ALIGN(start, align); 86219820Sjeff 87255932Salfred while ((start < nbits) && (test_bit(start, bitmap) || 88255932Salfred (start & skip_mask))) 89219820Sjeff start += align; 90219820Sjeff 91219820Sjeff if (start >= nbits) 92219820Sjeff return -1; 93219820Sjeff 94219820Sjeff end = start+len; 95219820Sjeff if (end > nbits) 96219820Sjeff return -1; 97219820Sjeff 98219820Sjeff for (i = start + 1; i < end; i++) { 99255932Salfred if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { 100219820Sjeff start = i + 1; 101219820Sjeff goto again; 102219820Sjeff } 103219820Sjeff } 104219820Sjeff 105219820Sjeff return start; 106219820Sjeff} 107219820Sjeff 108255932Salfredu32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, 109255932Salfred int align, u32 skip_mask) 110219820Sjeff{ 111255932Salfred u32 obj; 112219820Sjeff 113255932Salfred if (likely(cnt == 1 && align == 1 && !skip_mask)) 114219820Sjeff return mlx4_bitmap_alloc(bitmap); 115219820Sjeff 116219820Sjeff spin_lock(&bitmap->lock); 117219820Sjeff 118219820Sjeff obj = find_aligned_range(bitmap->table, bitmap->last, 119255932Salfred bitmap->max, cnt, align, skip_mask); 120219820Sjeff if (obj >= bitmap->max) { 121219820Sjeff bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 122219820Sjeff & bitmap->mask; 123219820Sjeff obj = find_aligned_range(bitmap->table, 0, bitmap->max, 124255932Salfred cnt, align, skip_mask); 125219820Sjeff } 126219820Sjeff 127219820Sjeff if (obj < bitmap->max) { 128255932Salfred bitmap_set(bitmap->table, obj, cnt); 129219820Sjeff if (obj == bitmap->last) { 130219820Sjeff bitmap->last = (obj + cnt); 131219820Sjeff if (bitmap->last >= bitmap->max) 132219820Sjeff bitmap->last = 0; 133219820Sjeff } 134219820Sjeff obj |= bitmap->top; 135219820Sjeff } else 136219820Sjeff obj = -1; 137219820Sjeff 138219820Sjeff if (obj != -1) 139219820Sjeff bitmap->avail -= cnt; 140219820Sjeff 141219820Sjeff spin_unlock(&bitmap->lock); 142219820Sjeff 143219820Sjeff return obj; 144219820Sjeff} 145219820Sjeff 146219820Sjeffu32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 147219820Sjeff{ 148219820Sjeff return bitmap->avail; 149219820Sjeff} 150219820Sjeff 151272407Shselaskyvoid mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, 152272407Shselasky int use_rr) 153219820Sjeff{ 154219820Sjeff obj &= bitmap->max + bitmap->reserved_top - 1; 155219820Sjeff 156219820Sjeff spin_lock(&bitmap->lock); 157272407Shselasky if (!use_rr) { 158272407Shselasky bitmap->last = min(bitmap->last, obj); 159272407Shselasky bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 160272407Shselasky & bitmap->mask; 161272407Shselasky } 162255932Salfred bitmap_clear(bitmap->table, obj, cnt); 163219820Sjeff bitmap->avail += cnt; 164219820Sjeff spin_unlock(&bitmap->lock); 165219820Sjeff} 166219820Sjeff 167219820Sjeffint mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 168219820Sjeff u32 reserved_bot, u32 reserved_top) 169219820Sjeff{ 170255932Salfred /* sanity check */ 171255932Salfred if (num <= (u64)reserved_top + reserved_bot) 172255932Salfred return -EINVAL; 173219820Sjeff 174219820Sjeff /* num must be a power of 2 */ 175219820Sjeff if (num != roundup_pow_of_two(num)) 176219820Sjeff return -EINVAL; 177219820Sjeff 178255932Salfred if (reserved_bot + reserved_top >= num) 179255932Salfred return -EINVAL; 180255932Salfred 181219820Sjeff bitmap->last = 0; 182219820Sjeff bitmap->top = 0; 183219820Sjeff bitmap->max = num - reserved_top; 184219820Sjeff bitmap->mask = mask; 185219820Sjeff bitmap->reserved_top = reserved_top; 186219820Sjeff bitmap->avail = num - reserved_top - reserved_bot; 187219820Sjeff spin_lock_init(&bitmap->lock); 188219820Sjeff bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * 189219820Sjeff sizeof (long), GFP_KERNEL); 190219820Sjeff if (!bitmap->table) 191219820Sjeff return -ENOMEM; 192219820Sjeff 193255932Salfred bitmap_set(bitmap->table, 0, reserved_bot); 194219820Sjeff 195219820Sjeff return 0; 196219820Sjeff} 197219820Sjeff 198219820Sjeffvoid mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 199219820Sjeff{ 200219820Sjeff kfree(bitmap->table); 201219820Sjeff} 202219820Sjeff 203219820Sjeff/* 204219820Sjeff * Handling for queue buffers -- we allocate a bunch of memory and 205219820Sjeff * register it in a memory region at HCA virtual address 0. If the 206219820Sjeff * requested size is > max_direct, we split the allocation into 207219820Sjeff * multiple pages, so we don't require too much contiguous memory. 208219820Sjeff */ 209219820Sjeff 210219820Sjeffint mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 211219820Sjeff struct mlx4_buf *buf) 212219820Sjeff{ 213219820Sjeff dma_addr_t t; 214219820Sjeff 215219820Sjeff if (size <= max_direct) { 216219820Sjeff buf->nbufs = 1; 217219820Sjeff buf->npages = 1; 218219820Sjeff buf->page_shift = get_order(size) + PAGE_SHIFT; 219219820Sjeff buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 220219820Sjeff size, &t, GFP_KERNEL); 221219820Sjeff if (!buf->direct.buf) 222219820Sjeff return -ENOMEM; 223219820Sjeff 224219820Sjeff buf->direct.map = t; 225219820Sjeff 226219820Sjeff while (t & ((1 << buf->page_shift) - 1)) { 227219820Sjeff --buf->page_shift; 228219820Sjeff buf->npages *= 2; 229219820Sjeff } 230219820Sjeff 231219820Sjeff memset(buf->direct.buf, 0, size); 232219820Sjeff } else { 233219820Sjeff int i; 234219820Sjeff 235219820Sjeff buf->direct.buf = NULL; 236219820Sjeff buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 237219820Sjeff buf->npages = buf->nbufs; 238219820Sjeff buf->page_shift = PAGE_SHIFT; 239255932Salfred buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 240219820Sjeff GFP_KERNEL); 241219820Sjeff if (!buf->page_list) 242219820Sjeff return -ENOMEM; 243219820Sjeff 244219820Sjeff for (i = 0; i < buf->nbufs; ++i) { 245219820Sjeff buf->page_list[i].buf = 246219820Sjeff dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 247219820Sjeff &t, GFP_KERNEL); 248219820Sjeff if (!buf->page_list[i].buf) 249219820Sjeff goto err_free; 250219820Sjeff 251219820Sjeff buf->page_list[i].map = t; 252219820Sjeff 253219820Sjeff memset(buf->page_list[i].buf, 0, PAGE_SIZE); 254219820Sjeff } 255219820Sjeff 256219820Sjeff if (BITS_PER_LONG == 64) { 257219820Sjeff struct page **pages; 258219820Sjeff pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); 259219820Sjeff if (!pages) 260219820Sjeff goto err_free; 261219820Sjeff for (i = 0; i < buf->nbufs; ++i) 262219820Sjeff pages[i] = virt_to_page(buf->page_list[i].buf); 263219820Sjeff buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 264219820Sjeff kfree(pages); 265219820Sjeff if (!buf->direct.buf) 266219820Sjeff goto err_free; 267219820Sjeff } 268219820Sjeff } 269219820Sjeff 270219820Sjeff return 0; 271219820Sjeff 272219820Sjefferr_free: 273219820Sjeff mlx4_buf_free(dev, size, buf); 274219820Sjeff 275219820Sjeff return -ENOMEM; 276219820Sjeff} 277219820SjeffEXPORT_SYMBOL_GPL(mlx4_buf_alloc); 278219820Sjeff 279219820Sjeffvoid mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 280219820Sjeff{ 281219820Sjeff int i; 282219820Sjeff 283219820Sjeff if (buf->nbufs == 1) 284219820Sjeff dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 285219820Sjeff buf->direct.map); 286219820Sjeff else { 287219820Sjeff if (BITS_PER_LONG == 64 && buf->direct.buf) 288219820Sjeff vunmap(buf->direct.buf); 289219820Sjeff 290219820Sjeff for (i = 0; i < buf->nbufs; ++i) 291219820Sjeff if (buf->page_list[i].buf) 292219820Sjeff dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 293219820Sjeff buf->page_list[i].buf, 294219820Sjeff buf->page_list[i].map); 295219820Sjeff kfree(buf->page_list); 296219820Sjeff } 297219820Sjeff} 298219820SjeffEXPORT_SYMBOL_GPL(mlx4_buf_free); 299219820Sjeff 300219820Sjeffstatic struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) 301219820Sjeff{ 302219820Sjeff struct mlx4_db_pgdir *pgdir; 303219820Sjeff 304219820Sjeff pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL); 305219820Sjeff if (!pgdir) 306219820Sjeff return NULL; 307219820Sjeff 308219820Sjeff bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 309219820Sjeff pgdir->bits[0] = pgdir->order0; 310219820Sjeff pgdir->bits[1] = pgdir->order1; 311219820Sjeff pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 312219820Sjeff &pgdir->db_dma, GFP_KERNEL); 313219820Sjeff if (!pgdir->db_page) { 314219820Sjeff kfree(pgdir); 315219820Sjeff return NULL; 316219820Sjeff } 317219820Sjeff 318219820Sjeff return pgdir; 319219820Sjeff} 320219820Sjeff 321219820Sjeffstatic int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 322219820Sjeff struct mlx4_db *db, int order) 323219820Sjeff{ 324219820Sjeff int o; 325219820Sjeff int i; 326219820Sjeff 327219820Sjeff for (o = order; o <= 1; ++o) { 328219820Sjeff i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 329219820Sjeff if (i < MLX4_DB_PER_PAGE >> o) 330219820Sjeff goto found; 331219820Sjeff } 332219820Sjeff 333219820Sjeff return -ENOMEM; 334219820Sjeff 335219820Sjefffound: 336219820Sjeff clear_bit(i, pgdir->bits[o]); 337219820Sjeff 338219820Sjeff i <<= o; 339219820Sjeff 340219820Sjeff if (o > order) 341219820Sjeff set_bit(i ^ 1, pgdir->bits[order]); 342219820Sjeff 343219820Sjeff db->u.pgdir = pgdir; 344219820Sjeff db->index = i; 345219820Sjeff db->db = pgdir->db_page + db->index; 346219820Sjeff db->dma = pgdir->db_dma + db->index * 4; 347219820Sjeff db->order = order; 348219820Sjeff 349219820Sjeff return 0; 350219820Sjeff} 351219820Sjeff 352219820Sjeffint mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) 353219820Sjeff{ 354219820Sjeff struct mlx4_priv *priv = mlx4_priv(dev); 355219820Sjeff struct mlx4_db_pgdir *pgdir; 356219820Sjeff int ret = 0; 357219820Sjeff 358219820Sjeff mutex_lock(&priv->pgdir_mutex); 359219820Sjeff 360219820Sjeff list_for_each_entry(pgdir, &priv->pgdir_list, list) 361219820Sjeff if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 362219820Sjeff goto out; 363219820Sjeff 364219820Sjeff pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev)); 365219820Sjeff if (!pgdir) { 366219820Sjeff ret = -ENOMEM; 367219820Sjeff goto out; 368219820Sjeff } 369219820Sjeff 370219820Sjeff list_add(&pgdir->list, &priv->pgdir_list); 371219820Sjeff 372219820Sjeff /* This should never fail -- we just allocated an empty page: */ 373219820Sjeff WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 374219820Sjeff 375219820Sjeffout: 376219820Sjeff mutex_unlock(&priv->pgdir_mutex); 377219820Sjeff 378219820Sjeff return ret; 379219820Sjeff} 380219820SjeffEXPORT_SYMBOL_GPL(mlx4_db_alloc); 381219820Sjeff 382219820Sjeffvoid mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 383219820Sjeff{ 384219820Sjeff struct mlx4_priv *priv = mlx4_priv(dev); 385219820Sjeff int o; 386219820Sjeff int i; 387219820Sjeff 388219820Sjeff mutex_lock(&priv->pgdir_mutex); 389219820Sjeff 390219820Sjeff o = db->order; 391219820Sjeff i = db->index; 392219820Sjeff 393219820Sjeff if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 394219820Sjeff clear_bit(i ^ 1, db->u.pgdir->order0); 395219820Sjeff ++o; 396219820Sjeff } 397219820Sjeff i >>= o; 398219820Sjeff set_bit(i, db->u.pgdir->bits[o]); 399219820Sjeff 400219820Sjeff if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 401219820Sjeff dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 402219820Sjeff db->u.pgdir->db_page, db->u.pgdir->db_dma); 403219820Sjeff list_del(&db->u.pgdir->list); 404219820Sjeff kfree(db->u.pgdir); 405219820Sjeff } 406219820Sjeff 407219820Sjeff mutex_unlock(&priv->pgdir_mutex); 408219820Sjeff} 409219820SjeffEXPORT_SYMBOL_GPL(mlx4_db_free); 410219820Sjeff 411219820Sjeffint mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 412219820Sjeff int size, int max_direct) 413219820Sjeff{ 414219820Sjeff int err; 415219820Sjeff 416219820Sjeff err = mlx4_db_alloc(dev, &wqres->db, 1); 417219820Sjeff if (err) 418219820Sjeff return err; 419219820Sjeff 420219820Sjeff *wqres->db.db = 0; 421219820Sjeff 422219820Sjeff err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf); 423219820Sjeff if (err) 424219820Sjeff goto err_db; 425219820Sjeff 426219820Sjeff err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 427219820Sjeff &wqres->mtt); 428219820Sjeff if (err) 429219820Sjeff goto err_buf; 430219820Sjeff 431219820Sjeff err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); 432219820Sjeff if (err) 433219820Sjeff goto err_mtt; 434219820Sjeff 435219820Sjeff return 0; 436219820Sjeff 437219820Sjefferr_mtt: 438219820Sjeff mlx4_mtt_cleanup(dev, &wqres->mtt); 439219820Sjefferr_buf: 440219820Sjeff mlx4_buf_free(dev, size, &wqres->buf); 441219820Sjefferr_db: 442219820Sjeff mlx4_db_free(dev, &wqres->db); 443219820Sjeff 444219820Sjeff return err; 445219820Sjeff} 446219820SjeffEXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 447219820Sjeff 448219820Sjeffvoid mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 449219820Sjeff int size) 450219820Sjeff{ 451219820Sjeff mlx4_mtt_cleanup(dev, &wqres->mtt); 452219820Sjeff mlx4_buf_free(dev, size, &wqres->buf); 453219820Sjeff mlx4_db_free(dev, &wqres->db); 454219820Sjeff} 455219820SjeffEXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 456