1219820Sjeff/* 2219820Sjeff * Copyright (c) 2004 Topspin Communications. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32219820Sjeff 33219820Sjeff#include <linux/errno.h> 34219820Sjeff#include <linux/slab.h> 35219820Sjeff#include <linux/bitmap.h> 36219820Sjeff 37219820Sjeff#include "mthca_dev.h" 38219820Sjeff 39219820Sjeff/* Trivial bitmap-based allocator */ 40219820Sjeffu32 mthca_alloc(struct mthca_alloc *alloc) 41219820Sjeff{ 42219820Sjeff unsigned long flags; 43219820Sjeff u32 obj; 44219820Sjeff 45219820Sjeff spin_lock_irqsave(&alloc->lock, flags); 46219820Sjeff 47219820Sjeff obj = find_next_zero_bit(alloc->table, alloc->max, alloc->last); 48219820Sjeff if (obj >= alloc->max) { 49219820Sjeff alloc->top = (alloc->top + alloc->max) & alloc->mask; 50219820Sjeff obj = find_first_zero_bit(alloc->table, alloc->max); 51219820Sjeff } 52219820Sjeff 53219820Sjeff if (obj < alloc->max) { 54219820Sjeff set_bit(obj, alloc->table); 55219820Sjeff obj |= alloc->top; 56219820Sjeff } else 57219820Sjeff obj = -1; 58219820Sjeff 59219820Sjeff spin_unlock_irqrestore(&alloc->lock, flags); 60219820Sjeff 61219820Sjeff return obj; 62219820Sjeff} 63219820Sjeff 64219820Sjeffvoid mthca_free(struct mthca_alloc *alloc, u32 obj) 65219820Sjeff{ 66219820Sjeff unsigned long flags; 67219820Sjeff 68219820Sjeff obj &= alloc->max - 1; 69219820Sjeff 70219820Sjeff spin_lock_irqsave(&alloc->lock, flags); 71219820Sjeff 72219820Sjeff clear_bit(obj, alloc->table); 73219820Sjeff alloc->last = min(alloc->last, obj); 74219820Sjeff alloc->top = (alloc->top + alloc->max) & alloc->mask; 75219820Sjeff 76219820Sjeff spin_unlock_irqrestore(&alloc->lock, flags); 77219820Sjeff} 78219820Sjeff 79219820Sjeffint mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, 80219820Sjeff u32 reserved) 81219820Sjeff{ 82219820Sjeff int i; 83219820Sjeff 84219820Sjeff /* num must be a power of 2 */ 85219820Sjeff if (num != 1 << (ffs(num) - 1)) 86219820Sjeff return -EINVAL; 87219820Sjeff 88219820Sjeff alloc->last = 0; 89219820Sjeff alloc->top = 0; 90219820Sjeff alloc->max = num; 91219820Sjeff alloc->mask = mask; 92219820Sjeff spin_lock_init(&alloc->lock); 93219820Sjeff alloc->table = kmalloc(BITS_TO_LONGS(num) * sizeof (long), 94219820Sjeff GFP_KERNEL); 95219820Sjeff if (!alloc->table) 96219820Sjeff return -ENOMEM; 97219820Sjeff 98219820Sjeff bitmap_zero(alloc->table, num); 99219820Sjeff for (i = 0; i < reserved; ++i) 100219820Sjeff set_bit(i, alloc->table); 101219820Sjeff 102219820Sjeff return 0; 103219820Sjeff} 104219820Sjeff 105219820Sjeffvoid mthca_alloc_cleanup(struct mthca_alloc *alloc) 106219820Sjeff{ 107219820Sjeff kfree(alloc->table); 108219820Sjeff} 109219820Sjeff 110219820Sjeff/* 111219820Sjeff * Array of pointers with lazy allocation of leaf pages. Callers of 112219820Sjeff * _get, _set and _clear methods must use a lock or otherwise 113219820Sjeff * serialize access to the array. 114219820Sjeff */ 115219820Sjeff 116219820Sjeff#define MTHCA_ARRAY_MASK (PAGE_SIZE / sizeof (void *) - 1) 117219820Sjeff 118219820Sjeffvoid *mthca_array_get(struct mthca_array *array, int index) 119219820Sjeff{ 120219820Sjeff int p = (index * sizeof (void *)) >> PAGE_SHIFT; 121219820Sjeff 122219820Sjeff if (array->page_list[p].page) 123219820Sjeff return array->page_list[p].page[index & MTHCA_ARRAY_MASK]; 124219820Sjeff else 125219820Sjeff return NULL; 126219820Sjeff} 127219820Sjeff 128219820Sjeffint mthca_array_set(struct mthca_array *array, int index, void *value) 129219820Sjeff{ 130219820Sjeff int p = (index * sizeof (void *)) >> PAGE_SHIFT; 131219820Sjeff 132219820Sjeff /* Allocate with GFP_ATOMIC because we'll be called with locks held. */ 133219820Sjeff if (!array->page_list[p].page) 134219820Sjeff array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC); 135219820Sjeff 136219820Sjeff if (!array->page_list[p].page) 137219820Sjeff return -ENOMEM; 138219820Sjeff 139219820Sjeff array->page_list[p].page[index & MTHCA_ARRAY_MASK] = value; 140219820Sjeff ++array->page_list[p].used; 141219820Sjeff 142219820Sjeff return 0; 143219820Sjeff} 144219820Sjeff 145219820Sjeffvoid mthca_array_clear(struct mthca_array *array, int index) 146219820Sjeff{ 147219820Sjeff int p = (index * sizeof (void *)) >> PAGE_SHIFT; 148219820Sjeff 149219820Sjeff if (--array->page_list[p].used == 0) { 150219820Sjeff free_page((unsigned long) array->page_list[p].page); 151219820Sjeff array->page_list[p].page = NULL; 152219820Sjeff } else 153219820Sjeff array->page_list[p].page[index & MTHCA_ARRAY_MASK] = NULL; 154219820Sjeff 155219820Sjeff if (array->page_list[p].used < 0) 156219820Sjeff pr_debug("Array %p index %d page %d with ref count %d < 0\n", 157219820Sjeff array, index, p, array->page_list[p].used); 158219820Sjeff} 159219820Sjeff 160219820Sjeffint mthca_array_init(struct mthca_array *array, int nent) 161219820Sjeff{ 162219820Sjeff int npage = (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; 163219820Sjeff int i; 164219820Sjeff 165219820Sjeff array->page_list = kmalloc(npage * sizeof *array->page_list, GFP_KERNEL); 166219820Sjeff if (!array->page_list) 167219820Sjeff return -ENOMEM; 168219820Sjeff 169219820Sjeff for (i = 0; i < npage; ++i) { 170219820Sjeff array->page_list[i].page = NULL; 171219820Sjeff array->page_list[i].used = 0; 172219820Sjeff } 173219820Sjeff 174219820Sjeff return 0; 175219820Sjeff} 176219820Sjeff 177219820Sjeffvoid mthca_array_cleanup(struct mthca_array *array, int nent) 178219820Sjeff{ 179219820Sjeff int i; 180219820Sjeff 181219820Sjeff for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i) 182219820Sjeff free_page((unsigned long) array->page_list[i].page); 183219820Sjeff 184219820Sjeff kfree(array->page_list); 185219820Sjeff} 186219820Sjeff 187219820Sjeff/* 188219820Sjeff * Handling for queue buffers -- we allocate a bunch of memory and 189219820Sjeff * register it in a memory region at HCA virtual address 0. If the 190219820Sjeff * requested size is > max_direct, we split the allocation into 191219820Sjeff * multiple pages, so we don't require too much contiguous memory. 192219820Sjeff */ 193219820Sjeff 194219820Sjeffint mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, 195219820Sjeff union mthca_buf *buf, int *is_direct, struct mthca_pd *pd, 196219820Sjeff int hca_write, struct mthca_mr *mr) 197219820Sjeff{ 198219820Sjeff int err = -ENOMEM; 199219820Sjeff int npages, shift; 200219820Sjeff u64 *dma_list = NULL; 201219820Sjeff dma_addr_t t; 202219820Sjeff int i; 203219820Sjeff 204219820Sjeff if (size <= max_direct) { 205219820Sjeff *is_direct = 1; 206219820Sjeff npages = 1; 207219820Sjeff shift = get_order(size) + PAGE_SHIFT; 208219820Sjeff 209219820Sjeff buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 210219820Sjeff size, &t, GFP_KERNEL); 211219820Sjeff if (!buf->direct.buf) 212219820Sjeff return -ENOMEM; 213219820Sjeff 214219820Sjeff pci_unmap_addr_set(&buf->direct, mapping, t); 215219820Sjeff 216219820Sjeff memset(buf->direct.buf, 0, size); 217219820Sjeff 218219820Sjeff while (t & ((1 << shift) - 1)) { 219219820Sjeff --shift; 220219820Sjeff npages *= 2; 221219820Sjeff } 222219820Sjeff 223219820Sjeff dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); 224219820Sjeff if (!dma_list) 225219820Sjeff goto err_free; 226219820Sjeff 227219820Sjeff for (i = 0; i < npages; ++i) 228219820Sjeff dma_list[i] = t + i * (1 << shift); 229219820Sjeff } else { 230219820Sjeff *is_direct = 0; 231219820Sjeff npages = (size + PAGE_SIZE - 1) / PAGE_SIZE; 232219820Sjeff shift = PAGE_SHIFT; 233219820Sjeff 234219820Sjeff dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); 235219820Sjeff if (!dma_list) 236219820Sjeff return -ENOMEM; 237219820Sjeff 238219820Sjeff buf->page_list = kmalloc(npages * sizeof *buf->page_list, 239219820Sjeff GFP_KERNEL); 240219820Sjeff if (!buf->page_list) 241219820Sjeff goto err_out; 242219820Sjeff 243219820Sjeff for (i = 0; i < npages; ++i) 244219820Sjeff buf->page_list[i].buf = NULL; 245219820Sjeff 246219820Sjeff for (i = 0; i < npages; ++i) { 247219820Sjeff buf->page_list[i].buf = 248219820Sjeff dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 249219820Sjeff &t, GFP_KERNEL); 250219820Sjeff if (!buf->page_list[i].buf) 251219820Sjeff goto err_free; 252219820Sjeff 253219820Sjeff dma_list[i] = t; 254219820Sjeff pci_unmap_addr_set(&buf->page_list[i], mapping, t); 255219820Sjeff 256219820Sjeff clear_page(buf->page_list[i].buf); 257219820Sjeff } 258219820Sjeff } 259219820Sjeff 260219820Sjeff err = mthca_mr_alloc_phys(dev, pd->pd_num, 261219820Sjeff dma_list, shift, npages, 262219820Sjeff 0, size, 263219820Sjeff MTHCA_MPT_FLAG_LOCAL_READ | 264219820Sjeff (hca_write ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0), 265219820Sjeff mr); 266219820Sjeff if (err) 267219820Sjeff goto err_free; 268219820Sjeff 269219820Sjeff kfree(dma_list); 270219820Sjeff 271219820Sjeff return 0; 272219820Sjeff 273219820Sjefferr_free: 274219820Sjeff mthca_buf_free(dev, size, buf, *is_direct, NULL); 275219820Sjeff 276219820Sjefferr_out: 277219820Sjeff kfree(dma_list); 278219820Sjeff 279219820Sjeff return err; 280219820Sjeff} 281219820Sjeff 282219820Sjeffvoid mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, 283219820Sjeff int is_direct, struct mthca_mr *mr) 284219820Sjeff{ 285219820Sjeff int i; 286219820Sjeff 287219820Sjeff if (mr) 288219820Sjeff mthca_free_mr(dev, mr); 289219820Sjeff 290219820Sjeff if (is_direct) 291219820Sjeff dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 292219820Sjeff pci_unmap_addr(&buf->direct, mapping)); 293219820Sjeff else { 294219820Sjeff for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i) 295219820Sjeff dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 296219820Sjeff buf->page_list[i].buf, 297219820Sjeff pci_unmap_addr(&buf->page_list[i], 298219820Sjeff mapping)); 299219820Sjeff kfree(buf->page_list); 300219820Sjeff } 301219820Sjeff} 302