1219820Sjeff/* 2219820Sjeff * Copyright (c) 2004 Topspin Communications. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32219820Sjeff 33219820Sjeff#include <linux/errno.h> 34219820Sjeff#include <linux/slab.h> 35219820Sjeff 36219820Sjeff#include "mthca_dev.h" 37219820Sjeff 38219820Sjeff/* Trivial bitmap-based allocator */ 39219820Sjeffu32 mthca_alloc(struct mthca_alloc *alloc) 40219820Sjeff{ 41219820Sjeff unsigned long flags; 42219820Sjeff u32 obj; 43219820Sjeff 44219820Sjeff spin_lock_irqsave(&alloc->lock, flags); 45219820Sjeff 46219820Sjeff obj = find_next_zero_bit(alloc->table, alloc->max, alloc->last); 47219820Sjeff if (obj >= alloc->max) { 48219820Sjeff alloc->top = (alloc->top + alloc->max) & alloc->mask; 49219820Sjeff obj = find_first_zero_bit(alloc->table, alloc->max); 50219820Sjeff } 51219820Sjeff 52219820Sjeff if (obj < alloc->max) { 53219820Sjeff set_bit(obj, alloc->table); 54219820Sjeff obj |= alloc->top; 55219820Sjeff } else 56219820Sjeff obj = -1; 57219820Sjeff 58219820Sjeff spin_unlock_irqrestore(&alloc->lock, flags); 59219820Sjeff 60219820Sjeff return obj; 61219820Sjeff} 62219820Sjeff 63219820Sjeffvoid mthca_free(struct mthca_alloc *alloc, u32 obj) 64219820Sjeff{ 65219820Sjeff unsigned long flags; 66219820Sjeff 67219820Sjeff obj &= alloc->max - 1; 68219820Sjeff 69219820Sjeff spin_lock_irqsave(&alloc->lock, flags); 70219820Sjeff 71219820Sjeff clear_bit(obj, alloc->table); 72219820Sjeff alloc->last = min(alloc->last, obj); 73219820Sjeff alloc->top = (alloc->top + alloc->max) & alloc->mask; 74219820Sjeff 75219820Sjeff spin_unlock_irqrestore(&alloc->lock, flags); 76219820Sjeff} 77219820Sjeff 78219820Sjeffint mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, 79219820Sjeff u32 reserved) 80219820Sjeff{ 81219820Sjeff int i; 82219820Sjeff 83219820Sjeff /* num must be a power of 2 */ 84219820Sjeff if (num != 1 << (ffs(num) - 1)) 85219820Sjeff return -EINVAL; 86219820Sjeff 87219820Sjeff alloc->last = 0; 88219820Sjeff alloc->top = 0; 89219820Sjeff alloc->max = num; 90219820Sjeff alloc->mask = mask; 91219820Sjeff spin_lock_init(&alloc->lock); 92219820Sjeff alloc->table = kmalloc(BITS_TO_LONGS(num) * sizeof (long), 93219820Sjeff GFP_KERNEL); 94219820Sjeff if (!alloc->table) 95219820Sjeff return -ENOMEM; 96219820Sjeff 97219820Sjeff bitmap_zero(alloc->table, num); 98219820Sjeff for (i = 0; i < reserved; ++i) 99219820Sjeff set_bit(i, alloc->table); 100219820Sjeff 101219820Sjeff return 0; 102219820Sjeff} 103219820Sjeff 104219820Sjeffvoid mthca_alloc_cleanup(struct mthca_alloc *alloc) 105219820Sjeff{ 106219820Sjeff kfree(alloc->table); 107219820Sjeff} 108219820Sjeff 109219820Sjeff/* 110219820Sjeff * Array of pointers with lazy allocation of leaf pages. Callers of 111219820Sjeff * _get, _set and _clear methods must use a lock or otherwise 112219820Sjeff * serialize access to the array. 113219820Sjeff */ 114219820Sjeff 115219820Sjeff#define MTHCA_ARRAY_MASK (PAGE_SIZE / sizeof (void *) - 1) 116219820Sjeff 117219820Sjeffvoid *mthca_array_get(struct mthca_array *array, int index) 118219820Sjeff{ 119219820Sjeff int p = (index * sizeof (void *)) >> PAGE_SHIFT; 120219820Sjeff 121219820Sjeff if (array->page_list[p].page) 122219820Sjeff return array->page_list[p].page[index & MTHCA_ARRAY_MASK]; 123219820Sjeff else 124219820Sjeff return NULL; 125219820Sjeff} 126219820Sjeff 127219820Sjeffint mthca_array_set(struct mthca_array *array, int index, void *value) 128219820Sjeff{ 129219820Sjeff int p = (index * sizeof (void *)) >> PAGE_SHIFT; 130219820Sjeff 131219820Sjeff /* Allocate with GFP_ATOMIC because we'll be called with locks held. */ 132219820Sjeff if (!array->page_list[p].page) 133219820Sjeff array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC); 134219820Sjeff 135219820Sjeff if (!array->page_list[p].page) 136219820Sjeff return -ENOMEM; 137219820Sjeff 138219820Sjeff array->page_list[p].page[index & MTHCA_ARRAY_MASK] = value; 139219820Sjeff ++array->page_list[p].used; 140219820Sjeff 141219820Sjeff return 0; 142219820Sjeff} 143219820Sjeff 144219820Sjeffvoid mthca_array_clear(struct mthca_array *array, int index) 145219820Sjeff{ 146219820Sjeff int p = (index * sizeof (void *)) >> PAGE_SHIFT; 147219820Sjeff 148219820Sjeff if (--array->page_list[p].used == 0) { 149219820Sjeff free_page((unsigned long) array->page_list[p].page); 150219820Sjeff array->page_list[p].page = NULL; 151219820Sjeff } else 152219820Sjeff array->page_list[p].page[index & MTHCA_ARRAY_MASK] = NULL; 153219820Sjeff 154219820Sjeff if (array->page_list[p].used < 0) 155219820Sjeff pr_debug("Array %p index %d page %d with ref count %d < 0\n", 156219820Sjeff array, index, p, array->page_list[p].used); 157219820Sjeff} 158219820Sjeff 159219820Sjeffint mthca_array_init(struct mthca_array *array, int nent) 160219820Sjeff{ 161219820Sjeff int npage = (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; 162219820Sjeff int i; 163219820Sjeff 164219820Sjeff array->page_list = kmalloc(npage * sizeof *array->page_list, GFP_KERNEL); 165219820Sjeff if (!array->page_list) 166219820Sjeff return -ENOMEM; 167219820Sjeff 168219820Sjeff for (i = 0; i < npage; ++i) { 169219820Sjeff array->page_list[i].page = NULL; 170219820Sjeff array->page_list[i].used = 0; 171219820Sjeff } 172219820Sjeff 173219820Sjeff return 0; 174219820Sjeff} 175219820Sjeff 176219820Sjeffvoid mthca_array_cleanup(struct mthca_array *array, int nent) 177219820Sjeff{ 178219820Sjeff int i; 179219820Sjeff 180219820Sjeff for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i) 181219820Sjeff free_page((unsigned long) array->page_list[i].page); 182219820Sjeff 183219820Sjeff kfree(array->page_list); 184219820Sjeff} 185219820Sjeff 186219820Sjeff/* 187219820Sjeff * Handling for queue buffers -- we allocate a bunch of memory and 188219820Sjeff * register it in a memory region at HCA virtual address 0. If the 189219820Sjeff * requested size is > max_direct, we split the allocation into 190219820Sjeff * multiple pages, so we don't require too much contiguous memory. 191219820Sjeff */ 192219820Sjeff 193219820Sjeffint mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, 194219820Sjeff union mthca_buf *buf, int *is_direct, struct mthca_pd *pd, 195219820Sjeff int hca_write, struct mthca_mr *mr) 196219820Sjeff{ 197219820Sjeff int err = -ENOMEM; 198219820Sjeff int npages, shift; 199219820Sjeff u64 *dma_list = NULL; 200219820Sjeff dma_addr_t t; 201219820Sjeff int i; 202219820Sjeff 203219820Sjeff if (size <= max_direct) { 204219820Sjeff *is_direct = 1; 205219820Sjeff npages = 1; 206219820Sjeff shift = get_order(size) + PAGE_SHIFT; 207219820Sjeff 208219820Sjeff buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 209219820Sjeff size, &t, GFP_KERNEL); 210219820Sjeff if (!buf->direct.buf) 211219820Sjeff return -ENOMEM; 212219820Sjeff 213219820Sjeff pci_unmap_addr_set(&buf->direct, mapping, t); 214219820Sjeff 215219820Sjeff memset(buf->direct.buf, 0, size); 216219820Sjeff 217219820Sjeff while (t & ((1 << shift) - 1)) { 218219820Sjeff --shift; 219219820Sjeff npages *= 2; 220219820Sjeff } 221219820Sjeff 222219820Sjeff dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); 223219820Sjeff if (!dma_list) 224219820Sjeff goto err_free; 225219820Sjeff 226219820Sjeff for (i = 0; i < npages; ++i) 227219820Sjeff dma_list[i] = t + i * (1 << shift); 228219820Sjeff } else { 229219820Sjeff *is_direct = 0; 230219820Sjeff npages = (size + PAGE_SIZE - 1) / PAGE_SIZE; 231219820Sjeff shift = PAGE_SHIFT; 232219820Sjeff 233219820Sjeff dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); 234219820Sjeff if (!dma_list) 235219820Sjeff return -ENOMEM; 236219820Sjeff 237219820Sjeff buf->page_list = kmalloc(npages * sizeof *buf->page_list, 238219820Sjeff GFP_KERNEL); 239219820Sjeff if (!buf->page_list) 240219820Sjeff goto err_out; 241219820Sjeff 242219820Sjeff for (i = 0; i < npages; ++i) 243219820Sjeff buf->page_list[i].buf = NULL; 244219820Sjeff 245219820Sjeff for (i = 0; i < npages; ++i) { 246219820Sjeff buf->page_list[i].buf = 247219820Sjeff dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 248219820Sjeff &t, GFP_KERNEL); 249219820Sjeff if (!buf->page_list[i].buf) 250219820Sjeff goto err_free; 251219820Sjeff 252219820Sjeff dma_list[i] = t; 253219820Sjeff pci_unmap_addr_set(&buf->page_list[i], mapping, t); 254219820Sjeff 255219820Sjeff clear_page(buf->page_list[i].buf); 256219820Sjeff } 257219820Sjeff } 258219820Sjeff 259219820Sjeff err = mthca_mr_alloc_phys(dev, pd->pd_num, 260219820Sjeff dma_list, shift, npages, 261219820Sjeff 0, size, 262219820Sjeff MTHCA_MPT_FLAG_LOCAL_READ | 263219820Sjeff (hca_write ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0), 264219820Sjeff mr); 265219820Sjeff if (err) 266219820Sjeff goto err_free; 267219820Sjeff 268219820Sjeff kfree(dma_list); 269219820Sjeff 270219820Sjeff return 0; 271219820Sjeff 272219820Sjefferr_free: 273219820Sjeff mthca_buf_free(dev, size, buf, *is_direct, NULL); 274219820Sjeff 275219820Sjefferr_out: 276219820Sjeff kfree(dma_list); 277219820Sjeff 278219820Sjeff return err; 279219820Sjeff} 280219820Sjeff 281219820Sjeffvoid mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, 282219820Sjeff int is_direct, struct mthca_mr *mr) 283219820Sjeff{ 284219820Sjeff int i; 285219820Sjeff 286219820Sjeff if (mr) 287219820Sjeff mthca_free_mr(dev, mr); 288219820Sjeff 289219820Sjeff if (is_direct) 290219820Sjeff dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 291219820Sjeff pci_unmap_addr(&buf->direct, mapping)); 292219820Sjeff else { 293219820Sjeff for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i) 294219820Sjeff dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 295219820Sjeff buf->page_list[i].buf, 296219820Sjeff pci_unmap_addr(&buf->page_list[i], 297219820Sjeff mapping)); 298219820Sjeff kfree(buf->page_list); 299219820Sjeff } 300219820Sjeff} 301