1290650Shselasky/*- 2290650Shselasky * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: releng/11.0/sys/dev/mlx5/mlx5_core/mlx5_alloc.c 290650 2015-11-10 12:20:22Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/errno.h> 29290650Shselasky#include <linux/slab.h> 30290650Shselasky#include <linux/mm.h> 31290650Shselasky#include <linux/dma-mapping.h> 32290650Shselasky#include <linux/vmalloc.h> 33290650Shselasky#include <dev/mlx5/driver.h> 34290650Shselasky 35290650Shselasky#include "mlx5_core.h" 36290650Shselasky 37290650Shselasky/* Handling for queue buffers -- we allocate a bunch of memory and 38290650Shselasky * register it in a memory region at HCA virtual address 0. If the 39290650Shselasky * requested size is > max_direct, we split the allocation into 40290650Shselasky * multiple pages, so we don't require too much contiguous memory. 41290650Shselasky */ 42290650Shselasky 43290650Shselaskystatic void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev, 44290650Shselasky size_t size, dma_addr_t *dma_handle, 45290650Shselasky int node) 46290650Shselasky{ 47290650Shselasky void *cpu_handle; 48290650Shselasky 49290650Shselasky cpu_handle = dma_zalloc_coherent(&dev->pdev->dev, size, 50290650Shselasky dma_handle, GFP_KERNEL); 51290650Shselasky return cpu_handle; 52290650Shselasky} 53290650Shselasky 54290650Shselaskyint mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, int max_direct, 55290650Shselasky struct mlx5_buf *buf, int node) 56290650Shselasky{ 57290650Shselasky dma_addr_t t; 58290650Shselasky 59290650Shselasky buf->size = size; 60290650Shselasky if (size <= max_direct) { 61290650Shselasky buf->nbufs = 1; 62290650Shselasky buf->npages = 1; 63290650Shselasky buf->page_shift = (u8)get_order(size) + PAGE_SHIFT; 64290650Shselasky buf->direct.buf = mlx5_dma_zalloc_coherent_node(dev, size, 65290650Shselasky &t, node); 66290650Shselasky if (!buf->direct.buf) 67290650Shselasky return -ENOMEM; 68290650Shselasky 69290650Shselasky buf->direct.map = t; 70290650Shselasky 71290650Shselasky while (t & ((1 << buf->page_shift) - 1)) { 72290650Shselasky --buf->page_shift; 73290650Shselasky buf->npages *= 2; 74290650Shselasky } 75290650Shselasky } else { 76290650Shselasky int i; 77290650Shselasky 78290650Shselasky buf->direct.buf = NULL; 79290650Shselasky buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 80290650Shselasky buf->npages = buf->nbufs; 81290650Shselasky buf->page_shift = PAGE_SHIFT; 82290650Shselasky buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 83290650Shselasky GFP_KERNEL); 84290650Shselasky 85290650Shselasky for (i = 0; i < buf->nbufs; i++) { 86290650Shselasky buf->page_list[i].buf = 87290650Shselasky mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE, 88290650Shselasky &t, node); 89290650Shselasky 90290650Shselasky buf->page_list[i].map = t; 91290650Shselasky } 92290650Shselasky 93290650Shselasky if (BITS_PER_LONG == 64) { 94290650Shselasky struct page **pages; 95290650Shselasky 96290650Shselasky pages = kmalloc(sizeof(*pages) * (buf->nbufs + 1), 97290650Shselasky GFP_KERNEL); 98290650Shselasky for (i = 0; i < buf->nbufs; i++) 99290650Shselasky pages[i] = virt_to_page(buf->page_list[i].buf); 100290650Shselasky pages[buf->nbufs] = pages[0]; 101290650Shselasky buf->direct.buf = vmap(pages, buf->nbufs + 1, VM_MAP, 102290650Shselasky PAGE_KERNEL); 103290650Shselasky kfree(pages); 104290650Shselasky if (!buf->direct.buf) 105290650Shselasky goto err_free; 106290650Shselasky } 107290650Shselasky } 108290650Shselasky 109290650Shselasky return 0; 110290650Shselasky 111290650Shselaskyerr_free: 112290650Shselasky mlx5_buf_free(dev, buf); 113290650Shselasky 114290650Shselasky return -ENOMEM; 115290650Shselasky} 116290650Shselasky 117290650Shselaskyint mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, int max_direct, 118290650Shselasky struct mlx5_buf *buf) 119290650Shselasky{ 120290650Shselasky return mlx5_buf_alloc_node(dev, size, max_direct, 121290650Shselasky buf, dev->priv.numa_node); 122290650Shselasky} 123290650ShselaskyEXPORT_SYMBOL_GPL(mlx5_buf_alloc); 124290650Shselasky 125290650Shselasky 126290650Shselaskyvoid mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf) 127290650Shselasky{ 128290650Shselasky if (buf->nbufs == 1) 129290650Shselasky dma_free_coherent(&dev->pdev->dev, buf->size, buf->direct.buf, 130290650Shselasky buf->direct.map); 131290650Shselasky else { 132290650Shselasky int i; 133290650Shselasky if (BITS_PER_LONG == 64 && buf->direct.buf) 134290650Shselasky vunmap(buf->direct.buf); 135290650Shselasky 136290650Shselasky for (i = 0; i < buf->nbufs; i++) 137290650Shselasky if (buf->page_list[i].buf) 138290650Shselasky dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 139290650Shselasky buf->page_list[i].buf, 140290650Shselasky buf->page_list[i].map); 141290650Shselasky kfree(buf->page_list); 142290650Shselasky } 143290650Shselasky} 144290650ShselaskyEXPORT_SYMBOL_GPL(mlx5_buf_free); 145290650Shselasky 146290650Shselaskystatic struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev, 147290650Shselasky int node) 148290650Shselasky{ 149290650Shselasky struct mlx5_db_pgdir *pgdir; 150290650Shselasky 151290650Shselasky pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); 152290650Shselasky 153290650Shselasky bitmap_fill(pgdir->bitmap, MLX5_DB_PER_PAGE); 154290650Shselasky 155290650Shselasky pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE, 156290650Shselasky &pgdir->db_dma, node); 157290650Shselasky if (!pgdir->db_page) { 158290650Shselasky kfree(pgdir); 159290650Shselasky return NULL; 160290650Shselasky } 161290650Shselasky 162290650Shselasky return pgdir; 163290650Shselasky} 164290650Shselasky 165290650Shselaskystatic int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir, 166290650Shselasky struct mlx5_db *db) 167290650Shselasky{ 168290650Shselasky int offset; 169290650Shselasky int i; 170290650Shselasky 171290650Shselasky i = find_first_bit(pgdir->bitmap, MLX5_DB_PER_PAGE); 172290650Shselasky if (i >= MLX5_DB_PER_PAGE) 173290650Shselasky return -ENOMEM; 174290650Shselasky 175290650Shselasky __clear_bit(i, pgdir->bitmap); 176290650Shselasky 177290650Shselasky db->u.pgdir = pgdir; 178290650Shselasky db->index = i; 179290650Shselasky offset = db->index * L1_CACHE_BYTES; 180290650Shselasky db->db = pgdir->db_page + offset / sizeof(*pgdir->db_page); 181290650Shselasky db->dma = pgdir->db_dma + offset; 182290650Shselasky 183290650Shselasky db->db[0] = 0; 184290650Shselasky db->db[1] = 0; 185290650Shselasky 186290650Shselasky return 0; 187290650Shselasky} 188290650Shselasky 189290650Shselaskyint mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node) 190290650Shselasky{ 191290650Shselasky struct mlx5_db_pgdir *pgdir; 192290650Shselasky int ret = 0; 193290650Shselasky 194290650Shselasky mutex_lock(&dev->priv.pgdir_mutex); 195290650Shselasky 196290650Shselasky list_for_each_entry(pgdir, &dev->priv.pgdir_list, list) 197290650Shselasky if (!mlx5_alloc_db_from_pgdir(pgdir, db)) 198290650Shselasky goto out; 199290650Shselasky 200290650Shselasky pgdir = mlx5_alloc_db_pgdir(dev, node); 201290650Shselasky if (!pgdir) { 202290650Shselasky ret = -ENOMEM; 203290650Shselasky goto out; 204290650Shselasky } 205290650Shselasky 206290650Shselasky list_add(&pgdir->list, &dev->priv.pgdir_list); 207290650Shselasky 208290650Shselasky /* This should never fail -- we just allocated an empty page: */ 209290650Shselasky WARN_ON(mlx5_alloc_db_from_pgdir(pgdir, db)); 210290650Shselasky 211290650Shselaskyout: 212290650Shselasky mutex_unlock(&dev->priv.pgdir_mutex); 213290650Shselasky 214290650Shselasky return ret; 215290650Shselasky} 216290650ShselaskyEXPORT_SYMBOL_GPL(mlx5_db_alloc_node); 217290650Shselasky 218290650Shselaskyint mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db) 219290650Shselasky{ 220290650Shselasky return mlx5_db_alloc_node(dev, db, dev->priv.numa_node); 221290650Shselasky} 222290650ShselaskyEXPORT_SYMBOL_GPL(mlx5_db_alloc); 223290650Shselasky 224290650Shselaskyvoid mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) 225290650Shselasky{ 226290650Shselasky mutex_lock(&dev->priv.pgdir_mutex); 227290650Shselasky 228290650Shselasky __set_bit(db->index, db->u.pgdir->bitmap); 229290650Shselasky 230290650Shselasky if (bitmap_full(db->u.pgdir->bitmap, MLX5_DB_PER_PAGE)) { 231290650Shselasky dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 232290650Shselasky db->u.pgdir->db_page, db->u.pgdir->db_dma); 233290650Shselasky list_del(&db->u.pgdir->list); 234290650Shselasky kfree(db->u.pgdir); 235290650Shselasky } 236290650Shselasky 237290650Shselasky mutex_unlock(&dev->priv.pgdir_mutex); 238290650Shselasky} 239290650ShselaskyEXPORT_SYMBOL_GPL(mlx5_db_free); 240290650Shselasky 241290650Shselasky 242290650Shselaskyvoid mlx5_fill_page_array(struct mlx5_buf *buf, __be64 *pas) 243290650Shselasky{ 244290650Shselasky u64 addr; 245290650Shselasky int i; 246290650Shselasky 247290650Shselasky for (i = 0; i < buf->npages; i++) { 248290650Shselasky if (buf->nbufs == 1) 249290650Shselasky addr = buf->direct.map + ((u64)i << buf->page_shift); 250290650Shselasky else 251290650Shselasky addr = buf->page_list[i].map; 252290650Shselasky 253290650Shselasky pas[i] = cpu_to_be64(addr); 254290650Shselasky } 255290650Shselasky} 256290650ShselaskyEXPORT_SYMBOL_GPL(mlx5_fill_page_array); 257