1/*
2 * Copyright (c) 2006, 2007 Cisco Systems, Inc.  All rights reserved.
3 * Copyright (c) 2007, 2008, 2014 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses.  You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 *     Redistribution and use in source and binary forms, with or
12 *     without modification, are permitted provided that the following
13 *     conditions are met:
14 *
15 *      - Redistributions of source code must retain the above
16 *        copyright notice, this list of conditions and the following
17 *        disclaimer.
18 *
19 *      - Redistributions in binary form must reproduce the above
20 *        copyright notice, this list of conditions and the following
21 *        disclaimer in the documentation and/or other materials
22 *        provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34/*#include <linux/errno.h>
35 #include <linux/slab.h>
36 #include <linux/mm.h>
37 #include <linux/module.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/vmalloc.h>
40 */
41#include <linux/log2.h>
42#include <linux/bitops.h>
43#include <linux/mm.h>
44#include <linux/gfp.h>
45#include "mlx4.h"
46
47uint32_t mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) {
48	uint32_t obj;
49
50	/*spin_lock(&bitmap->lock);*/
51
52	obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
53	if (obj >= bitmap->max) {
54		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
55				& bitmap->mask;
56		obj = find_first_zero_bit(bitmap->table, bitmap->max);
57	}
58
59	if (obj < bitmap->max) {
60		set_bit(obj, bitmap->table);
61		bitmap->last = (obj + 1);
62		if (bitmap->last == bitmap->max)
63			bitmap->last = 0;
64		obj |= bitmap->top;
65	} else
66		obj = -1;
67
68	if (obj != -1)
69		--bitmap->avail;
70
71	/*spin_unlock(&bitmap->lock);*/
72
73	return obj;
74}
75/*
76 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, uint32_t obj, int use_rr)
77 {
78 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr);
79 }
80 */
81static unsigned long find_aligned_range(unsigned long *bitmap, uint32_t start,
82		uint32_t nbits, int len, int align, uint32_t skip_mask) {
83	unsigned long end, i;
84
85	again: start = ALIGN(start, align);
86
87	while ((start < nbits) && (test_bit(start, bitmap) || (start & skip_mask)))
88		start += align;
89
90	if (start >= nbits)
91		return -1;
92
93	end = start + len;
94	if (end > nbits)
95		return -1;
96
97	for (i = start + 1; i < end; i++) {
98		if (test_bit(i, bitmap) || ((uint32_t) i & skip_mask)) {
99			start = i + 1;
100			goto again;
101		}
102	}
103
104	return start;
105}
106
107uint32_t mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align,
108		uint32_t skip_mask) {
109	uint32_t obj;
110
111	if (/*likely(*/cnt == 1 && align == 1 && !skip_mask/*)*/)
112		return mlx4_bitmap_alloc(bitmap);
113
114	/*spin_lock(&bitmap->lock);*/
115
116	obj = find_aligned_range(bitmap->table, bitmap->last, bitmap->max, cnt,
117			align, skip_mask);
118	if (obj >= bitmap->max) {
119		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
120				& bitmap->mask;
121		obj = find_aligned_range(bitmap->table, 0, bitmap->max, cnt, align,
122				skip_mask);
123	}
124
125	if (obj < bitmap->max) {
126		bitmap_set(bitmap->table, obj, cnt);
127		if (obj == bitmap->last) {
128			bitmap->last = (obj + cnt);
129			if (bitmap->last >= bitmap->max)
130				bitmap->last = 0;
131		}
132		obj |= bitmap->top;
133	} else
134		obj = -1;
135
136	if (obj != -1)
137		bitmap->avail -= cnt;
138
139	/*spin_unlock(&bitmap->lock);*/
140
141	return obj;
142}
143
144uint32_t mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) {
145	return bitmap->avail;
146}
147
148void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, uint32_t obj, int cnt,
149		int use_rr) {
150	obj &= bitmap->max + bitmap->reserved_top - 1;
151
152	/*spin_lock(&bitmap->lock);*/
153	if (!use_rr) {
154		bitmap->last = min(bitmap->last, obj);
155		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
156				& bitmap->mask;
157	}
158	bitmap_clear(bitmap->table, obj, cnt);
159	bitmap->avail += cnt;
160	/*spin_unlock(&bitmap->lock);*/
161}
162
163int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, uint32_t num, uint32_t mask,
164		uint32_t reserved_bot, uint32_t reserved_top) {
165	/*sanity check*/
166	if (num <= (u64) reserved_top + reserved_bot)
167		return -EINVAL;
168
169	/*num must be a power of 2*/
170	if (num != roundup_pow_of_two(num))
171		return -EINVAL;
172
173	if (reserved_bot + reserved_top >= num)
174		return -EINVAL;
175
176	bitmap->last = 0;
177	bitmap->top = 0;
178	bitmap->max = num - reserved_top;
179	bitmap->mask = mask;
180	bitmap->reserved_top = reserved_top;
181	bitmap->avail = num - reserved_top - reserved_bot;
182	/*spin_lock_init(&bitmap->lock);*/
183	bitmap->table = calloc(BITS_TO_LONGS(bitmap->max), sizeof(long));
184	if (!bitmap->table)
185		return -ENOMEM;
186
187	bitmap_set(bitmap->table, 0, reserved_bot);
188
189	return 0;
190}
191
192void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) {
193	free(bitmap->table);
194}
195
196/*
197 * Handling for queue buffers -- we allocate a bunch of memory and
198 * register it in a memory region at HCA virtual address 0.  If the
199 * requested size is > max_direct, we split the allocation into
200 * multiple pages, so we don't require too much contiguous memory.
201
202 */
203int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
204		struct mlx4_buf *buf) {
205	genpaddr_t t;
206
207	/*if (size <= max_direct) {*/
208	buf->nbufs = 1;
209	buf->npages = 1;
210	buf->page_shift = get_order(size) + PAGE_SHIFT;
211	buf->direct.buf = dma_alloc(size, &t);
212	if (!buf->direct.buf)
213		return -ENOMEM;
214
215	buf->direct.map = t;
216
217	/*printf("virt: %p; phys: %lx\n", buf->direct.buf, buf->direct.map);*/
218	/*printf("buf->page_shift: %d\n", buf->page_shift);*/
219
220	while (t & ((1 << buf->page_shift) - 1)) {
221		--buf->page_shift;
222		buf->npages *= 2;
223	}
224
225	/*printf("buf->page_shift: %d\n", buf->page_shift);*/
226
227	memset(buf->direct.buf, 0, size);
228
229	/*TODO: implement scattered allocation*/
230	/*} else {
231	 int i;
232
233	 buf->direct.buf = NULL;
234	 buf->nbufs = (size + BASE_PAGE_SIZE - 1) / BASE_PAGE_SIZE;
235	 buf->npages = buf->nbufs;
236	 buf->page_shift = PAGE_SHIFT;
237	 buf->page_list = calloc(buf->nbufs, sizeof(*buf->page_list));
238	 if (!buf->page_list)
239	 return -ENOMEM;
240
241	 for (i = 0; i < buf->nbufs; ++i) {
242	 buf->page_list[i].buf = dma_alloc(BASE_PAGE_SIZE, &t);
243	 if (!buf->page_list[i].buf)
244	 goto err_free;
245
246	 buf->page_list[i].map = t;
247
248	 memset(buf->page_list[i].buf, 0, BASE_PAGE_SIZE);
249	 }*/
250
251	/*if (BITS_PER_LONG == 64) {
252	 struct page **pages;
253	 pages = malloc(sizeof *pages * buf->nbufs);
254	 if (!pages)
255	 goto err_free;
256	 for (i = 0; i < buf->nbufs; ++i)
257	 pages[i] = virt_to_page(buf->page_list[i].buf);
258	 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
259	 kfree(pages);
260	 if (!buf->direct.buf)
261	 goto err_free;
262	 }*/
263
264	/*}*/
265
266	return 0;
267
268	/*err_free:*//*mlx4_buf_free(dev, size, buf);*/
269
270	return -ENOMEM;
271}
272/*
273 EXPORT_SYMBOL_GPL(mlx4_buf_alloc);
274
275 void mlx4_buf_free(struct mlx4_priv *priv, int size, struct mlx4_buf *buf)
276 {
277 int i;
278
279 if (buf->nbufs == 1)
280 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf,
281 buf->direct.map);
282 else {
283 if (BITS_PER_LONG == 64 && buf->direct.buf)
284 vunmap(buf->direct.buf);
285
286 for (i = 0; i < buf->nbufs; ++i)
287 if (buf->page_list[i].buf)
288 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
289 buf->page_list[i].buf,
290 buf->page_list[i].map);
291 kfree(buf->page_list);
292 }
293 }
294 EXPORT_SYMBOL_GPL(mlx4_buf_free);
295 */
296static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(void) {
297	struct mlx4_db_pgdir *pgdir;
298
299	pgdir = calloc(1, sizeof *pgdir);
300	if (!pgdir)
301		return NULL;
302
303	bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);
304	pgdir->bits[0] = pgdir->order0;
305	pgdir->bits[1] = pgdir->order1;
306	pgdir->db_page = dma_alloc(BASE_PAGE_SIZE, &pgdir->db_dma);
307	if (!pgdir->db_page) {
308		free(pgdir);
309		return NULL;
310	}
311
312	return pgdir;
313}
314
315static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir,
316		struct mlx4_db *db, int order) {
317	int o;
318	int i;
319
320	for (o = order; o <= 1; ++o) {
321		i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o);
322		/*printf("ffb: %d\n", i);*/
323		if (i < MLX4_DB_PER_PAGE >> o)
324			goto found;
325	}
326
327	return -ENOMEM;
328
329	found: clear_bit(i, pgdir->bits[o]);
330
331	i <<= o;
332
333	if (o > order)
334		set_bit(i ^ 1, pgdir->bits[order]);
335
336	db->u.pgdir = pgdir;
337	db->index = i;
338	db->db = pgdir->db_page + db->index;
339	db->dma = pgdir->db_dma + db->index * 4;
340	db->order = order;
341
342	return 0;
343}
344
345int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) {
346	struct mlx4_priv *priv = mlx4_priv(dev);
347	struct mlx4_db_pgdir *pgdir;
348	int ret = 0;
349
350	/*mutex_lock(&priv->pgdir_mutex);*/
351
352	list_for_each_entry(pgdir, &priv->pgdir_list, list)
353		if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
354			goto out;
355
356	pgdir = mlx4_alloc_db_pgdir();
357	if (!pgdir) {
358		ret = -ENOMEM;
359		goto out;
360	}
361
362	list_add(&pgdir->list, &priv->pgdir_list);
363
364	/*This should never fail -- we just allocated an empty page:*/
365	assert(!mlx4_alloc_db_from_pgdir(pgdir, db, order));
366
367	out: /*mutex_unlock(&priv->pgdir_mutex);*/
368
369	return ret;
370}
371/*
372 EXPORT_SYMBOL_GPL(mlx4_db_alloc);
373
374 void mlx4_db_free(struct mlx4_priv *priv, struct mlx4_db *db)
375 {
376 struct mlx4_priv *priv = mlx4_priv(dev);
377 int o;
378 int i;
379
380 mutex_lock(&priv->pgdir_mutex);
381
382 o = db->order;
383 i = db->index;
384
385 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
386 clear_bit(i ^ 1, db->u.pgdir->order0);
387 ++o;
388 }
389 i >>= o;
390 set_bit(i, db->u.pgdir->bits[o]);
391
392 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {
393 dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
394 db->u.pgdir->db_page, db->u.pgdir->db_dma);
395 list_del(&db->u.pgdir->list);
396 kfree(db->u.pgdir);
397 }
398
399 mutex_unlock(&priv->pgdir_mutex);
400 }
401 EXPORT_SYMBOL_GPL(mlx4_db_free);
402 */
403int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
404		int size, int max_direct) {
405	int err;
406
407	err = mlx4_db_alloc(dev, &wqres->db, 1);
408	if (err)
409		return err;
410
411	*wqres->db.db = 0;
412
413	err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf);
414	if (err)
415		goto err_db;
416
417	err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift,
418			&wqres->mtt);
419	if (err)
420		goto err_buf;
421
422	err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf);
423	if (err)
424		goto err_mtt;
425
426	return 0;
427
428	err_mtt: /*mlx4_mtt_cleanup(dev, &wqres->mtt);*/
429	err_buf: /*mlx4_buf_free(dev, size, &wqres->buf);*/
430	err_db: /*mlx4_db_free(dev, &wqres->db);*/
431
432	return err;
433}
434/*
435 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res);
436
437 void mlx4_free_hwq_res(struct mlx4_priv *priv, struct mlx4_hwq_resources *wqres,
438 int size)
439 {
440 mlx4_mtt_cleanup(dev, &wqres->mtt);
441 mlx4_buf_free(dev, size, &wqres->buf);
442 mlx4_db_free(dev, &wqres->db);
443 }
444 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res);*/
445