1219820Sjeff/*
2219820Sjeff * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3219820Sjeff * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4219820Sjeff * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
5219820Sjeff * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
6219820Sjeff * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
7219820Sjeff *
8219820Sjeff * This software is available to you under a choice of one of two
9219820Sjeff * licenses.  You may choose to be licensed under the terms of the GNU
10219820Sjeff * General Public License (GPL) Version 2, available from the file
11219820Sjeff * COPYING in the main directory of this source tree, or the
12219820Sjeff * OpenIB.org BSD license below:
13219820Sjeff *
14219820Sjeff *     Redistribution and use in source and binary forms, with or
15219820Sjeff *     without modification, are permitted provided that the following
16219820Sjeff *     conditions are met:
17219820Sjeff *
18219820Sjeff *      - Redistributions of source code must retain the above
19219820Sjeff *        copyright notice, this list of conditions and the following
20219820Sjeff *        disclaimer.
21219820Sjeff *
22219820Sjeff *      - Redistributions in binary form must reproduce the above
23219820Sjeff *        copyright notice, this list of conditions and the following
24219820Sjeff *        disclaimer in the documentation and/or other materials
25219820Sjeff *        provided with the distribution.
26219820Sjeff *
27219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34219820Sjeff * SOFTWARE.
35219820Sjeff */
36219820Sjeff
37219820Sjeff#include <linux/init.h>
38219820Sjeff#include <linux/hardirq.h>
39219820Sjeff
40219820Sjeff#include <linux/mlx4/cmd.h>
41219820Sjeff#include <linux/mlx4/cq.h>
42219820Sjeff
43219820Sjeff#include "mlx4.h"
44219820Sjeff#include "icm.h"
45219820Sjeff
46219820Sjeff#define MLX4_CQ_STATUS_OK		( 0 << 28)
47219820Sjeff#define MLX4_CQ_STATUS_OVERFLOW		( 9 << 28)
48219820Sjeff#define MLX4_CQ_STATUS_WRITE_FAIL	(10 << 28)
49219820Sjeff#define MLX4_CQ_FLAG_CC			( 1 << 18)
50219820Sjeff#define MLX4_CQ_FLAG_OI			( 1 << 17)
51219820Sjeff#define MLX4_CQ_STATE_ARMED		( 9 <<  8)
52219820Sjeff#define MLX4_CQ_STATE_ARMED_SOL		( 6 <<  8)
53219820Sjeff#define MLX4_EQ_STATE_FIRED		(10 <<  8)
54219820Sjeff
55219820Sjeffvoid mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn)
56219820Sjeff{
57255932Salfred	struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
58219820Sjeff	struct mlx4_cq *cq;
59219820Sjeff
60255932Salfred	spin_lock(&cq_table->lock);
61219820Sjeff	cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree,
62219820Sjeff			       cqn & (dev->caps.num_cqs - 1));
63255932Salfred	if (cq)
64255932Salfred		atomic_inc(&cq->refcount);
65255932Salfred	spin_unlock(&cq_table->lock);
66255932Salfred
67219820Sjeff	if (!cq) {
68219820Sjeff		mlx4_dbg(dev, "Completion event for bogus CQ %08x\n", cqn);
69219820Sjeff		return;
70219820Sjeff	}
71219820Sjeff
72219820Sjeff	++cq->arm_sn;
73219820Sjeff
74219820Sjeff	cq->comp(cq);
75255932Salfred
76255932Salfred	if (atomic_dec_and_test(&cq->refcount))
77255932Salfred		complete(&cq->free);
78219820Sjeff}
79219820Sjeff
80219820Sjeffvoid mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type)
81219820Sjeff{
82219820Sjeff	struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
83219820Sjeff	struct mlx4_cq *cq;
84219820Sjeff
85219820Sjeff	spin_lock(&cq_table->lock);
86219820Sjeff
87219820Sjeff	cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1));
88219820Sjeff	if (cq)
89219820Sjeff		atomic_inc(&cq->refcount);
90219820Sjeff
91219820Sjeff	spin_unlock(&cq_table->lock);
92219820Sjeff
93219820Sjeff	if (!cq) {
94219820Sjeff		mlx4_warn(dev, "Async event for bogus CQ %08x\n", cqn);
95219820Sjeff		return;
96219820Sjeff	}
97219820Sjeff
98219820Sjeff	cq->event(cq, event_type);
99219820Sjeff
100219820Sjeff	if (atomic_dec_and_test(&cq->refcount))
101219820Sjeff		complete(&cq->free);
102219820Sjeff}
103219820Sjeff
104219820Sjeffstatic int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
105219820Sjeff			 int cq_num)
106219820Sjeff{
107255932Salfred	return mlx4_cmd(dev, mailbox->dma, cq_num, 0,
108255932Salfred			MLX4_CMD_SW2HW_CQ, MLX4_CMD_TIME_CLASS_A,
109255932Salfred			MLX4_CMD_WRAPPED);
110219820Sjeff}
111219820Sjeff
112219820Sjeffstatic int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
113219820Sjeff			 int cq_num, u32 opmod)
114219820Sjeff{
115219820Sjeff	return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ,
116255932Salfred			MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
117219820Sjeff}
118219820Sjeff
119219820Sjeffstatic int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
120219820Sjeff			 int cq_num)
121219820Sjeff{
122255932Salfred	return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0,
123255932Salfred			    cq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ,
124255932Salfred			    MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
125219820Sjeff}
126219820Sjeff
127219820Sjeffint mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq,
128219820Sjeff		   u16 count, u16 period)
129219820Sjeff{
130219820Sjeff	struct mlx4_cmd_mailbox *mailbox;
131219820Sjeff	struct mlx4_cq_context *cq_context;
132219820Sjeff	int err;
133219820Sjeff
134219820Sjeff	mailbox = mlx4_alloc_cmd_mailbox(dev);
135219820Sjeff	if (IS_ERR(mailbox))
136219820Sjeff		return PTR_ERR(mailbox);
137219820Sjeff
138219820Sjeff	cq_context = mailbox->buf;
139219820Sjeff	memset(cq_context, 0, sizeof *cq_context);
140219820Sjeff
141219820Sjeff	cq_context->cq_max_count = cpu_to_be16(count);
142219820Sjeff	cq_context->cq_period    = cpu_to_be16(period);
143219820Sjeff
144219820Sjeff	err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1);
145219820Sjeff
146219820Sjeff	mlx4_free_cmd_mailbox(dev, mailbox);
147219820Sjeff	return err;
148219820Sjeff}
149219820SjeffEXPORT_SYMBOL_GPL(mlx4_cq_modify);
150219820Sjeff
151219820Sjeffint mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,
152219820Sjeff		   int entries, struct mlx4_mtt *mtt)
153219820Sjeff{
154219820Sjeff	struct mlx4_cmd_mailbox *mailbox;
155219820Sjeff	struct mlx4_cq_context *cq_context;
156219820Sjeff	u64 mtt_addr;
157219820Sjeff	int err;
158219820Sjeff
159219820Sjeff	mailbox = mlx4_alloc_cmd_mailbox(dev);
160219820Sjeff	if (IS_ERR(mailbox))
161219820Sjeff		return PTR_ERR(mailbox);
162219820Sjeff
163219820Sjeff	cq_context = mailbox->buf;
164219820Sjeff	memset(cq_context, 0, sizeof *cq_context);
165219820Sjeff
166219820Sjeff	cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24);
167219820Sjeff	cq_context->log_page_size   = mtt->page_shift - 12;
168219820Sjeff	mtt_addr = mlx4_mtt_addr(dev, mtt);
169219820Sjeff	cq_context->mtt_base_addr_h = mtt_addr >> 32;
170219820Sjeff	cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
171219820Sjeff
172219820Sjeff	err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0);
173219820Sjeff
174219820Sjeff	mlx4_free_cmd_mailbox(dev, mailbox);
175219820Sjeff	return err;
176219820Sjeff}
177219820SjeffEXPORT_SYMBOL_GPL(mlx4_cq_resize);
178219820Sjeff
179255932Salfredint mlx4_cq_ignore_overrun(struct mlx4_dev *dev, struct mlx4_cq *cq)
180219820Sjeff{
181255932Salfred	struct mlx4_cmd_mailbox *mailbox;
182255932Salfred	struct mlx4_cq_context *cq_context;
183255932Salfred	int err;
184219820Sjeff
185255932Salfred	mailbox = mlx4_alloc_cmd_mailbox(dev);
186255932Salfred	if (IS_ERR(mailbox))
187255932Salfred		return PTR_ERR(mailbox);
188255932Salfred
189255932Salfred	cq_context = mailbox->buf;
190255932Salfred	memset(cq_context, 0, sizeof *cq_context);
191255932Salfred
192255932Salfred	cq_context->flags |= cpu_to_be32(MLX4_CQ_FLAG_OI);
193255932Salfred
194255932Salfred	err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 3);
195255932Salfred
196255932Salfred	mlx4_free_cmd_mailbox(dev, mailbox);
197255932Salfred	return err;
198255932Salfred}
199255932SalfredEXPORT_SYMBOL_GPL(mlx4_cq_ignore_overrun);
200255932Salfred
201255932Salfredint __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
202255932Salfred{
203255932Salfred	struct mlx4_priv *priv = mlx4_priv(dev);
204255932Salfred	struct mlx4_cq_table *cq_table = &priv->cq_table;
205255932Salfred	int err;
206255932Salfred
207255932Salfred	*cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
208255932Salfred	if (*cqn == -1)
209255932Salfred		return -ENOMEM;
210255932Salfred
211255932Salfred	err = mlx4_table_get(dev, &cq_table->table, *cqn);
212255932Salfred	if (err)
213255932Salfred		goto err_out;
214255932Salfred
215255932Salfred	err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn);
216255932Salfred	if (err)
217255932Salfred		goto err_put;
218255932Salfred	return 0;
219255932Salfred
220255932Salfrederr_put:
221255932Salfred	mlx4_table_put(dev, &cq_table->table, *cqn);
222255932Salfred
223255932Salfrederr_out:
224255932Salfred	mlx4_bitmap_free(&cq_table->bitmap, *cqn);
225255932Salfred	return err;
226255932Salfred}
227255932Salfred
228255932Salfredstatic int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
229255932Salfred{
230255932Salfred	u64 out_param;
231255932Salfred	int err;
232255932Salfred
233255932Salfred	if (mlx4_is_mfunc(dev)) {
234255932Salfred		err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ,
235255932Salfred				   RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,
236255932Salfred				   MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
237255932Salfred		if (err)
238255932Salfred			return err;
239255932Salfred		else {
240255932Salfred			*cqn = get_param_l(&out_param);
241255932Salfred			return 0;
242219820Sjeff		}
243219820Sjeff	}
244255932Salfred	return __mlx4_cq_alloc_icm(dev, cqn);
245255932Salfred}
246219820Sjeff
247255932Salfredvoid __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
248255932Salfred{
249255932Salfred	struct mlx4_priv *priv = mlx4_priv(dev);
250255932Salfred	struct mlx4_cq_table *cq_table = &priv->cq_table;
251255932Salfred
252255932Salfred	mlx4_table_put(dev, &cq_table->cmpt_table, cqn);
253255932Salfred	mlx4_table_put(dev, &cq_table->table, cqn);
254255932Salfred	mlx4_bitmap_free(&cq_table->bitmap, cqn);
255219820Sjeff}
256219820Sjeff
257255932Salfredstatic void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
258219820Sjeff{
259255932Salfred	u64 in_param = 0;
260255932Salfred	int err;
261255932Salfred
262255932Salfred	if (mlx4_is_mfunc(dev)) {
263255932Salfred		set_param_l(&in_param, cqn);
264255932Salfred		err = mlx4_cmd(dev, in_param, RES_CQ, RES_OP_RESERVE_AND_MAP,
265255932Salfred			       MLX4_CMD_FREE_RES,
266255932Salfred			       MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
267255932Salfred		if (err)
268255932Salfred			mlx4_warn(dev, "Failed freeing cq:%d\n", cqn);
269255932Salfred	} else
270255932Salfred		__mlx4_cq_free_icm(dev, cqn);
271255932Salfred}
272255932Salfred
273255932Salfredstatic int mlx4_find_least_loaded_vector(struct mlx4_priv *priv)
274255932Salfred{
275255932Salfred        int i;
276255932Salfred        int index = 0;
277255932Salfred        int min = priv->eq_table.eq[0].load;
278255932Salfred
279255932Salfred        for (i = 1; i < priv->dev.caps.num_comp_vectors; i++) {
280255932Salfred                if (priv->eq_table.eq[i].load < min) {
281255932Salfred                        index = i;
282255932Salfred                        min = priv->eq_table.eq[i].load;
283255932Salfred                }
284255932Salfred        }
285255932Salfred
286255932Salfred        return index;
287255932Salfred}
288255932Salfred
289255932Salfred
290255932Salfredint mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
291255932Salfred		  struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec,
292255932Salfred		  struct mlx4_cq *cq, unsigned vector, int collapsed,
293255932Salfred		  int timestamp_en)
294255932Salfred{
295219820Sjeff	struct mlx4_priv *priv = mlx4_priv(dev);
296219820Sjeff	struct mlx4_cq_table *cq_table = &priv->cq_table;
297219820Sjeff	struct mlx4_cmd_mailbox *mailbox;
298219820Sjeff	struct mlx4_cq_context *cq_context;
299219820Sjeff	u64 mtt_addr;
300219820Sjeff	int err;
301219820Sjeff
302255932Salfred        cq->vector = (vector == MLX4_LEAST_ATTACHED_VECTOR) ?
303255932Salfred                mlx4_find_least_loaded_vector(priv) : vector;
304219820Sjeff
305255932Salfred	if (cq->vector > dev->caps.num_comp_vectors + dev->caps.comp_pool) {
306219820Sjeff		return -EINVAL;
307255932Salfred        }
308219820Sjeff
309255932Salfred	err = mlx4_cq_alloc_icm(dev, &cq->cqn);
310255932Salfred	if (err) {
311255932Salfred		return err;
312255932Salfred        }
313219820Sjeff
314219820Sjeff	spin_lock_irq(&cq_table->lock);
315219820Sjeff	err = radix_tree_insert(&cq_table->tree, cq->cqn, cq);
316219820Sjeff	spin_unlock_irq(&cq_table->lock);
317255932Salfred	if (err){
318255932Salfred		goto err_icm;
319255932Salfred        }
320219820Sjeff
321219820Sjeff	mailbox = mlx4_alloc_cmd_mailbox(dev);
322219820Sjeff	if (IS_ERR(mailbox)) {
323219820Sjeff		err = PTR_ERR(mailbox);
324219820Sjeff		goto err_radix;
325219820Sjeff	}
326219820Sjeff
327219820Sjeff	cq_context = mailbox->buf;
328219820Sjeff	memset(cq_context, 0, sizeof *cq_context);
329219820Sjeff
330219820Sjeff	cq_context->flags	    = cpu_to_be32(!!collapsed << 18);
331255932Salfred	if (timestamp_en)
332255932Salfred		cq_context->flags  |= cpu_to_be32(1 << 19);
333255932Salfred
334219820Sjeff	cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index);
335219820Sjeff	cq_context->comp_eqn	    = priv->eq_table.eq[cq->vector].eqn;
336219820Sjeff	cq_context->log_page_size   = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
337219820Sjeff
338219820Sjeff	mtt_addr = mlx4_mtt_addr(dev, mtt);
339219820Sjeff	cq_context->mtt_base_addr_h = mtt_addr >> 32;
340219820Sjeff	cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
341219820Sjeff	cq_context->db_rec_addr     = cpu_to_be64(db_rec);
342219820Sjeff
343219820Sjeff	err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn);
344219820Sjeff	mlx4_free_cmd_mailbox(dev, mailbox);
345219820Sjeff	if (err)
346219820Sjeff		goto err_radix;
347219820Sjeff
348255932Salfred        priv->eq_table.eq[cq->vector].load++;
349219820Sjeff	cq->cons_index = 0;
350219820Sjeff	cq->arm_sn     = 1;
351219820Sjeff	cq->uar        = uar;
352219820Sjeff	atomic_set(&cq->refcount, 1);
353219820Sjeff	init_completion(&cq->free);
354219820Sjeff
355255932Salfred	cq->eqn = priv->eq_table.eq[cq->vector].eqn;
356255932Salfred	cq->irq = priv->eq_table.eq[cq->vector].irq;
357255932Salfred
358219820Sjeff	return 0;
359219820Sjeff
360219820Sjefferr_radix:
361219820Sjeff	spin_lock_irq(&cq_table->lock);
362219820Sjeff	radix_tree_delete(&cq_table->tree, cq->cqn);
363219820Sjeff	spin_unlock_irq(&cq_table->lock);
364219820Sjeff
365255932Salfrederr_icm:
366255932Salfred	mlx4_cq_free_icm(dev, cq->cqn);
367219820Sjeff
368219820Sjeff	return err;
369219820Sjeff}
370219820SjeffEXPORT_SYMBOL_GPL(mlx4_cq_alloc);
371219820Sjeff
372219820Sjeffvoid mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
373219820Sjeff{
374219820Sjeff	struct mlx4_priv *priv = mlx4_priv(dev);
375219820Sjeff	struct mlx4_cq_table *cq_table = &priv->cq_table;
376219820Sjeff	int err;
377219820Sjeff
378219820Sjeff	err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn);
379219820Sjeff	if (err)
380219820Sjeff		mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn);
381219820Sjeff
382255932Salfred
383255932Salfred        priv->eq_table.eq[cq->vector].load--;
384219820Sjeff	synchronize_irq(priv->eq_table.eq[cq->vector].irq);
385219820Sjeff
386219820Sjeff	spin_lock_irq(&cq_table->lock);
387219820Sjeff	radix_tree_delete(&cq_table->tree, cq->cqn);
388219820Sjeff	spin_unlock_irq(&cq_table->lock);
389219820Sjeff
390219820Sjeff	if (atomic_dec_and_test(&cq->refcount))
391219820Sjeff		complete(&cq->free);
392219820Sjeff	wait_for_completion(&cq->free);
393219820Sjeff
394255932Salfred	mlx4_cq_free_icm(dev, cq->cqn);
395219820Sjeff}
396219820SjeffEXPORT_SYMBOL_GPL(mlx4_cq_free);
397219820Sjeff
398219820Sjeffint mlx4_init_cq_table(struct mlx4_dev *dev)
399219820Sjeff{
400219820Sjeff	struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
401219820Sjeff	int err;
402219820Sjeff
403219820Sjeff	spin_lock_init(&cq_table->lock);
404219820Sjeff	INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
405255932Salfred	if (mlx4_is_slave(dev))
406255932Salfred		return 0;
407219820Sjeff
408219820Sjeff	err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs,
409219820Sjeff			       dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0);
410219820Sjeff	if (err)
411219820Sjeff		return err;
412219820Sjeff
413219820Sjeff	return 0;
414219820Sjeff}
415219820Sjeff
416219820Sjeffvoid mlx4_cleanup_cq_table(struct mlx4_dev *dev)
417219820Sjeff{
418255932Salfred	if (mlx4_is_slave(dev))
419255932Salfred		return;
420219820Sjeff	/* Nothing to do to clean up radix_tree */
421219820Sjeff	mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap);
422219820Sjeff}
423