mlx5_en_main.c revision 306233
1117610Sdes/*-
2117610Sdes * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
3255376Sdes *
4117610Sdes * Redistribution and use in source and binary forms, with or without
5255376Sdes * modification, are permitted provided that the following conditions
6117610Sdes * are met:
7255376Sdes * 1. Redistributions of source code must retain the above copyright
8255376Sdes *    notice, this list of conditions and the following disclaimer.
9255376Sdes * 2. Redistributions in binary form must reproduce the above copyright
10117610Sdes *    notice, this list of conditions and the following disclaimer in the
11117610Sdes *    documentation and/or other materials provided with the distribution.
12255376Sdes *
13255376Sdes * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14255376Sdes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15255376Sdes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16117610Sdes * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17117610Sdes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18255376Sdes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19255376Sdes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20117610Sdes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21117610Sdes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22117610Sdes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23255376Sdes * SUCH DAMAGE.
24255376Sdes *
25255376Sdes * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_en/mlx5_en_main.c 306233 2016-09-23 08:17:51Z hselasky $
26117610Sdes */
27255376Sdes
28255376Sdes#include "en.h"
29117610Sdes
30117610Sdes#include <sys/sockio.h>
31117610Sdes#include <machine/atomic.h>
32117610Sdes
33117610Sdes#define	ETH_DRIVER_VERSION	"3.1.0-dev"
34117610Sdeschar mlx5e_version[] = "Mellanox Ethernet driver"
35255376Sdes    " (" ETH_DRIVER_VERSION ")";
36255376Sdes
37255376Sdesstruct mlx5e_rq_param {
38117610Sdes	u32	rqc [MLX5_ST_SZ_DW(rqc)];
39117610Sdes	struct mlx5_wq_param wq;
40117610Sdes};
41117610Sdes
42117610Sdesstruct mlx5e_sq_param {
43117610Sdes	u32	sqc [MLX5_ST_SZ_DW(sqc)];
44117610Sdes	struct mlx5_wq_param wq;
45117610Sdes};
46117610Sdes
47117610Sdesstruct mlx5e_cq_param {
48117610Sdes	u32	cqc [MLX5_ST_SZ_DW(cqc)];
49117610Sdes	struct mlx5_wq_param wq;
50117610Sdes	u16	eq_ix;
51117610Sdes};
52117610Sdes
53117610Sdesstruct mlx5e_channel_param {
54117610Sdes	struct mlx5e_rq_param rq;
55117610Sdes	struct mlx5e_sq_param sq;
56117610Sdes	struct mlx5e_cq_param rx_cq;
57117610Sdes	struct mlx5e_cq_param tx_cq;
58117610Sdes};
59117610Sdes
60117610Sdesstatic const struct {
61117610Sdes	u32	subtype;
62117610Sdes	u64	baudrate;
63117610Sdes}	mlx5e_mode_table[MLX5E_LINK_MODES_NUMBER] = {
64117610Sdes
65117610Sdes	[MLX5E_1000BASE_CX_SGMII] = {
66117610Sdes		.subtype = IFM_1000_CX_SGMII,
67117610Sdes		.baudrate = IF_Mbps(1000ULL),
68117610Sdes	},
69117610Sdes	[MLX5E_1000BASE_KX] = {
70117610Sdes		.subtype = IFM_1000_KX,
71255376Sdes		.baudrate = IF_Mbps(1000ULL),
72117610Sdes	},
73117610Sdes	[MLX5E_10GBASE_CX4] = {
74117610Sdes		.subtype = IFM_10G_CX4,
75117610Sdes		.baudrate = IF_Gbps(10ULL),
76117610Sdes	},
77117610Sdes	[MLX5E_10GBASE_KX4] = {
78117610Sdes		.subtype = IFM_10G_KX4,
79117610Sdes		.baudrate = IF_Gbps(10ULL),
80117610Sdes	},
81117610Sdes	[MLX5E_10GBASE_KR] = {
82117610Sdes		.subtype = IFM_10G_KR,
83255376Sdes		.baudrate = IF_Gbps(10ULL),
84117610Sdes	},
85255376Sdes	[MLX5E_20GBASE_KR2] = {
86117610Sdes		.subtype = IFM_20G_KR2,
87255376Sdes		.baudrate = IF_Gbps(20ULL),
88117610Sdes	},
89117610Sdes	[MLX5E_40GBASE_CR4] = {
90117610Sdes		.subtype = IFM_40G_CR4,
91117610Sdes		.baudrate = IF_Gbps(40ULL),
92117610Sdes	},
93117610Sdes	[MLX5E_40GBASE_KR4] = {
94117610Sdes		.subtype = IFM_40G_KR4,
95117610Sdes		.baudrate = IF_Gbps(40ULL),
96117610Sdes	},
97117610Sdes	[MLX5E_56GBASE_R4] = {
98117610Sdes		.subtype = IFM_56G_R4,
99255376Sdes		.baudrate = IF_Gbps(56ULL),
100117610Sdes	},
101117610Sdes	[MLX5E_10GBASE_CR] = {
102117610Sdes		.subtype = IFM_10G_CR1,
103117610Sdes		.baudrate = IF_Gbps(10ULL),
104117610Sdes	},
105117610Sdes	[MLX5E_10GBASE_SR] = {
106117610Sdes		.subtype = IFM_10G_SR,
107117610Sdes		.baudrate = IF_Gbps(10ULL),
108117610Sdes	},
109117610Sdes	[MLX5E_10GBASE_LR] = {
110117610Sdes		.subtype = IFM_10G_LR,
111117610Sdes		.baudrate = IF_Gbps(10ULL),
112117610Sdes	},
113117610Sdes	[MLX5E_40GBASE_SR4] = {
114117610Sdes		.subtype = IFM_40G_SR4,
115117610Sdes		.baudrate = IF_Gbps(40ULL),
116117610Sdes	},
117117610Sdes	[MLX5E_40GBASE_LR4] = {
118255376Sdes		.subtype = IFM_40G_LR4,
119255376Sdes		.baudrate = IF_Gbps(40ULL),
120255376Sdes	},
121255376Sdes	[MLX5E_100GBASE_CR4] = {
122255376Sdes		.subtype = IFM_100G_CR4,
123117610Sdes		.baudrate = IF_Gbps(100ULL),
124117610Sdes	},
125117610Sdes	[MLX5E_100GBASE_SR4] = {
126255376Sdes		.subtype = IFM_100G_SR4,
127255376Sdes		.baudrate = IF_Gbps(100ULL),
128255376Sdes	},
129255376Sdes	[MLX5E_100GBASE_KR4] = {
130117610Sdes		.subtype = IFM_100G_KR4,
131117610Sdes		.baudrate = IF_Gbps(100ULL),
132117610Sdes	},
133117610Sdes	[MLX5E_100GBASE_LR4] = {
134117610Sdes		.subtype = IFM_100G_LR4,
135117610Sdes		.baudrate = IF_Gbps(100ULL),
136117610Sdes	},
137117610Sdes	[MLX5E_100BASE_TX] = {
138117610Sdes		.subtype = IFM_100_TX,
139117610Sdes		.baudrate = IF_Mbps(100ULL),
140117610Sdes	},
141117610Sdes	[MLX5E_100BASE_T] = {
142117610Sdes		.subtype = IFM_100_T,
143117610Sdes		.baudrate = IF_Mbps(100ULL),
144117610Sdes	},
145117610Sdes	[MLX5E_10GBASE_T] = {
146117610Sdes		.subtype = IFM_10G_T,
147117610Sdes		.baudrate = IF_Gbps(10ULL),
148117610Sdes	},
149117610Sdes	[MLX5E_25GBASE_CR] = {
150117610Sdes		.subtype = IFM_25G_CR,
151117610Sdes		.baudrate = IF_Gbps(25ULL),
152255376Sdes	},
153117610Sdes	[MLX5E_25GBASE_KR] = {
154117610Sdes		.subtype = IFM_25G_KR,
155117610Sdes		.baudrate = IF_Gbps(25ULL),
156255376Sdes	},
157255376Sdes	[MLX5E_25GBASE_SR] = {
158255376Sdes		.subtype = IFM_25G_SR,
159117610Sdes		.baudrate = IF_Gbps(25ULL),
160117610Sdes	},
161117610Sdes	[MLX5E_50GBASE_CR2] = {
162117610Sdes		.subtype = IFM_50G_CR2,
163117610Sdes		.baudrate = IF_Gbps(50ULL),
164117610Sdes	},
165117610Sdes	[MLX5E_50GBASE_KR2] = {
166117610Sdes		.subtype = IFM_50G_KR2,
167117610Sdes		.baudrate = IF_Gbps(50ULL),
168117610Sdes	},
169117610Sdes};
170117610Sdes
171117610SdesMALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
172117610Sdes
173255376Sdesstatic void
174255376Sdesmlx5e_update_carrier(struct mlx5e_priv *priv)
175117610Sdes{
176255376Sdes	struct mlx5_core_dev *mdev = priv->mdev;
177117610Sdes	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
178117610Sdes	u32 eth_proto_oper;
179117610Sdes	int error;
180255376Sdes	u8 port_state;
181255376Sdes	u8 i;
182255376Sdes
183255376Sdes	port_state = mlx5_query_vport_state(mdev,
184117610Sdes	    MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
185117610Sdes
186117610Sdes	if (port_state == VPORT_STATE_UP) {
187117610Sdes		priv->media_status_last |= IFM_ACTIVE;
188117610Sdes	} else {
189117610Sdes		priv->media_status_last &= ~IFM_ACTIVE;
190117610Sdes		priv->media_active_last = IFM_ETHER;
191117610Sdes		if_link_state_change(priv->ifp, LINK_STATE_DOWN);
192117610Sdes		return;
193117610Sdes	}
194117610Sdes
195117610Sdes	error = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN);
196117610Sdes	if (error) {
197117610Sdes		priv->media_active_last = IFM_ETHER;
198117610Sdes		priv->ifp->if_baudrate = 1;
199117610Sdes		if_printf(priv->ifp, "%s: query port ptys failed: 0x%x\n",
200255376Sdes		    __func__, error);
201255376Sdes		return;
202255376Sdes	}
203255376Sdes	eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper);
204117610Sdes
205117610Sdes	for (i = 0; i != MLX5E_LINK_MODES_NUMBER; i++) {
206117610Sdes		if (mlx5e_mode_table[i].baudrate == 0)
207117610Sdes			continue;
208117610Sdes		if (MLX5E_PROT_MASK(i) & eth_proto_oper) {
209117610Sdes			priv->ifp->if_baudrate =
210117610Sdes			    mlx5e_mode_table[i].baudrate;
211117610Sdes			priv->media_active_last =
212117610Sdes			    mlx5e_mode_table[i].subtype | IFM_ETHER | IFM_FDX;
213117610Sdes		}
214117610Sdes	}
215117610Sdes	if_link_state_change(priv->ifp, LINK_STATE_UP);
216117610Sdes}
217117610Sdes
218117610Sdesstatic void
219117610Sdesmlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr)
220117610Sdes{
221255376Sdes	struct mlx5e_priv *priv = dev->if_softc;
222255376Sdes
223255376Sdes	ifmr->ifm_status = priv->media_status_last;
224255376Sdes	ifmr->ifm_active = priv->media_active_last |
225255376Sdes	    (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
226255376Sdes	    (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
227117610Sdes
228117610Sdes}
229117610Sdes
230117610Sdesstatic u32
231117610Sdesmlx5e_find_link_mode(u32 subtype)
232117610Sdes{
233117610Sdes	u32 i;
234117610Sdes	u32 link_mode = 0;
235117610Sdes
236117610Sdes	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
237117610Sdes		if (mlx5e_mode_table[i].baudrate == 0)
238117610Sdes			continue;
239117610Sdes		if (mlx5e_mode_table[i].subtype == subtype)
240117610Sdes			link_mode |= MLX5E_PROT_MASK(i);
241117610Sdes	}
242117610Sdes
243117610Sdes	return (link_mode);
244117610Sdes}
245117610Sdes
246117610Sdesstatic int
247117610Sdesmlx5e_media_change(struct ifnet *dev)
248117610Sdes{
249117610Sdes	struct mlx5e_priv *priv = dev->if_softc;
250117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
251255376Sdes	u32 eth_proto_cap;
252117610Sdes	u32 link_mode;
253117610Sdes	int was_opened;
254255376Sdes	int locked;
255255376Sdes	int error;
256255376Sdes
257255376Sdes	locked = PRIV_LOCKED(priv);
258255376Sdes	if (!locked)
259255376Sdes		PRIV_LOCK(priv);
260117610Sdes
261117610Sdes	if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
262255376Sdes		error = EINVAL;
263255376Sdes		goto done;
264117610Sdes	}
265255376Sdes	link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media));
266117610Sdes
267255376Sdes	/* query supported capabilities */
268255376Sdes	error = mlx5_query_port_proto_cap(mdev, &eth_proto_cap, MLX5_PTYS_EN);
269255376Sdes	if (error != 0) {
270255376Sdes		if_printf(dev, "Query port media capability failed\n");
271255376Sdes		goto done;
272117610Sdes	}
273117610Sdes	/* check for autoselect */
274117610Sdes	if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
275255376Sdes		link_mode = eth_proto_cap;
276255376Sdes		if (link_mode == 0) {
277255376Sdes			if_printf(dev, "Port media capability is zero\n");
278117610Sdes			error = EINVAL;
279117610Sdes			goto done;
280117610Sdes		}
281117610Sdes	} else {
282255376Sdes		link_mode = link_mode & eth_proto_cap;
283117610Sdes		if (link_mode == 0) {
284117610Sdes			if_printf(dev, "Not supported link mode requested\n");
285117610Sdes			error = EINVAL;
286255376Sdes			goto done;
287117610Sdes		}
288117610Sdes	}
289255376Sdes	/* update pauseframe control bits */
290117610Sdes	priv->params.rx_pauseframe_control =
291117610Sdes	    (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
292255376Sdes	priv->params.tx_pauseframe_control =
293255376Sdes	    (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
294117610Sdes
295255376Sdes	/* check if device is opened */
296255376Sdes	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
297117610Sdes
298255376Sdes	/* reconfigure the hardware */
299255376Sdes	mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
300117610Sdes	mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN);
301255376Sdes	mlx5_set_port_pause(mdev, 1,
302117610Sdes	    priv->params.rx_pauseframe_control,
303255376Sdes	    priv->params.tx_pauseframe_control);
304255376Sdes	if (was_opened)
305255376Sdes		mlx5_set_port_status(mdev, MLX5_PORT_UP);
306117610Sdes
307255376Sdesdone:
308255376Sdes	if (!locked)
309255376Sdes		PRIV_UNLOCK(priv);
310255376Sdes	return (error);
311255376Sdes}
312255376Sdes
313117610Sdesstatic void
314255376Sdesmlx5e_update_carrier_work(struct work_struct *work)
315255376Sdes{
316117610Sdes	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
317117610Sdes	    update_carrier_work);
318255376Sdes
319255376Sdes	PRIV_LOCK(priv);
320255376Sdes	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
321255376Sdes		mlx5e_update_carrier(priv);
322255376Sdes	PRIV_UNLOCK(priv);
323255376Sdes}
324255376Sdes
325255376Sdesstatic void
326255376Sdesmlx5e_update_pport_counters(struct mlx5e_priv *priv)
327255376Sdes{
328117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
329117610Sdes	struct mlx5e_pport_stats *s = &priv->stats.pport;
330117610Sdes	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
331117610Sdes	u32 *in;
332117610Sdes	u32 *out;
333255376Sdes	u64 *ptr;
334255376Sdes	unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
335255376Sdes	unsigned x;
336117610Sdes	unsigned y;
337255376Sdes
338255376Sdes	in = mlx5_vzalloc(sz);
339255376Sdes	out = mlx5_vzalloc(sz);
340255376Sdes	if (in == NULL || out == NULL)
341255376Sdes		goto free_out;
342255376Sdes
343255376Sdes	ptr = (uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
344255376Sdes
345255376Sdes	MLX5_SET(ppcnt_reg, in, local_port, 1);
346255376Sdes
347255376Sdes	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
348255376Sdes	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
349255376Sdes	for (x = y = 0; x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
350255376Sdes		s->arg[y] = be64toh(ptr[x]);
351255376Sdes
352117610Sdes	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
353117610Sdes	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
354117610Sdes	for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
355117610Sdes		s->arg[y] = be64toh(ptr[x]);
356117610Sdes	for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
357117610Sdes	    MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
358117610Sdes		s_debug->arg[y] = be64toh(ptr[x]);
359117610Sdes
360117610Sdes	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
361117610Sdes	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
362117610Sdes	for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
363117610Sdes		s_debug->arg[y] = be64toh(ptr[x]);
364117610Sdes
365117610Sdes	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
366255376Sdes	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
367117610Sdes	for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
368117610Sdes		s_debug->arg[y] = be64toh(ptr[x]);
369255376Sdesfree_out:
370117610Sdes	kvfree(in);
371255376Sdes	kvfree(out);
372255376Sdes}
373255376Sdes
374255376Sdesstatic void
375255376Sdesmlx5e_update_stats_work(struct work_struct *work)
376117610Sdes{
377117610Sdes	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
378255376Sdes	    update_stats_work);
379117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
380117610Sdes	struct mlx5e_vport_stats *s = &priv->stats.vport;
381255376Sdes	struct mlx5e_rq_stats *rq_stats;
382117610Sdes	struct mlx5e_sq_stats *sq_stats;
383255376Sdes	struct buf_ring *sq_br;
384255376Sdes#if (__FreeBSD_version < 1100000)
385255376Sdes	struct ifnet *ifp = priv->ifp;
386255376Sdes#endif
387117610Sdes
388255376Sdes	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
389255376Sdes	u32 *out;
390117610Sdes	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
391117610Sdes	u64 tso_packets = 0;
392117610Sdes	u64 tso_bytes = 0;
393255376Sdes	u64 tx_queue_dropped = 0;
394255376Sdes	u64 tx_defragged = 0;
395255376Sdes	u64 tx_offload_none = 0;
396117610Sdes	u64 lro_packets = 0;
397117610Sdes	u64 lro_bytes = 0;
398117610Sdes	u64 sw_lro_queued = 0;
399117610Sdes	u64 sw_lro_flushed = 0;
400255376Sdes	u64 rx_csum_none = 0;
401117610Sdes	u64 rx_wqe_err = 0;
402117610Sdes	u32 rx_out_of_buffer = 0;
403117610Sdes	int i;
404255376Sdes	int j;
405117610Sdes
406117610Sdes	PRIV_LOCK(priv);
407255376Sdes	out = mlx5_vzalloc(outlen);
408117610Sdes	if (out == NULL)
409255376Sdes		goto free_out;
410255376Sdes	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
411117610Sdes		goto free_out;
412255376Sdes
413255376Sdes	/* Collect firts the SW counters and then HW for consistency */
414255376Sdes	for (i = 0; i < priv->params.num_channels; i++) {
415255376Sdes		struct mlx5e_rq *rq = &priv->channel[i]->rq;
416117610Sdes
417117610Sdes		rq_stats = &priv->channel[i]->rq.stats;
418255376Sdes
419117610Sdes		/* collect stats from LRO */
420255376Sdes		rq_stats->sw_lro_queued = rq->lro.lro_queued;
421255376Sdes		rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
422117610Sdes		sw_lro_queued += rq_stats->sw_lro_queued;
423255376Sdes		sw_lro_flushed += rq_stats->sw_lro_flushed;
424255376Sdes		lro_packets += rq_stats->lro_packets;
425255376Sdes		lro_bytes += rq_stats->lro_bytes;
426255376Sdes		rx_csum_none += rq_stats->csum_none;
427117610Sdes		rx_wqe_err += rq_stats->wqe_err;
428255376Sdes
429117610Sdes		for (j = 0; j < priv->num_tc; j++) {
430255376Sdes			sq_stats = &priv->channel[i]->sq[j].stats;
431255376Sdes			sq_br = priv->channel[i]->sq[j].br;
432255376Sdes
433117610Sdes			tso_packets += sq_stats->tso_packets;
434255376Sdes			tso_bytes += sq_stats->tso_bytes;
435255376Sdes			tx_queue_dropped += sq_stats->dropped;
436117610Sdes			tx_queue_dropped += sq_br->br_drops;
437255376Sdes			tx_defragged += sq_stats->defragged;
438117610Sdes			tx_offload_none += sq_stats->csum_offload_none;
439255376Sdes		}
440255376Sdes	}
441255376Sdes
442255376Sdes	/* update counters */
443117610Sdes	s->tso_packets = tso_packets;
444117610Sdes	s->tso_bytes = tso_bytes;
445117610Sdes	s->tx_queue_dropped = tx_queue_dropped;
446117610Sdes	s->tx_defragged = tx_defragged;
447117610Sdes	s->lro_packets = lro_packets;
448117610Sdes	s->lro_bytes = lro_bytes;
449117610Sdes	s->sw_lro_queued = sw_lro_queued;
450117610Sdes	s->sw_lro_flushed = sw_lro_flushed;
451117610Sdes	s->rx_csum_none = rx_csum_none;
452117610Sdes	s->rx_wqe_err = rx_wqe_err;
453117610Sdes
454117610Sdes	/* HW counters */
455117610Sdes	memset(in, 0, sizeof(in));
456117610Sdes
457117610Sdes	MLX5_SET(query_vport_counter_in, in, opcode,
458117610Sdes	    MLX5_CMD_OP_QUERY_VPORT_COUNTER);
459255376Sdes	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
460255376Sdes	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
461255376Sdes
462117610Sdes	memset(out, 0, outlen);
463117610Sdes
464117610Sdes	/* get number of out-of-buffer drops first */
465117610Sdes	if (mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
466117610Sdes	    &rx_out_of_buffer))
467117610Sdes		goto free_out;
468117610Sdes
469117610Sdes	/* accumulate difference into a 64-bit counter */
470117610Sdes	s->rx_out_of_buffer += (u64)(u32)(rx_out_of_buffer - s->rx_out_of_buffer_prev);
471117610Sdes	s->rx_out_of_buffer_prev = rx_out_of_buffer;
472117610Sdes
473117610Sdes	/* get port statistics */
474117610Sdes	if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen))
475117610Sdes		goto free_out;
476117610Sdes
477117610Sdes#define	MLX5_GET_CTR(out, x) \
478117610Sdes	MLX5_GET64(query_vport_counter_out, out, x)
479117610Sdes
480117610Sdes	s->rx_error_packets =
481117610Sdes	    MLX5_GET_CTR(out, received_errors.packets);
482117610Sdes	s->rx_error_bytes =
483117610Sdes	    MLX5_GET_CTR(out, received_errors.octets);
484117610Sdes	s->tx_error_packets =
485117610Sdes	    MLX5_GET_CTR(out, transmit_errors.packets);
486117610Sdes	s->tx_error_bytes =
487117610Sdes	    MLX5_GET_CTR(out, transmit_errors.octets);
488117610Sdes
489117610Sdes	s->rx_unicast_packets =
490117610Sdes	    MLX5_GET_CTR(out, received_eth_unicast.packets);
491117610Sdes	s->rx_unicast_bytes =
492117610Sdes	    MLX5_GET_CTR(out, received_eth_unicast.octets);
493117610Sdes	s->tx_unicast_packets =
494117610Sdes	    MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
495117610Sdes	s->tx_unicast_bytes =
496117610Sdes	    MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
497117610Sdes
498117610Sdes	s->rx_multicast_packets =
499255376Sdes	    MLX5_GET_CTR(out, received_eth_multicast.packets);
500255376Sdes	s->rx_multicast_bytes =
501255376Sdes	    MLX5_GET_CTR(out, received_eth_multicast.octets);
502255376Sdes	s->tx_multicast_packets =
503117610Sdes	    MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
504117610Sdes	s->tx_multicast_bytes =
505117610Sdes	    MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
506117610Sdes
507117610Sdes	s->rx_broadcast_packets =
508117610Sdes	    MLX5_GET_CTR(out, received_eth_broadcast.packets);
509117610Sdes	s->rx_broadcast_bytes =
510117610Sdes	    MLX5_GET_CTR(out, received_eth_broadcast.octets);
511255376Sdes	s->tx_broadcast_packets =
512255376Sdes	    MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
513255376Sdes	s->tx_broadcast_bytes =
514255376Sdes	    MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
515255376Sdes
516255376Sdes	s->rx_packets =
517255376Sdes	    s->rx_unicast_packets +
518255376Sdes	    s->rx_multicast_packets +
519255376Sdes	    s->rx_broadcast_packets -
520255376Sdes	    s->rx_out_of_buffer;
521255376Sdes	s->rx_bytes =
522255376Sdes	    s->rx_unicast_bytes +
523255376Sdes	    s->rx_multicast_bytes +
524255376Sdes	    s->rx_broadcast_bytes;
525255376Sdes	s->tx_packets =
526255376Sdes	    s->tx_unicast_packets +
527255376Sdes	    s->tx_multicast_packets +
528255376Sdes	    s->tx_broadcast_packets;
529255376Sdes	s->tx_bytes =
530255376Sdes	    s->tx_unicast_bytes +
531255376Sdes	    s->tx_multicast_bytes +
532117610Sdes	    s->tx_broadcast_bytes;
533117610Sdes
534117610Sdes	/* Update calculated offload counters */
535117610Sdes	s->tx_csum_offload = s->tx_packets - tx_offload_none;
536255376Sdes	s->rx_csum_good = s->rx_packets - s->rx_csum_none;
537255376Sdes
538255376Sdes	/* Update per port counters */
539255376Sdes	mlx5e_update_pport_counters(priv);
540117610Sdes
541117610Sdes#if (__FreeBSD_version < 1100000)
542117610Sdes	/* no get_counters interface in fbsd 10 */
543117610Sdes	ifp->if_ipackets = s->rx_packets;
544117610Sdes	ifp->if_ierrors = s->rx_error_packets;
545117610Sdes	ifp->if_iqdrops = s->rx_out_of_buffer;
546117610Sdes	ifp->if_opackets = s->tx_packets;
547117610Sdes	ifp->if_oerrors = s->tx_error_packets;
548117610Sdes	ifp->if_snd.ifq_drops = s->tx_queue_dropped;
549117610Sdes	ifp->if_ibytes = s->rx_bytes;
550117610Sdes	ifp->if_obytes = s->tx_bytes;
551117610Sdes#endif
552117610Sdes
553117610Sdesfree_out:
554117610Sdes	kvfree(out);
555117610Sdes	PRIV_UNLOCK(priv);
556117610Sdes}
557117610Sdes
558117610Sdesstatic void
559117610Sdesmlx5e_update_stats(void *arg)
560117610Sdes{
561117610Sdes	struct mlx5e_priv *priv = arg;
562117610Sdes
563117610Sdes	schedule_work(&priv->update_stats_work);
564255376Sdes
565255376Sdes	callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv);
566255376Sdes}
567255376Sdes
568255376Sdesstatic void
569255376Sdesmlx5e_async_event_sub(struct mlx5e_priv *priv,
570255376Sdes    enum mlx5_dev_event event)
571255376Sdes{
572117610Sdes	switch (event) {
573117610Sdes	case MLX5_DEV_EVENT_PORT_UP:
574117610Sdes	case MLX5_DEV_EVENT_PORT_DOWN:
575255376Sdes		schedule_work(&priv->update_carrier_work);
576255376Sdes		break;
577255376Sdes
578117610Sdes	default:
579117610Sdes		break;
580117610Sdes	}
581255376Sdes}
582255376Sdes
583255376Sdesstatic void
584255376Sdesmlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
585117610Sdes    enum mlx5_dev_event event, unsigned long param)
586117610Sdes{
587117610Sdes	struct mlx5e_priv *priv = vpriv;
588117610Sdes
589117610Sdes	mtx_lock(&priv->async_events_mtx);
590117610Sdes	if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
591117610Sdes		mlx5e_async_event_sub(priv, event);
592117610Sdes	mtx_unlock(&priv->async_events_mtx);
593117610Sdes}
594117610Sdes
595117610Sdesstatic void
596117610Sdesmlx5e_enable_async_events(struct mlx5e_priv *priv)
597117610Sdes{
598117610Sdes	set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
599117610Sdes}
600117610Sdes
601117610Sdesstatic void
602117610Sdesmlx5e_disable_async_events(struct mlx5e_priv *priv)
603117610Sdes{
604117610Sdes	mtx_lock(&priv->async_events_mtx);
605117610Sdes	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
606117610Sdes	mtx_unlock(&priv->async_events_mtx);
607255376Sdes}
608255376Sdes
609255376Sdesstatic const char *mlx5e_rq_stats_desc[] = {
610255376Sdes	MLX5E_RQ_STATS(MLX5E_STATS_DESC)
611255376Sdes};
612255376Sdes
613255376Sdesstatic int
614255376Sdesmlx5e_create_rq(struct mlx5e_channel *c,
615117610Sdes    struct mlx5e_rq_param *param,
616117610Sdes    struct mlx5e_rq *rq)
617117610Sdes{
618117610Sdes	struct mlx5e_priv *priv = c->priv;
619117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
620117610Sdes	char buffer[16];
621117610Sdes	void *rqc = param->rqc;
622117610Sdes	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
623117610Sdes	int wq_sz;
624117610Sdes	int err;
625117610Sdes	int i;
626117610Sdes
627117610Sdes	/* Create DMA descriptor TAG */
628117610Sdes	if ((err = -bus_dma_tag_create(
629117610Sdes	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
630117610Sdes	    1,				/* any alignment */
631117610Sdes	    0,				/* no boundary */
632117610Sdes	    BUS_SPACE_MAXADDR,		/* lowaddr */
633117610Sdes	    BUS_SPACE_MAXADDR,		/* highaddr */
634117610Sdes	    NULL, NULL,			/* filter, filterarg */
635117610Sdes	    MJUM16BYTES,		/* maxsize */
636117610Sdes	    1,				/* nsegments */
637117610Sdes	    MJUM16BYTES,		/* maxsegsize */
638117610Sdes	    0,				/* flags */
639117610Sdes	    NULL, NULL,			/* lockfunc, lockfuncarg */
640117610Sdes	    &rq->dma_tag)))
641117610Sdes		goto done;
642117610Sdes
643117610Sdes	err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
644117610Sdes	    &rq->wq_ctrl);
645117610Sdes	if (err)
646117610Sdes		goto err_free_dma_tag;
647117610Sdes
648117610Sdes	rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
649117610Sdes
650117610Sdes	if (priv->params.hw_lro_en) {
651117610Sdes		rq->wqe_sz = priv->params.lro_wqe_sz;
652117610Sdes	} else {
653117610Sdes		rq->wqe_sz = MLX5E_SW2MB_MTU(priv->ifp->if_mtu);
654117610Sdes	}
655117610Sdes	if (rq->wqe_sz > MJUM16BYTES) {
656117610Sdes		err = -ENOMEM;
657117610Sdes		goto err_rq_wq_destroy;
658117610Sdes	} else if (rq->wqe_sz > MJUM9BYTES) {
659117610Sdes		rq->wqe_sz = MJUM16BYTES;
660117610Sdes	} else if (rq->wqe_sz > MJUMPAGESIZE) {
661117610Sdes		rq->wqe_sz = MJUM9BYTES;
662117610Sdes	} else if (rq->wqe_sz > MCLBYTES) {
663117610Sdes		rq->wqe_sz = MJUMPAGESIZE;
664117610Sdes	} else {
665117610Sdes		rq->wqe_sz = MCLBYTES;
666117610Sdes	}
667117610Sdes
668117610Sdes	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
669117610Sdes
670117610Sdes	err = -tcp_lro_init_args(&rq->lro, c->ifp, TCP_LRO_ENTRIES, wq_sz);
671117610Sdes	if (err)
672117610Sdes		goto err_rq_wq_destroy;
673117610Sdes
674117610Sdes	rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
675117610Sdes	if (rq->mbuf == NULL) {
676117610Sdes		err = -ENOMEM;
677117610Sdes		goto err_lro_init;
678117610Sdes	}
679117610Sdes	for (i = 0; i != wq_sz; i++) {
680117610Sdes		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
681117610Sdes		uint32_t byte_count = rq->wqe_sz - MLX5E_NET_IP_ALIGN;
682117610Sdes
683117610Sdes		err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
684117610Sdes		if (err != 0) {
685117610Sdes			while (i--)
686117610Sdes				bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
687117610Sdes			goto err_rq_mbuf_free;
688117610Sdes		}
689117610Sdes		wqe->data.lkey = c->mkey_be;
690117610Sdes		wqe->data.byte_count = cpu_to_be32(byte_count | MLX5_HW_START_PADDING);
691117610Sdes	}
692117610Sdes
693117610Sdes	rq->pdev = c->pdev;
694117610Sdes	rq->ifp = c->ifp;
695117610Sdes	rq->channel = c;
696117610Sdes	rq->ix = c->ix;
697117610Sdes
698117610Sdes	snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
699117610Sdes	mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
700117610Sdes	    buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
701117610Sdes	    rq->stats.arg);
702117610Sdes	return (0);
703117610Sdes
704117610Sdeserr_rq_mbuf_free:
705117610Sdes	free(rq->mbuf, M_MLX5EN);
706117610Sdeserr_lro_init:
707117610Sdes	tcp_lro_free(&rq->lro);
708117610Sdeserr_rq_wq_destroy:
709117610Sdes	mlx5_wq_destroy(&rq->wq_ctrl);
710117610Sdeserr_free_dma_tag:
711117610Sdes	bus_dma_tag_destroy(rq->dma_tag);
712117610Sdesdone:
713117610Sdes	return (err);
714117610Sdes}
715117610Sdes
716117610Sdesstatic void
717117610Sdesmlx5e_destroy_rq(struct mlx5e_rq *rq)
718117610Sdes{
719117610Sdes	int wq_sz;
720117610Sdes	int i;
721117610Sdes
722117610Sdes	/* destroy all sysctl nodes */
723117610Sdes	sysctl_ctx_free(&rq->stats.ctx);
724117610Sdes
725117610Sdes	/* free leftover LRO packets, if any */
726117610Sdes	tcp_lro_free(&rq->lro);
727117610Sdes
728117610Sdes	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
729117610Sdes	for (i = 0; i != wq_sz; i++) {
730117610Sdes		if (rq->mbuf[i].mbuf != NULL) {
731117610Sdes			bus_dmamap_unload(rq->dma_tag,
732117610Sdes			    rq->mbuf[i].dma_map);
733117610Sdes			m_freem(rq->mbuf[i].mbuf);
734117610Sdes		}
735117610Sdes		bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
736117610Sdes	}
737117610Sdes	free(rq->mbuf, M_MLX5EN);
738117610Sdes	mlx5_wq_destroy(&rq->wq_ctrl);
739117610Sdes}
740117610Sdes
741117610Sdesstatic int
742117610Sdesmlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
743117610Sdes{
744117610Sdes	struct mlx5e_channel *c = rq->channel;
745117610Sdes	struct mlx5e_priv *priv = c->priv;
746117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
747117610Sdes
748117610Sdes	void *in;
749117610Sdes	void *rqc;
750117610Sdes	void *wq;
751117610Sdes	int inlen;
752117610Sdes	int err;
753117610Sdes
754117610Sdes	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
755117610Sdes	    sizeof(u64) * rq->wq_ctrl.buf.npages;
756117610Sdes	in = mlx5_vzalloc(inlen);
757117610Sdes	if (in == NULL)
758117610Sdes		return (-ENOMEM);
759117610Sdes
760117610Sdes	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
761117610Sdes	wq = MLX5_ADDR_OF(rqc, rqc, wq);
762117610Sdes
763117610Sdes	memcpy(rqc, param->rqc, sizeof(param->rqc));
764255376Sdes
765255376Sdes	MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
766255376Sdes	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
767255376Sdes	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
768255376Sdes	if (priv->counter_set_id >= 0)
769255376Sdes		MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
770255376Sdes	MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
771255376Sdes	    PAGE_SHIFT);
772117610Sdes	MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
773117610Sdes
774117610Sdes	mlx5_fill_page_array(&rq->wq_ctrl.buf,
775117610Sdes	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
776117610Sdes
777117610Sdes	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
778117610Sdes
779117610Sdes	kvfree(in);
780117610Sdes
781117610Sdes	return (err);
782117610Sdes}
783255376Sdes
784255376Sdesstatic int
785255376Sdesmlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
786255376Sdes{
787255376Sdes	struct mlx5e_channel *c = rq->channel;
788255376Sdes	struct mlx5e_priv *priv = c->priv;
789255376Sdes	struct mlx5_core_dev *mdev = priv->mdev;
790117610Sdes
791117610Sdes	void *in;
792117610Sdes	void *rqc;
793117610Sdes	int inlen;
794255376Sdes	int err;
795255376Sdes
796255376Sdes	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
797255376Sdes	in = mlx5_vzalloc(inlen);
798117610Sdes	if (in == NULL)
799117610Sdes		return (-ENOMEM);
800117610Sdes
801117610Sdes	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
802117610Sdes
803117610Sdes	MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
804117610Sdes	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
805117610Sdes	MLX5_SET(rqc, rqc, state, next_state);
806117610Sdes
807117610Sdes	err = mlx5_core_modify_rq(mdev, in, inlen);
808117610Sdes
809117610Sdes	kvfree(in);
810117610Sdes
811117610Sdes	return (err);
812117610Sdes}
813117610Sdes
814117610Sdesstatic void
815117610Sdesmlx5e_disable_rq(struct mlx5e_rq *rq)
816117610Sdes{
817117610Sdes	struct mlx5e_channel *c = rq->channel;
818117610Sdes	struct mlx5e_priv *priv = c->priv;
819117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
820117610Sdes
821117610Sdes	mlx5_core_destroy_rq(mdev, rq->rqn);
822117610Sdes}
823255376Sdes
824255376Sdesstatic int
825255376Sdesmlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
826255376Sdes{
827255376Sdes	struct mlx5e_channel *c = rq->channel;
828255376Sdes	struct mlx5e_priv *priv = c->priv;
829255376Sdes	struct mlx5_wq_ll *wq = &rq->wq;
830117610Sdes	int i;
831117610Sdes
832117610Sdes	for (i = 0; i < 1000; i++) {
833117610Sdes		if (wq->cur_sz >= priv->params.min_rx_wqes)
834255376Sdes			return (0);
835255376Sdes
836255376Sdes		msleep(4);
837255376Sdes	}
838117610Sdes	return (-ETIMEDOUT);
839117610Sdes}
840117610Sdes
841117610Sdesstatic int
842117610Sdesmlx5e_open_rq(struct mlx5e_channel *c,
843117610Sdes    struct mlx5e_rq_param *param,
844117610Sdes    struct mlx5e_rq *rq)
845117610Sdes{
846117610Sdes	int err;
847117610Sdes
848117610Sdes	err = mlx5e_create_rq(c, param, rq);
849117610Sdes	if (err)
850117610Sdes		return (err);
851117610Sdes
852117610Sdes	err = mlx5e_enable_rq(rq, param);
853117610Sdes	if (err)
854117610Sdes		goto err_destroy_rq;
855117610Sdes
856117610Sdes	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
857117610Sdes	if (err)
858117610Sdes		goto err_disable_rq;
859117610Sdes
860117610Sdes	c->rq.enabled = 1;
861117610Sdes
862117610Sdes	return (0);
863117610Sdes
864117610Sdeserr_disable_rq:
865117610Sdes	mlx5e_disable_rq(rq);
866117610Sdeserr_destroy_rq:
867117610Sdes	mlx5e_destroy_rq(rq);
868117610Sdes
869117610Sdes	return (err);
870117610Sdes}
871117610Sdes
872117610Sdesstatic void
873117610Sdesmlx5e_close_rq(struct mlx5e_rq *rq)
874117610Sdes{
875117610Sdes	rq->enabled = 0;
876117610Sdes	mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
877117610Sdes}
878117610Sdes
879117610Sdesstatic void
880117610Sdesmlx5e_close_rq_wait(struct mlx5e_rq *rq)
881117610Sdes{
882117610Sdes	/* wait till RQ is empty */
883117610Sdes	while (!mlx5_wq_ll_is_empty(&rq->wq)) {
884117610Sdes		msleep(4);
885117610Sdes		rq->cq.mcq.comp(&rq->cq.mcq);
886117610Sdes	}
887117610Sdes
888117610Sdes	mlx5e_disable_rq(rq);
889117610Sdes	mlx5e_destroy_rq(rq);
890117610Sdes}
891117610Sdes
892117610Sdesstatic void
893117610Sdesmlx5e_free_sq_db(struct mlx5e_sq *sq)
894117610Sdes{
895117610Sdes	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
896117610Sdes	int x;
897117610Sdes
898117610Sdes	for (x = 0; x != wq_sz; x++)
899117610Sdes		bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
900117610Sdes	free(sq->mbuf, M_MLX5EN);
901117610Sdes}
902255376Sdes
903255376Sdesstatic int
904117610Sdesmlx5e_alloc_sq_db(struct mlx5e_sq *sq)
905255376Sdes{
906255376Sdes	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
907255376Sdes	int err;
908117610Sdes	int x;
909117610Sdes
910117610Sdes	sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
911117610Sdes	if (sq->mbuf == NULL)
912117610Sdes		return (-ENOMEM);
913117610Sdes
914117610Sdes	/* Create DMA descriptor MAPs */
915255376Sdes	for (x = 0; x != wq_sz; x++) {
916117610Sdes		err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
917117610Sdes		if (err != 0) {
918255376Sdes			while (x--)
919255376Sdes				bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
920255376Sdes			free(sq->mbuf, M_MLX5EN);
921255376Sdes			return (err);
922117610Sdes		}
923117610Sdes	}
924117610Sdes	return (0);
925117610Sdes}
926117610Sdes
927117610Sdesstatic const char *mlx5e_sq_stats_desc[] = {
928117610Sdes	MLX5E_SQ_STATS(MLX5E_STATS_DESC)
929117610Sdes};
930117610Sdes
931117610Sdesstatic int
932117610Sdesmlx5e_create_sq(struct mlx5e_channel *c,
933117610Sdes    int tc,
934117610Sdes    struct mlx5e_sq_param *param,
935117610Sdes    struct mlx5e_sq *sq)
936117610Sdes{
937117610Sdes	struct mlx5e_priv *priv = c->priv;
938255376Sdes	struct mlx5_core_dev *mdev = priv->mdev;
939255376Sdes	char buffer[16];
940255376Sdes
941255376Sdes	void *sqc = param->sqc;
942255376Sdes	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
943255376Sdes#ifdef RSS
944255376Sdes	cpuset_t cpu_mask;
945255376Sdes	int cpu_id;
946117610Sdes#endif
947117610Sdes	int err;
948117610Sdes
949117610Sdes	/* Create DMA descriptor TAG */
950117610Sdes	if ((err = -bus_dma_tag_create(
951117610Sdes	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
952117610Sdes	    1,				/* any alignment */
953117610Sdes	    0,				/* no boundary */
954117610Sdes	    BUS_SPACE_MAXADDR,		/* lowaddr */
955255376Sdes	    BUS_SPACE_MAXADDR,		/* highaddr */
956255376Sdes	    NULL, NULL,			/* filter, filterarg */
957255376Sdes	    MLX5E_MAX_TX_PAYLOAD_SIZE,	/* maxsize */
958255376Sdes	    MLX5E_MAX_TX_MBUF_FRAGS,	/* nsegments */
959255376Sdes	    MLX5E_MAX_TX_MBUF_SIZE,	/* maxsegsize */
960255376Sdes	    0,				/* flags */
961117610Sdes	    NULL, NULL,			/* lockfunc, lockfuncarg */
962117610Sdes	    &sq->dma_tag)))
963117610Sdes		goto done;
964117610Sdes
965117610Sdes	err = mlx5_alloc_map_uar(mdev, &sq->uar);
966117610Sdes	if (err)
967117610Sdes		goto err_free_dma_tag;
968117610Sdes
969117610Sdes	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
970117610Sdes	    &sq->wq_ctrl);
971117610Sdes	if (err)
972117610Sdes		goto err_unmap_free_uar;
973117610Sdes
974117610Sdes	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
975117610Sdes	sq->uar_map = sq->uar.map;
976117610Sdes	sq->uar_bf_map = sq->uar.bf_map;
977117610Sdes	sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
978117610Sdes
979117610Sdes	err = mlx5e_alloc_sq_db(sq);
980117610Sdes	if (err)
981117610Sdes		goto err_sq_wq_destroy;
982117610Sdes
983117610Sdes	sq->pdev = c->pdev;
984117610Sdes	sq->mkey_be = c->mkey_be;
985117610Sdes	sq->channel = c;
986117610Sdes	sq->tc = tc;
987117610Sdes
988117610Sdes	sq->br = buf_ring_alloc(MLX5E_SQ_TX_QUEUE_SIZE, M_MLX5EN,
989117610Sdes	    M_WAITOK, &sq->lock);
990255376Sdes	if (sq->br == NULL) {
991117610Sdes		if_printf(c->ifp, "%s: Failed allocating sq drbr buffer\n",
992255376Sdes		    __func__);
993255376Sdes		err = -ENOMEM;
994117610Sdes		goto err_free_sq_db;
995117610Sdes	}
996117610Sdes
997117610Sdes	sq->sq_tq = taskqueue_create_fast("mlx5e_que", M_WAITOK,
998117610Sdes	    taskqueue_thread_enqueue, &sq->sq_tq);
999117610Sdes	if (sq->sq_tq == NULL) {
1000117610Sdes		if_printf(c->ifp, "%s: Failed allocating taskqueue\n",
1001117610Sdes		    __func__);
1002117610Sdes		err = -ENOMEM;
1003117610Sdes		goto err_free_drbr;
1004117610Sdes	}
1005117610Sdes
1006117610Sdes	TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq);
1007117610Sdes#ifdef RSS
1008117610Sdes	cpu_id = rss_getcpu(c->ix % rss_getnumbuckets());
1009117610Sdes	CPU_SETOF(cpu_id, &cpu_mask);
1010117610Sdes	taskqueue_start_threads_cpuset(&sq->sq_tq, 1, PI_NET, &cpu_mask,
1011117610Sdes	    "%s TX SQ%d.%d CPU%d", c->ifp->if_xname, c->ix, tc, cpu_id);
1012117610Sdes#else
1013117610Sdes	taskqueue_start_threads(&sq->sq_tq, 1, PI_NET,
1014117610Sdes	    "%s TX SQ%d.%d", c->ifp->if_xname, c->ix, tc);
1015117610Sdes#endif
1016117610Sdes	snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1017117610Sdes	mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1018255376Sdes	    buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1019255376Sdes	    sq->stats.arg);
1020255376Sdes
1021255376Sdes	return (0);
1022255376Sdes
1023255376Sdeserr_free_drbr:
1024255376Sdes	buf_ring_free(sq->br, M_MLX5EN);
1025255376Sdeserr_free_sq_db:
1026117610Sdes	mlx5e_free_sq_db(sq);
1027117610Sdeserr_sq_wq_destroy:
1028117610Sdes	mlx5_wq_destroy(&sq->wq_ctrl);
1029117610Sdes
1030117610Sdeserr_unmap_free_uar:
1031117610Sdes	mlx5_unmap_free_uar(mdev, &sq->uar);
1032117610Sdes
1033117610Sdeserr_free_dma_tag:
1034117610Sdes	bus_dma_tag_destroy(sq->dma_tag);
1035117610Sdesdone:
1036117610Sdes	return (err);
1037117610Sdes}
1038117610Sdes
1039117610Sdesstatic void
1040117610Sdesmlx5e_destroy_sq(struct mlx5e_sq *sq)
1041117610Sdes{
1042117610Sdes	struct mlx5e_channel *c = sq->channel;
1043117610Sdes	struct mlx5e_priv *priv = c->priv;
1044117610Sdes
1045117610Sdes	/* destroy all sysctl nodes */
1046117610Sdes	sysctl_ctx_free(&sq->stats.ctx);
1047117610Sdes
1048117610Sdes	mlx5e_free_sq_db(sq);
1049117610Sdes	mlx5_wq_destroy(&sq->wq_ctrl);
1050117610Sdes	mlx5_unmap_free_uar(priv->mdev, &sq->uar);
1051117610Sdes	taskqueue_drain(sq->sq_tq, &sq->sq_task);
1052255376Sdes	taskqueue_free(sq->sq_tq);
1053255376Sdes	buf_ring_free(sq->br, M_MLX5EN);
1054255376Sdes}
1055255376Sdes
1056117610Sdesstatic int
1057117610Sdesmlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
1058117610Sdes{
1059117610Sdes	struct mlx5e_channel *c = sq->channel;
1060117610Sdes	struct mlx5e_priv *priv = c->priv;
1061117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
1062117610Sdes
1063117610Sdes	void *in;
1064117610Sdes	void *sqc;
1065117610Sdes	void *wq;
1066117610Sdes	int inlen;
1067255376Sdes	int err;
1068255376Sdes
1069255376Sdes	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1070117610Sdes	    sizeof(u64) * sq->wq_ctrl.buf.npages;
1071117610Sdes	in = mlx5_vzalloc(inlen);
1072117610Sdes	if (in == NULL)
1073117610Sdes		return (-ENOMEM);
1074117610Sdes
1075117610Sdes	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1076117610Sdes	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1077117610Sdes
1078117610Sdes	memcpy(sqc, param->sqc, sizeof(param->sqc));
1079117610Sdes
1080117610Sdes	MLX5_SET(sqc, sqc, tis_num_0, priv->tisn[sq->tc]);
1081117610Sdes	MLX5_SET(sqc, sqc, cqn, c->sq[sq->tc].cq.mcq.cqn);
1082117610Sdes	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1083117610Sdes	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1084117610Sdes	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1085117610Sdes
1086117610Sdes	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1087117610Sdes	MLX5_SET(wq, wq, uar_page, sq->uar.index);
1088117610Sdes	MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1089117610Sdes	    PAGE_SHIFT);
1090117610Sdes	MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1091255376Sdes
1092255376Sdes	mlx5_fill_page_array(&sq->wq_ctrl.buf,
1093255376Sdes	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1094117610Sdes
1095117610Sdes	err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
1096117610Sdes
1097117610Sdes	kvfree(in);
1098117610Sdes
1099117610Sdes	return (err);
1100117610Sdes}
1101117610Sdes
1102117610Sdesstatic int
1103117610Sdesmlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1104117610Sdes{
1105117610Sdes	struct mlx5e_channel *c = sq->channel;
1106117610Sdes	struct mlx5e_priv *priv = c->priv;
1107117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
1108117610Sdes
1109117610Sdes	void *in;
1110117610Sdes	void *sqc;
1111117610Sdes	int inlen;
1112117610Sdes	int err;
1113117610Sdes
1114117610Sdes	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1115117610Sdes	in = mlx5_vzalloc(inlen);
1116117610Sdes	if (in == NULL)
1117117610Sdes		return (-ENOMEM);
1118117610Sdes
1119117610Sdes	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1120117610Sdes
1121117610Sdes	MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1122117610Sdes	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1123117610Sdes	MLX5_SET(sqc, sqc, state, next_state);
1124117610Sdes
1125117610Sdes	err = mlx5_core_modify_sq(mdev, in, inlen);
1126117610Sdes
1127117610Sdes	kvfree(in);
1128117610Sdes
1129117610Sdes	return (err);
1130117610Sdes}
1131117610Sdes
1132117610Sdesstatic void
1133117610Sdesmlx5e_disable_sq(struct mlx5e_sq *sq)
1134117610Sdes{
1135117610Sdes	struct mlx5e_channel *c = sq->channel;
1136117610Sdes	struct mlx5e_priv *priv = c->priv;
1137117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
1138117610Sdes
1139117610Sdes	mlx5_core_destroy_sq(mdev, sq->sqn);
1140117610Sdes}
1141117610Sdes
1142117610Sdesstatic int
1143117610Sdesmlx5e_open_sq(struct mlx5e_channel *c,
1144117610Sdes    int tc,
1145117610Sdes    struct mlx5e_sq_param *param,
1146117610Sdes    struct mlx5e_sq *sq)
1147117610Sdes{
1148117610Sdes	int err;
1149117610Sdes
1150255376Sdes	err = mlx5e_create_sq(c, tc, param, sq);
1151255376Sdes	if (err)
1152255376Sdes		return (err);
1153117610Sdes
1154117610Sdes	err = mlx5e_enable_sq(sq, param);
1155117610Sdes	if (err)
1156117610Sdes		goto err_destroy_sq;
1157117610Sdes
1158117610Sdes	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1159117610Sdes	if (err)
1160117610Sdes		goto err_disable_sq;
1161117610Sdes
1162117610Sdes	atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_READY);
1163117610Sdes
1164117610Sdes	return (0);
1165117610Sdes
1166117610Sdeserr_disable_sq:
1167255376Sdes	mlx5e_disable_sq(sq);
1168255376Sdeserr_destroy_sq:
1169255376Sdes	mlx5e_destroy_sq(sq);
1170255376Sdes
1171117610Sdes	return (err);
1172117610Sdes}
1173117610Sdes
1174117610Sdesstatic void
1175117610Sdesmlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1176117610Sdes{
1177117610Sdes	/* fill up remainder with NOPs */
1178117610Sdes	while (sq->cev_counter != 0) {
1179117610Sdes		while (!mlx5e_sq_has_room_for(sq, 1)) {
1180117610Sdes			if (can_sleep != 0) {
1181117610Sdes				mtx_unlock(&sq->lock);
1182117610Sdes				msleep(4);
1183117610Sdes				mtx_lock(&sq->lock);
1184117610Sdes			} else {
1185117610Sdes				goto done;
1186117610Sdes			}
1187117610Sdes		}
1188117610Sdes		/* send a single NOP */
1189117610Sdes		mlx5e_send_nop(sq, 1);
1190117610Sdes		wmb();
1191117610Sdes	}
1192117610Sdesdone:
1193117610Sdes	/* Check if we need to write the doorbell */
1194117610Sdes	if (likely(sq->doorbell.d64 != 0)) {
1195117610Sdes		mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
1196117610Sdes		sq->doorbell.d64 = 0;
1197117610Sdes	}
1198117610Sdes	return;
1199117610Sdes}
1200117610Sdes
1201117610Sdesvoid
1202117610Sdesmlx5e_sq_cev_timeout(void *arg)
1203117610Sdes{
1204117610Sdes	struct mlx5e_sq *sq = arg;
1205117610Sdes
1206117610Sdes	mtx_assert(&sq->lock, MA_OWNED);
1207117610Sdes
1208117610Sdes	/* check next state */
1209117610Sdes	switch (sq->cev_next_state) {
1210117610Sdes	case MLX5E_CEV_STATE_SEND_NOPS:
1211117610Sdes		/* fill TX ring with NOPs, if any */
1212117610Sdes		mlx5e_sq_send_nops_locked(sq, 0);
1213117610Sdes
1214255376Sdes		/* check if completed */
1215255376Sdes		if (sq->cev_counter == 0) {
1216255376Sdes			sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1217255376Sdes			return;
1218117610Sdes		}
1219117610Sdes		break;
1220117610Sdes	default:
1221255376Sdes		/* send NOPs on next timeout */
1222255376Sdes		sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1223255376Sdes		break;
1224117610Sdes	}
1225117610Sdes
1226117610Sdes	/* restart timer */
1227117610Sdes	callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1228117610Sdes}
1229117610Sdes
1230117610Sdesstatic void
1231117610Sdesmlx5e_close_sq_wait(struct mlx5e_sq *sq)
1232255376Sdes{
1233255376Sdes
1234255376Sdes	mtx_lock(&sq->lock);
1235255376Sdes	/* teardown event factor timer, if any */
1236117610Sdes	sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1237117610Sdes	callout_stop(&sq->cev_callout);
1238117610Sdes
1239117610Sdes	/* send dummy NOPs in order to flush the transmit ring */
1240117610Sdes	mlx5e_sq_send_nops_locked(sq, 1);
1241117610Sdes	mtx_unlock(&sq->lock);
1242117610Sdes
1243117610Sdes	/* make sure it is safe to free the callout */
1244117610Sdes	callout_drain(&sq->cev_callout);
1245117610Sdes
1246117610Sdes	/* error out remaining requests */
1247117610Sdes	mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1248117610Sdes
1249117610Sdes	/* wait till SQ is empty */
1250117610Sdes	mtx_lock(&sq->lock);
1251117610Sdes	while (sq->cc != sq->pc) {
1252117610Sdes		mtx_unlock(&sq->lock);
1253117610Sdes		msleep(4);
1254117610Sdes		sq->cq.mcq.comp(&sq->cq.mcq);
1255255376Sdes		mtx_lock(&sq->lock);
1256255376Sdes	}
1257255376Sdes	mtx_unlock(&sq->lock);
1258117610Sdes
1259117610Sdes	mlx5e_disable_sq(sq);
1260117610Sdes	mlx5e_destroy_sq(sq);
1261117610Sdes}
1262117610Sdes
1263117610Sdesstatic int
1264117610Sdesmlx5e_create_cq(struct mlx5e_channel *c,
1265117610Sdes    struct mlx5e_cq_param *param,
1266117610Sdes    struct mlx5e_cq *cq,
1267117610Sdes    mlx5e_cq_comp_t *comp)
1268117610Sdes{
1269117610Sdes	struct mlx5e_priv *priv = c->priv;
1270117610Sdes	struct mlx5_core_dev *mdev = priv->mdev;
1271117610Sdes	struct mlx5_core_cq *mcq = &cq->mcq;
1272117610Sdes	int eqn_not_used;
1273117610Sdes	int irqn;
1274255376Sdes	int err;
1275117610Sdes	u32 i;
1276117610Sdes
1277255376Sdes	param->wq.buf_numa_node = 0;
1278117610Sdes	param->wq.db_numa_node = 0;
1279117610Sdes	param->eq_ix = c->ix;
1280117610Sdes
1281117610Sdes	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1282117610Sdes	    &cq->wq_ctrl);
1283117610Sdes	if (err)
1284117610Sdes		return (err);
1285117610Sdes
1286117610Sdes	mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1287117610Sdes
1288117610Sdes	mcq->cqe_sz = 64;
1289117610Sdes	mcq->set_ci_db = cq->wq_ctrl.db.db;
1290117610Sdes	mcq->arm_db = cq->wq_ctrl.db.db + 1;
1291117610Sdes	*mcq->set_ci_db = 0;
1292117610Sdes	*mcq->arm_db = 0;
1293117610Sdes	mcq->vector = param->eq_ix;
1294117610Sdes	mcq->comp = comp;
1295117610Sdes	mcq->event = mlx5e_cq_error_event;
1296117610Sdes	mcq->irqn = irqn;
1297117610Sdes	mcq->uar = &priv->cq_uar;
1298117610Sdes
1299117610Sdes	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1300117610Sdes		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1301117610Sdes
1302117610Sdes		cqe->op_own = 0xf1;
1303117610Sdes	}
1304117610Sdes
1305117610Sdes	cq->channel = c;
1306117610Sdes
1307117610Sdes	return (0);
1308117610Sdes}
1309117610Sdes
1310117610Sdesstatic void
1311117610Sdesmlx5e_destroy_cq(struct mlx5e_cq *cq)
1312117610Sdes{
1313117610Sdes	mlx5_wq_destroy(&cq->wq_ctrl);
1314117610Sdes}
1315117610Sdes
1316117610Sdesstatic int
1317117610Sdesmlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param,
1318117610Sdes    u8 moderation_mode)
1319117610Sdes{
1320117610Sdes	struct mlx5e_channel *c = cq->channel;
1321255376Sdes	struct mlx5e_priv *priv = c->priv;
1322255376Sdes	struct mlx5_core_dev *mdev = priv->mdev;
1323117610Sdes	struct mlx5_core_cq *mcq = &cq->mcq;
1324255376Sdes	void *in;
1325255376Sdes	void *cqc;
1326255376Sdes	int inlen;
1327117610Sdes	int irqn_not_used;
1328117610Sdes	int eqn;
1329117610Sdes	int err;
1330117610Sdes
1331117610Sdes	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1332117610Sdes	    sizeof(u64) * cq->wq_ctrl.buf.npages;
1333117610Sdes	in = mlx5_vzalloc(inlen);
1334117610Sdes	if (in == NULL)
1335117610Sdes		return (-ENOMEM);
1336117610Sdes
1337117610Sdes	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1338117610Sdes
1339117610Sdes	memcpy(cqc, param->cqc, sizeof(param->cqc));
1340117610Sdes
1341117610Sdes	mlx5_fill_page_array(&cq->wq_ctrl.buf,
1342117610Sdes	    (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
1343117610Sdes
1344117610Sdes	mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1345117610Sdes
1346117610Sdes	MLX5_SET(cqc, cqc, cq_period_mode, moderation_mode);
1347255376Sdes	MLX5_SET(cqc, cqc, c_eqn, eqn);
1348255376Sdes	MLX5_SET(cqc, cqc, uar_page, mcq->uar->index);
1349255376Sdes	MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1350117610Sdes	    PAGE_SHIFT);
1351255376Sdes	MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1352117610Sdes
1353117610Sdes	err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1354255376Sdes
1355255376Sdes	kvfree(in);
1356255376Sdes
1357255376Sdes	if (err)
1358117610Sdes		return (err);
1359117610Sdes
1360255376Sdes	mlx5e_cq_arm(cq);
1361255376Sdes
1362255376Sdes	return (0);
1363255376Sdes}
1364255376Sdes
1365117610Sdesstatic void
1366117610Sdesmlx5e_disable_cq(struct mlx5e_cq *cq)
1367117610Sdes{
1368117610Sdes	struct mlx5e_channel *c = cq->channel;
1369117610Sdes	struct mlx5e_priv *priv = c->priv;
1370255376Sdes	struct mlx5_core_dev *mdev = priv->mdev;
1371255376Sdes
1372117610Sdes	mlx5_core_destroy_cq(mdev, &cq->mcq);
1373117610Sdes}
1374117610Sdes
1375117610Sdesstatic int
1376117610Sdesmlx5e_open_cq(struct mlx5e_channel *c,
1377117610Sdes    struct mlx5e_cq_param *param,
1378117610Sdes    struct mlx5e_cq *cq,
1379117610Sdes    mlx5e_cq_comp_t *comp,
1380117610Sdes    u8 moderation_mode)
1381117610Sdes{
1382117610Sdes	int err;
1383117610Sdes
1384117610Sdes	err = mlx5e_create_cq(c, param, cq, comp);
1385117610Sdes	if (err)
1386117610Sdes		return (err);
1387117610Sdes
1388117610Sdes	err = mlx5e_enable_cq(cq, param, moderation_mode);
1389255376Sdes	if (err)
1390117610Sdes		goto err_destroy_cq;
1391117610Sdes
1392117610Sdes	return (0);
1393117610Sdes
1394117610Sdeserr_destroy_cq:
1395255376Sdes	mlx5e_destroy_cq(cq);
1396255376Sdes
1397255376Sdes	return (err);
1398117610Sdes}
1399117610Sdes
1400117610Sdesstatic void
1401117610Sdesmlx5e_close_cq(struct mlx5e_cq *cq)
1402117610Sdes{
1403117610Sdes	mlx5e_disable_cq(cq);
1404117610Sdes	mlx5e_destroy_cq(cq);
1405117610Sdes}
1406117610Sdes
1407117610Sdesstatic int
1408117610Sdesmlx5e_open_tx_cqs(struct mlx5e_channel *c,
1409117610Sdes    struct mlx5e_channel_param *cparam)
1410255376Sdes{
1411255376Sdes	u8 tx_moderation_mode;
1412255376Sdes	int err;
1413117610Sdes	int tc;
1414117610Sdes
1415117610Sdes	switch (c->priv->params.tx_cq_moderation_mode) {
1416117610Sdes	case 0:
1417117610Sdes		tx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1418117610Sdes		break;
1419117610Sdes	default:
1420117610Sdes		if (MLX5_CAP_GEN(c->priv->mdev, cq_period_start_from_cqe))
1421117610Sdes			tx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
1422117610Sdes		else
1423117610Sdes			tx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1424117610Sdes		break;
1425117610Sdes	}
1426117610Sdes	for (tc = 0; tc < c->num_tc; tc++) {
1427117610Sdes		/* open completion queue */
1428117610Sdes		err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
1429117610Sdes		    &mlx5e_tx_cq_comp, tx_moderation_mode);
1430117610Sdes		if (err)
1431117610Sdes			goto err_close_tx_cqs;
1432117610Sdes	}
1433117610Sdes	return (0);
1434255376Sdes
1435255376Sdeserr_close_tx_cqs:
1436255376Sdes	for (tc--; tc >= 0; tc--)
1437117610Sdes		mlx5e_close_cq(&c->sq[tc].cq);
1438117610Sdes
1439117610Sdes	return (err);
1440117610Sdes}
1441117610Sdes
1442117610Sdesstatic void
1443117610Sdesmlx5e_close_tx_cqs(struct mlx5e_channel *c)
1444117610Sdes{
1445117610Sdes	int tc;
1446117610Sdes
1447117610Sdes	for (tc = 0; tc < c->num_tc; tc++)
1448117610Sdes		mlx5e_close_cq(&c->sq[tc].cq);
1449117610Sdes}
1450117610Sdes
1451117610Sdesstatic int
1452117610Sdesmlx5e_open_sqs(struct mlx5e_channel *c,
1453117610Sdes    struct mlx5e_channel_param *cparam)
1454117610Sdes{
1455117610Sdes	int err;
1456117610Sdes	int tc;
1457117610Sdes
1458117610Sdes	for (tc = 0; tc < c->num_tc; tc++) {
1459255376Sdes		err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1460255376Sdes		if (err)
1461255376Sdes			goto err_close_sqs;
1462117610Sdes	}
1463117610Sdes
1464117610Sdes	return (0);
1465117610Sdes
1466117610Sdeserr_close_sqs:
1467117610Sdes	for (tc--; tc >= 0; tc--)
1468117610Sdes		mlx5e_close_sq_wait(&c->sq[tc]);
1469117610Sdes
1470117610Sdes	return (err);
1471117610Sdes}
1472117610Sdes
1473117610Sdesstatic void
1474117610Sdesmlx5e_close_sqs_wait(struct mlx5e_channel *c)
1475117610Sdes{
1476117610Sdes	int tc;
1477117610Sdes
1478117610Sdes	for (tc = 0; tc < c->num_tc; tc++)
1479117610Sdes		mlx5e_close_sq_wait(&c->sq[tc]);
1480117610Sdes}
1481117610Sdes
1482117610Sdesstatic void
1483117610Sdesmlx5e_chan_mtx_init(struct mlx5e_channel *c)
1484117610Sdes{
1485117610Sdes	int tc;
1486117610Sdes
1487117610Sdes	mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
1488117610Sdes
1489117610Sdes	for (tc = 0; tc < c->num_tc; tc++) {
1490117610Sdes		struct mlx5e_sq *sq = c->sq + tc;
1491117610Sdes
1492117610Sdes		mtx_init(&sq->lock, "mlx5tx", MTX_NETWORK_LOCK, MTX_DEF);
1493117610Sdes		mtx_init(&sq->comp_lock, "mlx5comp", MTX_NETWORK_LOCK,
1494117610Sdes		    MTX_DEF);
1495255376Sdes
1496255376Sdes		callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
1497117610Sdes
1498255376Sdes		sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1499255376Sdes
1500255376Sdes		/* ensure the TX completion event factor is not zero */
1501255376Sdes		if (sq->cev_factor == 0)
1502255376Sdes			sq->cev_factor = 1;
1503117610Sdes	}
1504117610Sdes}
1505117610Sdes
1506117610Sdesstatic void
1507117610Sdesmlx5e_chan_mtx_destroy(struct mlx5e_channel *c)
1508117610Sdes{
1509117610Sdes	int tc;
1510117610Sdes
1511117610Sdes	mtx_destroy(&c->rq.mtx);
1512117610Sdes
1513117610Sdes	for (tc = 0; tc < c->num_tc; tc++) {
1514117610Sdes		mtx_destroy(&c->sq[tc].lock);
1515117610Sdes		mtx_destroy(&c->sq[tc].comp_lock);
1516117610Sdes	}
1517117610Sdes}
1518117610Sdes
1519117610Sdesstatic int
1520117610Sdesmlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1521117610Sdes    struct mlx5e_channel_param *cparam,
1522117610Sdes    struct mlx5e_channel *volatile *cp)
1523117610Sdes{
1524117610Sdes	struct mlx5e_channel *c;
1525255376Sdes	u8 rx_moderation_mode;
1526255376Sdes	int err;
1527255376Sdes
1528255376Sdes	c = malloc(sizeof(*c), M_MLX5EN, M_WAITOK | M_ZERO);
1529255376Sdes	if (c == NULL)
1530255376Sdes		return (-ENOMEM);
1531117610Sdes
1532117610Sdes	c->priv = priv;
1533117610Sdes	c->ix = ix;
1534117610Sdes	c->cpu = 0;
1535117610Sdes	c->pdev = &priv->mdev->pdev->dev;
1536117610Sdes	c->ifp = priv->ifp;
1537117610Sdes	c->mkey_be = cpu_to_be32(priv->mr.key);
1538117610Sdes	c->num_tc = priv->num_tc;
1539117610Sdes
1540255376Sdes	/* init mutexes */
1541255376Sdes	mlx5e_chan_mtx_init(c);
1542255376Sdes
1543255376Sdes	/* open transmit completion queue */
1544255376Sdes	err = mlx5e_open_tx_cqs(c, cparam);
1545255376Sdes	if (err)
1546255376Sdes		goto err_free;
1547255376Sdes
1548255376Sdes	switch (priv->params.rx_cq_moderation_mode) {
1549255376Sdes	case 0:
1550255376Sdes		rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1551255376Sdes		break;
1552255376Sdes	default:
1553255376Sdes		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
1554255376Sdes			rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
1555117610Sdes		else
1556117610Sdes			rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1557117610Sdes		break;
1558117610Sdes	}
1559117610Sdes
1560117610Sdes	/* open receive completion queue */
1561117610Sdes	err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
1562117610Sdes	    &mlx5e_rx_cq_comp, rx_moderation_mode);
1563117610Sdes	if (err)
1564117610Sdes		goto err_close_tx_cqs;
1565117610Sdes
1566117610Sdes	err = mlx5e_open_sqs(c, cparam);
1567117610Sdes	if (err)
1568117610Sdes		goto err_close_rx_cq;
1569117610Sdes
1570117610Sdes	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1571117610Sdes	if (err)
1572117610Sdes		goto err_close_sqs;
1573117610Sdes
1574117610Sdes	/* store channel pointer */
1575117610Sdes	*cp = c;
1576117610Sdes
1577255376Sdes	/* poll receive queue initially */
1578255376Sdes	c->rq.cq.mcq.comp(&c->rq.cq.mcq);
1579255376Sdes
1580117610Sdes	return (0);
1581117610Sdes
1582117610Sdeserr_close_sqs:
1583117610Sdes	mlx5e_close_sqs_wait(c);
1584117610Sdes
1585117610Sdeserr_close_rx_cq:
1586255376Sdes	mlx5e_close_cq(&c->rq.cq);
1587255376Sdes
1588255376Sdeserr_close_tx_cqs:
1589117610Sdes	mlx5e_close_tx_cqs(c);
1590117610Sdes
1591117610Sdeserr_free:
1592117610Sdes	/* destroy mutexes */
1593117610Sdes	mlx5e_chan_mtx_destroy(c);
1594117610Sdes	free(c, M_MLX5EN);
1595117610Sdes	return (err);
1596117610Sdes}
1597117610Sdes
1598117610Sdesstatic void
1599117610Sdesmlx5e_close_channel(struct mlx5e_channel *volatile *pp)
1600117610Sdes{
1601255376Sdes	struct mlx5e_channel *c = *pp;
1602255376Sdes
1603255376Sdes	/* check if channel is already closed */
1604117610Sdes	if (c == NULL)
1605117610Sdes		return;
1606117610Sdes	mlx5e_close_rq(&c->rq);
1607255376Sdes}
1608255376Sdes
1609255376Sdesstatic void
1610117610Sdesmlx5e_close_channel_wait(struct mlx5e_channel *volatile *pp)
1611117610Sdes{
1612117610Sdes	struct mlx5e_channel *c = *pp;
1613117610Sdes
1614117610Sdes	/* check if channel is already closed */
1615117610Sdes	if (c == NULL)
1616117610Sdes		return;
1617117610Sdes	/* ensure channel pointer is no longer used */
1618117610Sdes	*pp = NULL;
1619117610Sdes
1620117610Sdes	mlx5e_close_rq_wait(&c->rq);
1621117610Sdes	mlx5e_close_sqs_wait(c);
1622117610Sdes	mlx5e_close_cq(&c->rq.cq);
1623117610Sdes	mlx5e_close_tx_cqs(c);
1624117610Sdes	/* destroy mutexes */
1625117610Sdes	mlx5e_chan_mtx_destroy(c);
1626117610Sdes	free(c, M_MLX5EN);
1627117610Sdes}
1628117610Sdes
1629117610Sdesstatic void
1630117610Sdesmlx5e_build_rq_param(struct mlx5e_priv *priv,
1631117610Sdes    struct mlx5e_rq_param *param)
1632117610Sdes{
1633117610Sdes	void *rqc = param->rqc;
1634117610Sdes	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1635117610Sdes
1636117610Sdes	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1637117610Sdes	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1638117610Sdes	MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
1639117610Sdes	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
1640117610Sdes	MLX5_SET(wq, wq, pd, priv->pdn);
1641117610Sdes
1642117610Sdes	param->wq.buf_numa_node = 0;
1643117610Sdes	param->wq.db_numa_node = 0;
1644117610Sdes	param->wq.linear = 1;
1645117610Sdes}
1646117610Sdes
1647117610Sdesstatic void
1648117610Sdesmlx5e_build_sq_param(struct mlx5e_priv *priv,
1649117610Sdes    struct mlx5e_sq_param *param)
1650117610Sdes{
1651117610Sdes	void *sqc = param->sqc;
1652117610Sdes	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1653117610Sdes
1654117610Sdes	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
1655117610Sdes	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1656117610Sdes	MLX5_SET(wq, wq, pd, priv->pdn);
1657117610Sdes
1658117610Sdes	param->wq.buf_numa_node = 0;
1659117610Sdes	param->wq.db_numa_node = 0;
1660117610Sdes	param->wq.linear = 1;
1661117610Sdes}
1662117610Sdes
1663117610Sdesstatic void
1664117610Sdesmlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1665117610Sdes    struct mlx5e_cq_param *param)
1666117610Sdes{
1667117610Sdes	void *cqc = param->cqc;
1668117610Sdes
1669117610Sdes	MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index);
1670117610Sdes}
1671117610Sdes
1672117610Sdesstatic void
1673117610Sdesmlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1674117610Sdes    struct mlx5e_cq_param *param)
1675117610Sdes{
1676117610Sdes	void *cqc = param->cqc;
1677117610Sdes
1678117610Sdes
1679117610Sdes	/*
1680117610Sdes	 * TODO The sysctl to control on/off is a bool value for now, which means
1681117610Sdes	 * we only support CSUM, once HASH is implemnted we'll need to address that.
1682117610Sdes	 */
1683117610Sdes	if (priv->params.cqe_zipping_en) {
1684117610Sdes		MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1685117610Sdes		MLX5_SET(cqc, cqc, cqe_compression_en, 1);
1686117610Sdes	}
1687117610Sdes
1688117610Sdes	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
1689117610Sdes	MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
1690117610Sdes	MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
1691117610Sdes
1692117610Sdes	mlx5e_build_common_cq_param(priv, param);
1693117610Sdes}
1694117610Sdes
1695117610Sdesstatic void
1696117610Sdesmlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1697117610Sdes    struct mlx5e_cq_param *param)
1698117610Sdes{
1699117610Sdes	void *cqc = param->cqc;
1700117610Sdes
1701117610Sdes	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
1702117610Sdes	MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
1703117610Sdes	MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
1704117610Sdes
1705117610Sdes	mlx5e_build_common_cq_param(priv, param);
1706117610Sdes}
1707117610Sdes
1708117610Sdesstatic void
1709117610Sdesmlx5e_build_channel_param(struct mlx5e_priv *priv,
1710117610Sdes    struct mlx5e_channel_param *cparam)
1711117610Sdes{
1712255376Sdes	memset(cparam, 0, sizeof(*cparam));
1713117610Sdes
1714117610Sdes	mlx5e_build_rq_param(priv, &cparam->rq);
1715117610Sdes	mlx5e_build_sq_param(priv, &cparam->sq);
1716117610Sdes	mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1717117610Sdes	mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
1718117610Sdes}
1719117610Sdes
1720117610Sdesstatic int
1721117610Sdesmlx5e_open_channels(struct mlx5e_priv *priv)
1722117610Sdes{
1723117610Sdes	struct mlx5e_channel_param cparam;
1724117610Sdes	void *ptr;
1725117610Sdes	int err;
1726117610Sdes	int i;
1727117610Sdes	int j;
1728117610Sdes
1729117610Sdes	priv->channel = malloc(priv->params.num_channels *
1730117610Sdes	    sizeof(struct mlx5e_channel *), M_MLX5EN, M_WAITOK | M_ZERO);
1731117610Sdes	if (priv->channel == NULL)
1732117610Sdes		return (-ENOMEM);
1733117610Sdes
1734117610Sdes	mlx5e_build_channel_param(priv, &cparam);
1735117610Sdes	for (i = 0; i < priv->params.num_channels; i++) {
1736117610Sdes		err = mlx5e_open_channel(priv, i, &cparam, &priv->channel[i]);
1737117610Sdes		if (err)
1738117610Sdes			goto err_close_channels;
1739117610Sdes	}
1740117610Sdes
1741117610Sdes	for (j = 0; j < priv->params.num_channels; j++) {
1742255376Sdes		err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1743255376Sdes		if (err)
1744255376Sdes			goto err_close_channels;
1745117610Sdes	}
1746117610Sdes
1747117610Sdes	return (0);
1748255376Sdes
1749255376Sdeserr_close_channels:
1750255376Sdes	for (i--; i >= 0; i--) {
1751117610Sdes		mlx5e_close_channel(&priv->channel[i]);
1752117610Sdes		mlx5e_close_channel_wait(&priv->channel[i]);
1753117610Sdes	}
1754117610Sdes
1755117610Sdes	/* remove "volatile" attribute from "channel" pointer */
1756117610Sdes	ptr = __DECONST(void *, priv->channel);
1757117610Sdes	priv->channel = NULL;
1758117610Sdes
1759117610Sdes	free(ptr, M_MLX5EN);
1760117610Sdes
1761117610Sdes	return (err);
1762117610Sdes}
1763117610Sdes
1764117610Sdesstatic void
1765117610Sdesmlx5e_close_channels(struct mlx5e_priv *priv)
1766117610Sdes{
1767117610Sdes	void *ptr;
1768117610Sdes	int i;
1769117610Sdes
1770117610Sdes	if (priv->channel == NULL)
1771117610Sdes		return;
1772117610Sdes
1773117610Sdes	for (i = 0; i < priv->params.num_channels; i++)
1774117610Sdes		mlx5e_close_channel(&priv->channel[i]);
1775255376Sdes	for (i = 0; i < priv->params.num_channels; i++)
1776117610Sdes		mlx5e_close_channel_wait(&priv->channel[i]);
1777117610Sdes
1778117610Sdes	/* remove "volatile" attribute from "channel" pointer */
1779117610Sdes	ptr = __DECONST(void *, priv->channel);
1780117610Sdes	priv->channel = NULL;
1781117610Sdes
1782117610Sdes	free(ptr, M_MLX5EN);
1783}
1784
1785static int
1786mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
1787{
1788	return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
1789	    priv->params.tx_cq_moderation_usec,
1790	    priv->params.tx_cq_moderation_pkts));
1791}
1792
1793static int
1794mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
1795{
1796	return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
1797	    priv->params.rx_cq_moderation_usec,
1798	    priv->params.rx_cq_moderation_pkts));
1799}
1800
1801static int
1802mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1803{
1804	int err;
1805	int i;
1806
1807	if (c == NULL)
1808		return (EINVAL);
1809
1810	err = mlx5e_refresh_rq_params(priv, &c->rq);
1811	if (err)
1812		goto done;
1813
1814	for (i = 0; i != c->num_tc; i++) {
1815		err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
1816		if (err)
1817			goto done;
1818	}
1819done:
1820	return (err);
1821}
1822
1823int
1824mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
1825{
1826	int i;
1827
1828	if (priv->channel == NULL)
1829		return (EINVAL);
1830
1831	for (i = 0; i < priv->params.num_channels; i++) {
1832		int err;
1833
1834		err = mlx5e_refresh_channel_params_sub(priv, priv->channel[i]);
1835		if (err)
1836			return (err);
1837	}
1838	return (0);
1839}
1840
1841static int
1842mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
1843{
1844	struct mlx5_core_dev *mdev = priv->mdev;
1845	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
1846	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1847
1848	memset(in, 0, sizeof(in));
1849
1850	MLX5_SET(tisc, tisc, prio, tc);
1851	MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
1852
1853	return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
1854}
1855
1856static void
1857mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
1858{
1859	mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
1860}
1861
1862static int
1863mlx5e_open_tises(struct mlx5e_priv *priv)
1864{
1865	int num_tc = priv->num_tc;
1866	int err;
1867	int tc;
1868
1869	for (tc = 0; tc < num_tc; tc++) {
1870		err = mlx5e_open_tis(priv, tc);
1871		if (err)
1872			goto err_close_tises;
1873	}
1874
1875	return (0);
1876
1877err_close_tises:
1878	for (tc--; tc >= 0; tc--)
1879		mlx5e_close_tis(priv, tc);
1880
1881	return (err);
1882}
1883
1884static void
1885mlx5e_close_tises(struct mlx5e_priv *priv)
1886{
1887	int num_tc = priv->num_tc;
1888	int tc;
1889
1890	for (tc = 0; tc < num_tc; tc++)
1891		mlx5e_close_tis(priv, tc);
1892}
1893
1894static int
1895mlx5e_open_rqt(struct mlx5e_priv *priv)
1896{
1897	struct mlx5_core_dev *mdev = priv->mdev;
1898	u32 *in;
1899	u32 out[MLX5_ST_SZ_DW(create_rqt_out)];
1900	void *rqtc;
1901	int inlen;
1902	int err;
1903	int sz;
1904	int i;
1905
1906	sz = 1 << priv->params.rx_hash_log_tbl_sz;
1907
1908	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1909	in = mlx5_vzalloc(inlen);
1910	if (in == NULL)
1911		return (-ENOMEM);
1912	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1913
1914	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1915	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1916
1917	for (i = 0; i < sz; i++) {
1918		int ix;
1919#ifdef RSS
1920		ix = rss_get_indirection_to_bucket(i);
1921#else
1922		ix = i;
1923#endif
1924		/* ensure we don't overflow */
1925		ix %= priv->params.num_channels;
1926		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn);
1927	}
1928
1929	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
1930
1931	memset(out, 0, sizeof(out));
1932	err = mlx5_cmd_exec_check_status(mdev, in, inlen, out, sizeof(out));
1933	if (!err)
1934		priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn);
1935
1936	kvfree(in);
1937
1938	return (err);
1939}
1940
1941static void
1942mlx5e_close_rqt(struct mlx5e_priv *priv)
1943{
1944	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)];
1945	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)];
1946
1947	memset(in, 0, sizeof(in));
1948
1949	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
1950	MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn);
1951
1952	mlx5_cmd_exec_check_status(priv->mdev, in, sizeof(in), out,
1953	    sizeof(out));
1954}
1955
1956static void
1957mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
1958{
1959	void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1960	__be32 *hkey;
1961
1962	MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
1963
1964#define	ROUGH_MAX_L2_L3_HDR_SZ 256
1965
1966#define	MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
1967			  MLX5_HASH_FIELD_SEL_DST_IP)
1968
1969#define	MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
1970			  MLX5_HASH_FIELD_SEL_DST_IP   |\
1971			  MLX5_HASH_FIELD_SEL_L4_SPORT |\
1972			  MLX5_HASH_FIELD_SEL_L4_DPORT)
1973
1974#define	MLX5_HASH_IP_IPSEC_SPI	(MLX5_HASH_FIELD_SEL_SRC_IP   |\
1975				 MLX5_HASH_FIELD_SEL_DST_IP   |\
1976				 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
1977
1978	if (priv->params.hw_lro_en) {
1979		MLX5_SET(tirc, tirc, lro_enable_mask,
1980		    MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
1981		    MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
1982		MLX5_SET(tirc, tirc, lro_max_msg_sz,
1983		    (priv->params.lro_wqe_sz -
1984		    ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
1985		/* TODO: add the option to choose timer value dynamically */
1986		MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
1987		    MLX5_CAP_ETH(priv->mdev,
1988		    lro_timer_supported_periods[2]));
1989	}
1990
1991	/* setup parameters for hashing TIR type, if any */
1992	switch (tt) {
1993	case MLX5E_TT_ANY:
1994		MLX5_SET(tirc, tirc, disp_type,
1995		    MLX5_TIRC_DISP_TYPE_DIRECT);
1996		MLX5_SET(tirc, tirc, inline_rqn,
1997		    priv->channel[0]->rq.rqn);
1998		break;
1999	default:
2000		MLX5_SET(tirc, tirc, disp_type,
2001		    MLX5_TIRC_DISP_TYPE_INDIRECT);
2002		MLX5_SET(tirc, tirc, indirect_table,
2003		    priv->rqtn);
2004		MLX5_SET(tirc, tirc, rx_hash_fn,
2005		    MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
2006		hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
2007#ifdef RSS
2008		/*
2009		 * The FreeBSD RSS implementation does currently not
2010		 * support symmetric Toeplitz hashes:
2011		 */
2012		MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
2013		rss_getkey((uint8_t *)hkey);
2014#else
2015		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2016		hkey[0] = cpu_to_be32(0xD181C62C);
2017		hkey[1] = cpu_to_be32(0xF7F4DB5B);
2018		hkey[2] = cpu_to_be32(0x1983A2FC);
2019		hkey[3] = cpu_to_be32(0x943E1ADB);
2020		hkey[4] = cpu_to_be32(0xD9389E6B);
2021		hkey[5] = cpu_to_be32(0xD1039C2C);
2022		hkey[6] = cpu_to_be32(0xA74499AD);
2023		hkey[7] = cpu_to_be32(0x593D56D9);
2024		hkey[8] = cpu_to_be32(0xF3253C06);
2025		hkey[9] = cpu_to_be32(0x2ADC1FFC);
2026#endif
2027		break;
2028	}
2029
2030	switch (tt) {
2031	case MLX5E_TT_IPV4_TCP:
2032		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2033		    MLX5_L3_PROT_TYPE_IPV4);
2034		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2035		    MLX5_L4_PROT_TYPE_TCP);
2036#ifdef RSS
2037		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
2038			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2039			    MLX5_HASH_IP);
2040		} else
2041#endif
2042		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2043		    MLX5_HASH_ALL);
2044		break;
2045
2046	case MLX5E_TT_IPV6_TCP:
2047		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2048		    MLX5_L3_PROT_TYPE_IPV6);
2049		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2050		    MLX5_L4_PROT_TYPE_TCP);
2051#ifdef RSS
2052		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
2053			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2054			    MLX5_HASH_IP);
2055		} else
2056#endif
2057		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2058		    MLX5_HASH_ALL);
2059		break;
2060
2061	case MLX5E_TT_IPV4_UDP:
2062		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2063		    MLX5_L3_PROT_TYPE_IPV4);
2064		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2065		    MLX5_L4_PROT_TYPE_UDP);
2066#ifdef RSS
2067		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
2068			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2069			    MLX5_HASH_IP);
2070		} else
2071#endif
2072		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2073		    MLX5_HASH_ALL);
2074		break;
2075
2076	case MLX5E_TT_IPV6_UDP:
2077		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2078		    MLX5_L3_PROT_TYPE_IPV6);
2079		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2080		    MLX5_L4_PROT_TYPE_UDP);
2081#ifdef RSS
2082		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
2083			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2084			    MLX5_HASH_IP);
2085		} else
2086#endif
2087		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2088		    MLX5_HASH_ALL);
2089		break;
2090
2091	case MLX5E_TT_IPV4_IPSEC_AH:
2092		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2093		    MLX5_L3_PROT_TYPE_IPV4);
2094		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2095		    MLX5_HASH_IP_IPSEC_SPI);
2096		break;
2097
2098	case MLX5E_TT_IPV6_IPSEC_AH:
2099		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2100		    MLX5_L3_PROT_TYPE_IPV6);
2101		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2102		    MLX5_HASH_IP_IPSEC_SPI);
2103		break;
2104
2105	case MLX5E_TT_IPV4_IPSEC_ESP:
2106		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2107		    MLX5_L3_PROT_TYPE_IPV4);
2108		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2109		    MLX5_HASH_IP_IPSEC_SPI);
2110		break;
2111
2112	case MLX5E_TT_IPV6_IPSEC_ESP:
2113		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2114		    MLX5_L3_PROT_TYPE_IPV6);
2115		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2116		    MLX5_HASH_IP_IPSEC_SPI);
2117		break;
2118
2119	case MLX5E_TT_IPV4:
2120		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2121		    MLX5_L3_PROT_TYPE_IPV4);
2122		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2123		    MLX5_HASH_IP);
2124		break;
2125
2126	case MLX5E_TT_IPV6:
2127		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2128		    MLX5_L3_PROT_TYPE_IPV6);
2129		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2130		    MLX5_HASH_IP);
2131		break;
2132
2133	default:
2134		break;
2135	}
2136}
2137
2138static int
2139mlx5e_open_tir(struct mlx5e_priv *priv, int tt)
2140{
2141	struct mlx5_core_dev *mdev = priv->mdev;
2142	u32 *in;
2143	void *tirc;
2144	int inlen;
2145	int err;
2146
2147	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2148	in = mlx5_vzalloc(inlen);
2149	if (in == NULL)
2150		return (-ENOMEM);
2151	tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
2152
2153	mlx5e_build_tir_ctx(priv, tirc, tt);
2154
2155	err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]);
2156
2157	kvfree(in);
2158
2159	return (err);
2160}
2161
2162static void
2163mlx5e_close_tir(struct mlx5e_priv *priv, int tt)
2164{
2165	mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]);
2166}
2167
2168static int
2169mlx5e_open_tirs(struct mlx5e_priv *priv)
2170{
2171	int err;
2172	int i;
2173
2174	for (i = 0; i < MLX5E_NUM_TT; i++) {
2175		err = mlx5e_open_tir(priv, i);
2176		if (err)
2177			goto err_close_tirs;
2178	}
2179
2180	return (0);
2181
2182err_close_tirs:
2183	for (i--; i >= 0; i--)
2184		mlx5e_close_tir(priv, i);
2185
2186	return (err);
2187}
2188
2189static void
2190mlx5e_close_tirs(struct mlx5e_priv *priv)
2191{
2192	int i;
2193
2194	for (i = 0; i < MLX5E_NUM_TT; i++)
2195		mlx5e_close_tir(priv, i);
2196}
2197
2198/*
2199 * SW MTU does not include headers,
2200 * HW MTU includes all headers and checksums.
2201 */
2202static int
2203mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu)
2204{
2205	struct mlx5e_priv *priv = ifp->if_softc;
2206	struct mlx5_core_dev *mdev = priv->mdev;
2207	int hw_mtu;
2208	int err;
2209
2210
2211	err = mlx5_set_port_mtu(mdev, MLX5E_SW2HW_MTU(sw_mtu));
2212	if (err) {
2213		if_printf(ifp, "%s: mlx5_set_port_mtu failed setting %d, err=%d\n",
2214		    __func__, sw_mtu, err);
2215		return (err);
2216	}
2217	err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
2218	if (!err) {
2219		ifp->if_mtu = MLX5E_HW2SW_MTU(hw_mtu);
2220
2221		if (ifp->if_mtu != sw_mtu) {
2222			if_printf(ifp, "Port MTU %d is different than "
2223			    "ifp mtu %d\n", sw_mtu, (int)ifp->if_mtu);
2224		}
2225	} else {
2226		if_printf(ifp, "Query port MTU, after setting new "
2227		    "MTU value, failed\n");
2228		ifp->if_mtu = sw_mtu;
2229	}
2230	return (0);
2231}
2232
2233int
2234mlx5e_open_locked(struct ifnet *ifp)
2235{
2236	struct mlx5e_priv *priv = ifp->if_softc;
2237	int err;
2238	u16 set_id;
2239
2240	/* check if already opened */
2241	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
2242		return (0);
2243
2244#ifdef RSS
2245	if (rss_getnumbuckets() > priv->params.num_channels) {
2246		if_printf(ifp, "NOTE: There are more RSS buckets(%u) than "
2247		    "channels(%u) available\n", rss_getnumbuckets(),
2248		    priv->params.num_channels);
2249	}
2250#endif
2251	err = mlx5e_open_tises(priv);
2252	if (err) {
2253		if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n",
2254		    __func__, err);
2255		return (err);
2256	}
2257	err = mlx5_vport_alloc_q_counter(priv->mdev,
2258	    MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
2259	if (err) {
2260		if_printf(priv->ifp,
2261		    "%s: mlx5_vport_alloc_q_counter failed: %d\n",
2262		    __func__, err);
2263		goto err_close_tises;
2264	}
2265	/* store counter set ID */
2266	priv->counter_set_id = set_id;
2267
2268	err = mlx5e_open_channels(priv);
2269	if (err) {
2270		if_printf(ifp, "%s: mlx5e_open_channels failed, %d\n",
2271		    __func__, err);
2272		goto err_dalloc_q_counter;
2273	}
2274	err = mlx5e_open_rqt(priv);
2275	if (err) {
2276		if_printf(ifp, "%s: mlx5e_open_rqt failed, %d\n",
2277		    __func__, err);
2278		goto err_close_channels;
2279	}
2280	err = mlx5e_open_tirs(priv);
2281	if (err) {
2282		if_printf(ifp, "%s: mlx5e_open_tir failed, %d\n",
2283		    __func__, err);
2284		goto err_close_rqls;
2285	}
2286	err = mlx5e_open_flow_table(priv);
2287	if (err) {
2288		if_printf(ifp, "%s: mlx5e_open_flow_table failed, %d\n",
2289		    __func__, err);
2290		goto err_close_tirs;
2291	}
2292	err = mlx5e_add_all_vlan_rules(priv);
2293	if (err) {
2294		if_printf(ifp, "%s: mlx5e_add_all_vlan_rules failed, %d\n",
2295		    __func__, err);
2296		goto err_close_flow_table;
2297	}
2298	set_bit(MLX5E_STATE_OPENED, &priv->state);
2299
2300	mlx5e_update_carrier(priv);
2301	mlx5e_set_rx_mode_core(priv);
2302
2303	return (0);
2304
2305err_close_flow_table:
2306	mlx5e_close_flow_table(priv);
2307
2308err_close_tirs:
2309	mlx5e_close_tirs(priv);
2310
2311err_close_rqls:
2312	mlx5e_close_rqt(priv);
2313
2314err_close_channels:
2315	mlx5e_close_channels(priv);
2316
2317err_dalloc_q_counter:
2318	mlx5_vport_dealloc_q_counter(priv->mdev,
2319	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
2320
2321err_close_tises:
2322	mlx5e_close_tises(priv);
2323
2324	return (err);
2325}
2326
2327static void
2328mlx5e_open(void *arg)
2329{
2330	struct mlx5e_priv *priv = arg;
2331
2332	PRIV_LOCK(priv);
2333	if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
2334		if_printf(priv->ifp,
2335		    "%s: Setting port status to up failed\n",
2336		    __func__);
2337
2338	mlx5e_open_locked(priv->ifp);
2339	priv->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2340	PRIV_UNLOCK(priv);
2341}
2342
2343int
2344mlx5e_close_locked(struct ifnet *ifp)
2345{
2346	struct mlx5e_priv *priv = ifp->if_softc;
2347
2348	/* check if already closed */
2349	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2350		return (0);
2351
2352	clear_bit(MLX5E_STATE_OPENED, &priv->state);
2353
2354	mlx5e_set_rx_mode_core(priv);
2355	mlx5e_del_all_vlan_rules(priv);
2356	if_link_state_change(priv->ifp, LINK_STATE_DOWN);
2357	mlx5e_close_flow_table(priv);
2358	mlx5e_close_tirs(priv);
2359	mlx5e_close_rqt(priv);
2360	mlx5e_close_channels(priv);
2361	mlx5_vport_dealloc_q_counter(priv->mdev,
2362	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
2363	mlx5e_close_tises(priv);
2364
2365	return (0);
2366}
2367
2368#if (__FreeBSD_version >= 1100000)
2369static uint64_t
2370mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt)
2371{
2372	struct mlx5e_priv *priv = ifp->if_softc;
2373	u64 retval;
2374
2375	/* PRIV_LOCK(priv); XXX not allowed */
2376	switch (cnt) {
2377	case IFCOUNTER_IPACKETS:
2378		retval = priv->stats.vport.rx_packets;
2379		break;
2380	case IFCOUNTER_IERRORS:
2381		retval = priv->stats.vport.rx_error_packets;
2382		break;
2383	case IFCOUNTER_IQDROPS:
2384		retval = priv->stats.vport.rx_out_of_buffer;
2385		break;
2386	case IFCOUNTER_OPACKETS:
2387		retval = priv->stats.vport.tx_packets;
2388		break;
2389	case IFCOUNTER_OERRORS:
2390		retval = priv->stats.vport.tx_error_packets;
2391		break;
2392	case IFCOUNTER_IBYTES:
2393		retval = priv->stats.vport.rx_bytes;
2394		break;
2395	case IFCOUNTER_OBYTES:
2396		retval = priv->stats.vport.tx_bytes;
2397		break;
2398	case IFCOUNTER_IMCASTS:
2399		retval = priv->stats.vport.rx_multicast_packets;
2400		break;
2401	case IFCOUNTER_OMCASTS:
2402		retval = priv->stats.vport.tx_multicast_packets;
2403		break;
2404	case IFCOUNTER_OQDROPS:
2405		retval = priv->stats.vport.tx_queue_dropped;
2406		break;
2407	default:
2408		retval = if_get_counter_default(ifp, cnt);
2409		break;
2410	}
2411	/* PRIV_UNLOCK(priv); XXX not allowed */
2412	return (retval);
2413}
2414#endif
2415
2416static void
2417mlx5e_set_rx_mode(struct ifnet *ifp)
2418{
2419	struct mlx5e_priv *priv = ifp->if_softc;
2420
2421	schedule_work(&priv->set_rx_mode_work);
2422}
2423
2424static int
2425mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
2426{
2427	struct mlx5e_priv *priv;
2428	struct ifreq *ifr;
2429	struct ifi2creq i2c;
2430	int error = 0;
2431	int mask = 0;
2432	int size_read = 0;
2433	int module_num;
2434	int max_mtu;
2435	uint8_t read_addr;
2436
2437	priv = ifp->if_softc;
2438
2439	/* check if detaching */
2440	if (priv == NULL || priv->gone != 0)
2441		return (ENXIO);
2442
2443	switch (command) {
2444	case SIOCSIFMTU:
2445		ifr = (struct ifreq *)data;
2446
2447		PRIV_LOCK(priv);
2448		mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
2449
2450		if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
2451		    ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
2452			int was_opened;
2453
2454			was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2455			if (was_opened)
2456				mlx5e_close_locked(ifp);
2457
2458			/* set new MTU */
2459			mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
2460
2461			if (was_opened)
2462				mlx5e_open_locked(ifp);
2463		} else {
2464			error = EINVAL;
2465			if_printf(ifp, "Invalid MTU value. Min val: %d, Max val: %d\n",
2466			    MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
2467		}
2468		PRIV_UNLOCK(priv);
2469		break;
2470	case SIOCSIFFLAGS:
2471		if ((ifp->if_flags & IFF_UP) &&
2472		    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2473			mlx5e_set_rx_mode(ifp);
2474			break;
2475		}
2476		PRIV_LOCK(priv);
2477		if (ifp->if_flags & IFF_UP) {
2478			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2479				if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2480					mlx5e_open_locked(ifp);
2481				ifp->if_drv_flags |= IFF_DRV_RUNNING;
2482				mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
2483			}
2484		} else {
2485			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2486				mlx5_set_port_status(priv->mdev,
2487				    MLX5_PORT_DOWN);
2488				if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
2489					mlx5e_close_locked(ifp);
2490				mlx5e_update_carrier(priv);
2491				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2492			}
2493		}
2494		PRIV_UNLOCK(priv);
2495		break;
2496	case SIOCADDMULTI:
2497	case SIOCDELMULTI:
2498		mlx5e_set_rx_mode(ifp);
2499		break;
2500	case SIOCSIFMEDIA:
2501	case SIOCGIFMEDIA:
2502	case SIOCGIFXMEDIA:
2503		ifr = (struct ifreq *)data;
2504		error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
2505		break;
2506	case SIOCSIFCAP:
2507		ifr = (struct ifreq *)data;
2508		PRIV_LOCK(priv);
2509		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2510
2511		if (mask & IFCAP_TXCSUM) {
2512			ifp->if_capenable ^= IFCAP_TXCSUM;
2513			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
2514
2515			if (IFCAP_TSO4 & ifp->if_capenable &&
2516			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
2517				ifp->if_capenable &= ~IFCAP_TSO4;
2518				ifp->if_hwassist &= ~CSUM_IP_TSO;
2519				if_printf(ifp,
2520				    "tso4 disabled due to -txcsum.\n");
2521			}
2522		}
2523		if (mask & IFCAP_TXCSUM_IPV6) {
2524			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
2525			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
2526
2527			if (IFCAP_TSO6 & ifp->if_capenable &&
2528			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
2529				ifp->if_capenable &= ~IFCAP_TSO6;
2530				ifp->if_hwassist &= ~CSUM_IP6_TSO;
2531				if_printf(ifp,
2532				    "tso6 disabled due to -txcsum6.\n");
2533			}
2534		}
2535		if (mask & IFCAP_RXCSUM)
2536			ifp->if_capenable ^= IFCAP_RXCSUM;
2537		if (mask & IFCAP_RXCSUM_IPV6)
2538			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
2539		if (mask & IFCAP_TSO4) {
2540			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
2541			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
2542				if_printf(ifp, "enable txcsum first.\n");
2543				error = EAGAIN;
2544				goto out;
2545			}
2546			ifp->if_capenable ^= IFCAP_TSO4;
2547			ifp->if_hwassist ^= CSUM_IP_TSO;
2548		}
2549		if (mask & IFCAP_TSO6) {
2550			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
2551			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
2552				if_printf(ifp, "enable txcsum6 first.\n");
2553				error = EAGAIN;
2554				goto out;
2555			}
2556			ifp->if_capenable ^= IFCAP_TSO6;
2557			ifp->if_hwassist ^= CSUM_IP6_TSO;
2558		}
2559		if (mask & IFCAP_VLAN_HWFILTER) {
2560			if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2561				mlx5e_disable_vlan_filter(priv);
2562			else
2563				mlx5e_enable_vlan_filter(priv);
2564
2565			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
2566		}
2567		if (mask & IFCAP_VLAN_HWTAGGING)
2568			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2569		if (mask & IFCAP_WOL_MAGIC)
2570			ifp->if_capenable ^= IFCAP_WOL_MAGIC;
2571
2572		VLAN_CAPABILITIES(ifp);
2573		/* turn off LRO means also turn of HW LRO - if it's on */
2574		if (mask & IFCAP_LRO) {
2575			int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2576			bool need_restart = false;
2577
2578			ifp->if_capenable ^= IFCAP_LRO;
2579			if (!(ifp->if_capenable & IFCAP_LRO)) {
2580				if (priv->params.hw_lro_en) {
2581					priv->params.hw_lro_en = false;
2582					need_restart = true;
2583					/* Not sure this is the correct way */
2584					priv->params_ethtool.hw_lro = priv->params.hw_lro_en;
2585				}
2586			}
2587			if (was_opened && need_restart) {
2588				mlx5e_close_locked(ifp);
2589				mlx5e_open_locked(ifp);
2590			}
2591		}
2592out:
2593		PRIV_UNLOCK(priv);
2594		break;
2595
2596	case SIOCGI2C:
2597		ifr = (struct ifreq *)data;
2598
2599		/*
2600		 * Copy from the user-space address ifr_data to the
2601		 * kernel-space address i2c
2602		 */
2603		error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
2604		if (error)
2605			break;
2606
2607		if (i2c.len > sizeof(i2c.data)) {
2608			error = EINVAL;
2609			break;
2610		}
2611
2612		PRIV_LOCK(priv);
2613		/* Get module_num which is required for the query_eeprom */
2614		error = mlx5_query_module_num(priv->mdev, &module_num);
2615		if (error) {
2616			if_printf(ifp, "Query module num failed, eeprom "
2617			    "reading is not supported\n");
2618			error = EINVAL;
2619			goto err_i2c;
2620		}
2621		/* Check if module is present before doing an access */
2622		if (mlx5_query_module_status(priv->mdev, module_num) !=
2623		    MLX5_MODULE_STATUS_PLUGGED) {
2624			error = EINVAL;
2625			goto err_i2c;
2626		}
2627		/*
2628		 * Currently 0XA0 and 0xA2 are the only addresses permitted.
2629		 * The internal conversion is as follows:
2630		 */
2631		if (i2c.dev_addr == 0xA0)
2632			read_addr = MLX5E_I2C_ADDR_LOW;
2633		else if (i2c.dev_addr == 0xA2)
2634			read_addr = MLX5E_I2C_ADDR_HIGH;
2635		else {
2636			if_printf(ifp, "Query eeprom failed, "
2637			    "Invalid Address: %X\n", i2c.dev_addr);
2638			error = EINVAL;
2639			goto err_i2c;
2640		}
2641		error = mlx5_query_eeprom(priv->mdev,
2642		    read_addr, MLX5E_EEPROM_LOW_PAGE,
2643		    (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
2644		    (uint32_t *)i2c.data, &size_read);
2645		if (error) {
2646			if_printf(ifp, "Query eeprom failed, eeprom "
2647			    "reading is not supported\n");
2648			error = EINVAL;
2649			goto err_i2c;
2650		}
2651
2652		if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
2653			error = mlx5_query_eeprom(priv->mdev,
2654			    read_addr, MLX5E_EEPROM_LOW_PAGE,
2655			    (uint32_t)(i2c.offset + size_read),
2656			    (uint32_t)(i2c.len - size_read), module_num,
2657			    (uint32_t *)(i2c.data + size_read), &size_read);
2658		}
2659		if (error) {
2660			if_printf(ifp, "Query eeprom failed, eeprom "
2661			    "reading is not supported\n");
2662			error = EINVAL;
2663			goto err_i2c;
2664		}
2665
2666		error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
2667err_i2c:
2668		PRIV_UNLOCK(priv);
2669		break;
2670
2671	default:
2672		error = ether_ioctl(ifp, command, data);
2673		break;
2674	}
2675	return (error);
2676}
2677
2678static int
2679mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
2680{
2681	/*
2682	 * TODO: uncoment once FW really sets all these bits if
2683	 * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
2684	 * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
2685	 * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
2686	 * -ENOTSUPP;
2687	 */
2688
2689	/* TODO: add more must-to-have features */
2690
2691	return (0);
2692}
2693
2694static void
2695mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
2696    struct mlx5e_priv *priv,
2697    int num_comp_vectors)
2698{
2699	/*
2700	 * TODO: Consider link speed for setting "log_sq_size",
2701	 * "log_rq_size" and "cq_moderation_xxx":
2702	 */
2703	priv->params.log_sq_size =
2704	    MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
2705	priv->params.log_rq_size =
2706	    MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
2707	priv->params.rx_cq_moderation_usec =
2708	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
2709	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
2710	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
2711	priv->params.rx_cq_moderation_mode =
2712	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
2713	priv->params.rx_cq_moderation_pkts =
2714	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
2715	priv->params.tx_cq_moderation_usec =
2716	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
2717	priv->params.tx_cq_moderation_pkts =
2718	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
2719	priv->params.min_rx_wqes =
2720	    MLX5E_PARAMS_DEFAULT_MIN_RX_WQES;
2721	priv->params.rx_hash_log_tbl_sz =
2722	    (order_base_2(num_comp_vectors) >
2723	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
2724	    order_base_2(num_comp_vectors) :
2725	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
2726	priv->params.num_tc = 1;
2727	priv->params.default_vlan_prio = 0;
2728	priv->counter_set_id = -1;
2729
2730	/*
2731	 * hw lro is currently defaulted to off. when it won't anymore we
2732	 * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
2733	 */
2734	priv->params.hw_lro_en = false;
2735	priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
2736
2737	priv->params.cqe_zipping_en = !!MLX5_CAP_GEN(mdev, cqe_compression);
2738
2739	priv->mdev = mdev;
2740	priv->params.num_channels = num_comp_vectors;
2741	priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
2742	priv->queue_mapping_channel_mask =
2743	    roundup_pow_of_two(num_comp_vectors) - 1;
2744	priv->num_tc = priv->params.num_tc;
2745	priv->default_vlan_prio = priv->params.default_vlan_prio;
2746
2747	INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
2748	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
2749	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
2750}
2751
2752static int
2753mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
2754    struct mlx5_core_mr *mr)
2755{
2756	struct ifnet *ifp = priv->ifp;
2757	struct mlx5_core_dev *mdev = priv->mdev;
2758	struct mlx5_create_mkey_mbox_in *in;
2759	int err;
2760
2761	in = mlx5_vzalloc(sizeof(*in));
2762	if (in == NULL) {
2763		if_printf(ifp, "%s: failed to allocate inbox\n", __func__);
2764		return (-ENOMEM);
2765	}
2766	in->seg.flags = MLX5_PERM_LOCAL_WRITE |
2767	    MLX5_PERM_LOCAL_READ |
2768	    MLX5_ACCESS_MODE_PA;
2769	in->seg.flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
2770	in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
2771
2772	err = mlx5_core_create_mkey(mdev, mr, in, sizeof(*in), NULL, NULL,
2773	    NULL);
2774	if (err)
2775		if_printf(ifp, "%s: mlx5_core_create_mkey failed, %d\n",
2776		    __func__, err);
2777
2778	kvfree(in);
2779
2780	return (err);
2781}
2782
2783static const char *mlx5e_vport_stats_desc[] = {
2784	MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
2785};
2786
2787static const char *mlx5e_pport_stats_desc[] = {
2788	MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
2789};
2790
2791static void
2792mlx5e_priv_mtx_init(struct mlx5e_priv *priv)
2793{
2794	mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
2795	sx_init(&priv->state_lock, "mlx5state");
2796	callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
2797}
2798
2799static void
2800mlx5e_priv_mtx_destroy(struct mlx5e_priv *priv)
2801{
2802	mtx_destroy(&priv->async_events_mtx);
2803	sx_destroy(&priv->state_lock);
2804}
2805
2806static int
2807sysctl_firmware(SYSCTL_HANDLER_ARGS)
2808{
2809	/*
2810	 * %d.%d%.d the string format.
2811	 * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
2812	 * We need at most 5 chars to store that.
2813	 * It also has: two "." and NULL at the end, which means we need 18
2814	 * (5*3 + 3) chars at most.
2815	 */
2816	char fw[18];
2817	struct mlx5e_priv *priv = arg1;
2818	int error;
2819
2820	snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
2821	    fw_rev_sub(priv->mdev));
2822	error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
2823	return (error);
2824}
2825
2826static void
2827mlx5e_add_hw_stats(struct mlx5e_priv *priv)
2828{
2829	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
2830	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, priv, 0,
2831	    sysctl_firmware, "A", "HCA firmware version");
2832
2833	SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
2834	    OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
2835	    "Board ID");
2836}
2837
2838static void
2839mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
2840{
2841#if (__FreeBSD_version < 1100000)
2842	char path[64];
2843
2844#endif
2845	/* Only receiving pauseframes is enabled by default */
2846	priv->params.tx_pauseframe_control = 0;
2847	priv->params.rx_pauseframe_control = 1;
2848
2849#if (__FreeBSD_version < 1100000)
2850	/* compute path for sysctl */
2851	snprintf(path, sizeof(path), "dev.mce.%d.tx_pauseframe_control",
2852	    device_get_unit(priv->mdev->pdev->dev.bsddev));
2853
2854	/* try to fetch tunable, if any */
2855	TUNABLE_INT_FETCH(path, &priv->params.tx_pauseframe_control);
2856
2857	/* compute path for sysctl */
2858	snprintf(path, sizeof(path), "dev.mce.%d.rx_pauseframe_control",
2859	    device_get_unit(priv->mdev->pdev->dev.bsddev));
2860
2861	/* try to fetch tunable, if any */
2862	TUNABLE_INT_FETCH(path, &priv->params.rx_pauseframe_control);
2863#endif
2864
2865	/* register pausframe SYSCTLs */
2866	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
2867	    OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
2868	    &priv->params.tx_pauseframe_control, 0,
2869	    "Set to enable TX pause frames. Clear to disable.");
2870
2871	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
2872	    OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
2873	    &priv->params.rx_pauseframe_control, 0,
2874	    "Set to enable RX pause frames. Clear to disable.");
2875
2876	/* range check */
2877	priv->params.tx_pauseframe_control =
2878	    priv->params.tx_pauseframe_control ? 1 : 0;
2879	priv->params.rx_pauseframe_control =
2880	    priv->params.rx_pauseframe_control ? 1 : 0;
2881
2882	/* update firmware */
2883	mlx5_set_port_pause(priv->mdev, 1,
2884	    priv->params.rx_pauseframe_control,
2885	    priv->params.tx_pauseframe_control);
2886}
2887
2888static void *
2889mlx5e_create_ifp(struct mlx5_core_dev *mdev)
2890{
2891	static volatile int mlx5_en_unit;
2892	struct ifnet *ifp;
2893	struct mlx5e_priv *priv;
2894	u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
2895	struct sysctl_oid_list *child;
2896	int ncv = mdev->priv.eq_table.num_comp_vectors;
2897	char unit[16];
2898	int err;
2899	int i;
2900	u32 eth_proto_cap;
2901
2902	if (mlx5e_check_required_hca_cap(mdev)) {
2903		mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
2904		return (NULL);
2905	}
2906	priv = malloc(sizeof(*priv), M_MLX5EN, M_WAITOK | M_ZERO);
2907	if (priv == NULL) {
2908		mlx5_core_err(mdev, "malloc() failed\n");
2909		return (NULL);
2910	}
2911	mlx5e_priv_mtx_init(priv);
2912
2913	ifp = priv->ifp = if_alloc(IFT_ETHER);
2914	if (ifp == NULL) {
2915		mlx5_core_err(mdev, "if_alloc() failed\n");
2916		goto err_free_priv;
2917	}
2918	ifp->if_softc = priv;
2919	if_initname(ifp, "mce", atomic_fetchadd_int(&mlx5_en_unit, 1));
2920	ifp->if_mtu = ETHERMTU;
2921	ifp->if_init = mlx5e_open;
2922	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2923	ifp->if_ioctl = mlx5e_ioctl;
2924	ifp->if_transmit = mlx5e_xmit;
2925	ifp->if_qflush = if_qflush;
2926#if (__FreeBSD_version >= 1100000)
2927	ifp->if_get_counter = mlx5e_get_counter;
2928#endif
2929	ifp->if_snd.ifq_maxlen = ifqmaxlen;
2930	/*
2931         * Set driver features
2932         */
2933	ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6;
2934	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
2935	ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER;
2936	ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU;
2937	ifp->if_capabilities |= IFCAP_LRO;
2938	ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO;
2939
2940	/* set TSO limits so that we don't have to drop TX packets */
2941	ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2942	ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */;
2943	ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE;
2944
2945	ifp->if_capenable = ifp->if_capabilities;
2946	ifp->if_hwassist = 0;
2947	if (ifp->if_capenable & IFCAP_TSO)
2948		ifp->if_hwassist |= CSUM_TSO;
2949	if (ifp->if_capenable & IFCAP_TXCSUM)
2950		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP);
2951	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
2952		ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
2953
2954	/* ifnet sysctl tree */
2955	sysctl_ctx_init(&priv->sysctl_ctx);
2956	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
2957	    OID_AUTO, ifp->if_dname, CTLFLAG_RD, 0, "MLX5 ethernet - interface name");
2958	if (priv->sysctl_ifnet == NULL) {
2959		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
2960		goto err_free_sysctl;
2961	}
2962	snprintf(unit, sizeof(unit), "%d", ifp->if_dunit);
2963	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
2964	    OID_AUTO, unit, CTLFLAG_RD, 0, "MLX5 ethernet - interface unit");
2965	if (priv->sysctl_ifnet == NULL) {
2966		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
2967		goto err_free_sysctl;
2968	}
2969
2970	/* HW sysctl tree */
2971	child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
2972	priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
2973	    OID_AUTO, "hw", CTLFLAG_RD, 0, "MLX5 ethernet dev hw");
2974	if (priv->sysctl_hw == NULL) {
2975		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
2976		goto err_free_sysctl;
2977	}
2978	mlx5e_build_ifp_priv(mdev, priv, ncv);
2979	err = mlx5_alloc_map_uar(mdev, &priv->cq_uar);
2980	if (err) {
2981		if_printf(ifp, "%s: mlx5_alloc_map_uar failed, %d\n",
2982		    __func__, err);
2983		goto err_free_sysctl;
2984	}
2985	err = mlx5_core_alloc_pd(mdev, &priv->pdn);
2986	if (err) {
2987		if_printf(ifp, "%s: mlx5_core_alloc_pd failed, %d\n",
2988		    __func__, err);
2989		goto err_unmap_free_uar;
2990	}
2991	err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
2992	if (err) {
2993		if_printf(ifp, "%s: mlx5_alloc_transport_domain failed, %d\n",
2994		    __func__, err);
2995		goto err_dealloc_pd;
2996	}
2997	err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
2998	if (err) {
2999		if_printf(ifp, "%s: mlx5e_create_mkey failed, %d\n",
3000		    __func__, err);
3001		goto err_dealloc_transport_domain;
3002	}
3003	mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
3004
3005	/* check if we should generate a random MAC address */
3006	if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
3007	    is_zero_ether_addr(dev_addr)) {
3008		random_ether_addr(dev_addr);
3009		if_printf(ifp, "Assigned random MAC address\n");
3010	}
3011
3012	/* set default MTU */
3013	mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
3014
3015	/* Set desc */
3016	device_set_desc(mdev->pdev->dev.bsddev, mlx5e_version);
3017
3018	/* Set default media status */
3019	priv->media_status_last = IFM_AVALID;
3020	priv->media_active_last = IFM_ETHER | IFM_AUTO |
3021	    IFM_ETH_RXPAUSE | IFM_FDX;
3022
3023	/* setup default pauseframes configuration */
3024	mlx5e_setup_pauseframes(priv);
3025
3026	err = mlx5_query_port_proto_cap(mdev, &eth_proto_cap, MLX5_PTYS_EN);
3027	if (err) {
3028		eth_proto_cap = 0;
3029		if_printf(ifp, "%s: Query port media capability failed, %d\n",
3030		    __func__, err);
3031	}
3032
3033	/* Setup supported medias */
3034	ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK,
3035	    mlx5e_media_change, mlx5e_media_status);
3036
3037	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
3038		if (mlx5e_mode_table[i].baudrate == 0)
3039			continue;
3040		if (MLX5E_PROT_MASK(i) & eth_proto_cap) {
3041			ifmedia_add(&priv->media,
3042			    mlx5e_mode_table[i].subtype |
3043			    IFM_ETHER, 0, NULL);
3044			ifmedia_add(&priv->media,
3045			    mlx5e_mode_table[i].subtype |
3046			    IFM_ETHER | IFM_FDX |
3047			    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
3048		}
3049	}
3050
3051	ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3052	ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
3053	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
3054
3055	/* Set autoselect by default */
3056	ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
3057	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
3058	ether_ifattach(ifp, dev_addr);
3059
3060	/* Register for VLAN events */
3061	priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
3062	    mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
3063	priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
3064	    mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
3065
3066	/* Link is down by default */
3067	if_link_state_change(ifp, LINK_STATE_DOWN);
3068
3069	mlx5e_enable_async_events(priv);
3070
3071	mlx5e_add_hw_stats(priv);
3072
3073	mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3074	    "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
3075	    priv->stats.vport.arg);
3076
3077	mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3078	    "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
3079	    priv->stats.pport.arg);
3080
3081	mlx5e_create_ethtool(priv);
3082
3083	mtx_lock(&priv->async_events_mtx);
3084	mlx5e_update_stats(priv);
3085	mtx_unlock(&priv->async_events_mtx);
3086
3087	return (priv);
3088
3089err_dealloc_transport_domain:
3090	mlx5_dealloc_transport_domain(mdev, priv->tdn);
3091
3092err_dealloc_pd:
3093	mlx5_core_dealloc_pd(mdev, priv->pdn);
3094
3095err_unmap_free_uar:
3096	mlx5_unmap_free_uar(mdev, &priv->cq_uar);
3097
3098err_free_sysctl:
3099	sysctl_ctx_free(&priv->sysctl_ctx);
3100
3101	if_free(ifp);
3102
3103err_free_priv:
3104	mlx5e_priv_mtx_destroy(priv);
3105	free(priv, M_MLX5EN);
3106	return (NULL);
3107}
3108
3109static void
3110mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
3111{
3112	struct mlx5e_priv *priv = vpriv;
3113	struct ifnet *ifp = priv->ifp;
3114
3115	/* don't allow more IOCTLs */
3116	priv->gone = 1;
3117
3118	/*
3119	 * Clear the device description to avoid use after free,
3120	 * because the bsddev is not destroyed when this module is
3121	 * unloaded:
3122	 */
3123	device_set_desc(mdev->pdev->dev.bsddev, NULL);
3124
3125	/* XXX wait a bit to allow IOCTL handlers to complete */
3126	pause("W", hz);
3127
3128	/* stop watchdog timer */
3129	callout_drain(&priv->watchdog);
3130
3131	if (priv->vlan_attach != NULL)
3132		EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
3133	if (priv->vlan_detach != NULL)
3134		EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
3135
3136	/* make sure device gets closed */
3137	PRIV_LOCK(priv);
3138	mlx5e_close_locked(ifp);
3139	PRIV_UNLOCK(priv);
3140
3141	/* unregister device */
3142	ifmedia_removeall(&priv->media);
3143	ether_ifdetach(ifp);
3144	if_free(ifp);
3145
3146	/* destroy all remaining sysctl nodes */
3147	if (priv->sysctl_debug)
3148		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
3149	sysctl_ctx_free(&priv->stats.vport.ctx);
3150	sysctl_ctx_free(&priv->stats.pport.ctx);
3151	sysctl_ctx_free(&priv->sysctl_ctx);
3152
3153	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
3154	mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
3155	mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
3156	mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
3157	mlx5e_disable_async_events(priv);
3158	flush_scheduled_work();
3159	mlx5e_priv_mtx_destroy(priv);
3160	free(priv, M_MLX5EN);
3161}
3162
3163static void *
3164mlx5e_get_ifp(void *vpriv)
3165{
3166	struct mlx5e_priv *priv = vpriv;
3167
3168	return (priv->ifp);
3169}
3170
3171static struct mlx5_interface mlx5e_interface = {
3172	.add = mlx5e_create_ifp,
3173	.remove = mlx5e_destroy_ifp,
3174	.event = mlx5e_async_event,
3175	.protocol = MLX5_INTERFACE_PROTOCOL_ETH,
3176	.get_dev = mlx5e_get_ifp,
3177};
3178
3179void
3180mlx5e_init(void)
3181{
3182	mlx5_register_interface(&mlx5e_interface);
3183}
3184
3185void
3186mlx5e_cleanup(void)
3187{
3188	mlx5_unregister_interface(&mlx5e_interface);
3189}
3190
3191module_init_order(mlx5e_init, SI_ORDER_THIRD);
3192module_exit_order(mlx5e_cleanup, SI_ORDER_THIRD);
3193
3194#if (__FreeBSD_version >= 1100000)
3195MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
3196#endif
3197MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
3198MODULE_VERSION(mlx5en, 1);
3199