1// SPDX-License-Identifier: GPL-2.0-only
2/* Atlantic Network Driver
3 *
4 * Copyright (C) 2014-2019 aQuantia Corporation
5 * Copyright (C) 2019-2020 Marvell International Ltd.
6 */
7
8/* File aq_main.c: Main file for aQuantia Linux driver. */
9
10#include "aq_main.h"
11#include "aq_nic.h"
12#include "aq_pci_func.h"
13#include "aq_ethtool.h"
14#include "aq_ptp.h"
15#include "aq_filters.h"
16#include "aq_hw_utils.h"
17#include "aq_vec.h"
18
19#include <linux/netdevice.h>
20#include <linux/module.h>
21#include <linux/ip.h>
22#include <linux/udp.h>
23#include <net/pkt_cls.h>
24#include <net/pkt_sched.h>
25#include <linux/filter.h>
26
27MODULE_LICENSE("GPL v2");
28MODULE_AUTHOR(AQ_CFG_DRV_AUTHOR);
29MODULE_DESCRIPTION(AQ_CFG_DRV_DESC);
30
31DEFINE_STATIC_KEY_FALSE(aq_xdp_locking_key);
32EXPORT_SYMBOL(aq_xdp_locking_key);
33
34static const char aq_ndev_driver_name[] = AQ_CFG_DRV_NAME;
35
36static const struct net_device_ops aq_ndev_ops;
37
38static struct workqueue_struct *aq_ndev_wq;
39
40void aq_ndev_schedule_work(struct work_struct *work)
41{
42	queue_work(aq_ndev_wq, work);
43}
44
45struct net_device *aq_ndev_alloc(void)
46{
47	struct net_device *ndev = NULL;
48	struct aq_nic_s *aq_nic = NULL;
49
50	ndev = alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_HW_QUEUES_MAX);
51	if (!ndev)
52		return NULL;
53
54	aq_nic = netdev_priv(ndev);
55	aq_nic->ndev = ndev;
56	ndev->netdev_ops = &aq_ndev_ops;
57	ndev->ethtool_ops = &aq_ethtool_ops;
58
59	return ndev;
60}
61
62int aq_ndev_open(struct net_device *ndev)
63{
64	struct aq_nic_s *aq_nic = netdev_priv(ndev);
65	int err = 0;
66
67	err = aq_nic_init(aq_nic);
68	if (err < 0)
69		goto err_exit;
70
71	err = aq_reapply_rxnfc_all_rules(aq_nic);
72	if (err < 0)
73		goto err_exit;
74
75	err = aq_filters_vlans_update(aq_nic);
76	if (err < 0)
77		goto err_exit;
78
79	err = aq_nic_start(aq_nic);
80	if (err < 0) {
81		aq_nic_stop(aq_nic);
82		goto err_exit;
83	}
84
85err_exit:
86	if (err < 0)
87		aq_nic_deinit(aq_nic, true);
88
89	return err;
90}
91
92int aq_ndev_close(struct net_device *ndev)
93{
94	struct aq_nic_s *aq_nic = netdev_priv(ndev);
95	int err = 0;
96
97	err = aq_nic_stop(aq_nic);
98	aq_nic_deinit(aq_nic, true);
99
100	return err;
101}
102
103static netdev_tx_t aq_ndev_start_xmit(struct sk_buff *skb, struct net_device *ndev)
104{
105	struct aq_nic_s *aq_nic = netdev_priv(ndev);
106
107#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
108	if (unlikely(aq_utils_obj_test(&aq_nic->flags, AQ_NIC_PTP_DPATH_UP))) {
109		/* Hardware adds the Timestamp for PTPv2 802.AS1
110		 * and PTPv2 IPv4 UDP.
111		 * We have to push even general 320 port messages to the ptp
112		 * queue explicitly. This is a limitation of current firmware
113		 * and hardware PTP design of the chip. Otherwise ptp stream
114		 * will fail to sync
115		 */
116		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ||
117		    unlikely((ip_hdr(skb)->version == 4) &&
118			     (ip_hdr(skb)->protocol == IPPROTO_UDP) &&
119			     ((udp_hdr(skb)->dest == htons(319)) ||
120			      (udp_hdr(skb)->dest == htons(320)))) ||
121		    unlikely(eth_hdr(skb)->h_proto == htons(ETH_P_1588)))
122			return aq_ptp_xmit(aq_nic, skb);
123	}
124#endif
125
126	skb_tx_timestamp(skb);
127	return aq_nic_xmit(aq_nic, skb);
128}
129
130static int aq_ndev_change_mtu(struct net_device *ndev, int new_mtu)
131{
132	int new_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;
133	struct aq_nic_s *aq_nic = netdev_priv(ndev);
134	struct bpf_prog *prog;
135	int err;
136
137	prog = READ_ONCE(aq_nic->xdp_prog);
138	if (prog && !prog->aux->xdp_has_frags &&
139	    new_frame_size > AQ_CFG_RX_FRAME_MAX) {
140		netdev_err(ndev, "Illegal MTU %d for XDP prog without frags\n",
141			   ndev->mtu);
142		return -EOPNOTSUPP;
143	}
144
145	err = aq_nic_set_mtu(aq_nic, new_mtu + ETH_HLEN);
146
147	if (err < 0)
148		goto err_exit;
149	ndev->mtu = new_mtu;
150
151err_exit:
152	return err;
153}
154
155static int aq_ndev_set_features(struct net_device *ndev,
156				netdev_features_t features)
157{
158	bool is_vlan_tx_insert = !!(features & NETIF_F_HW_VLAN_CTAG_TX);
159	bool is_vlan_rx_strip = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
160	struct aq_nic_s *aq_nic = netdev_priv(ndev);
161	bool need_ndev_restart = false;
162	struct aq_nic_cfg_s *aq_cfg;
163	bool is_lro = false;
164	int err = 0;
165
166	aq_cfg = aq_nic_get_cfg(aq_nic);
167
168	if (!(features & NETIF_F_NTUPLE)) {
169		if (aq_nic->ndev->features & NETIF_F_NTUPLE) {
170			err = aq_clear_rxnfc_all_rules(aq_nic);
171			if (unlikely(err))
172				goto err_exit;
173		}
174	}
175	if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER)) {
176		if (aq_nic->ndev->features & NETIF_F_HW_VLAN_CTAG_FILTER) {
177			err = aq_filters_vlan_offload_off(aq_nic);
178			if (unlikely(err))
179				goto err_exit;
180		}
181	}
182
183	aq_cfg->features = features;
184
185	if (aq_cfg->aq_hw_caps->hw_features & NETIF_F_LRO) {
186		is_lro = features & NETIF_F_LRO;
187
188		if (aq_cfg->is_lro != is_lro) {
189			aq_cfg->is_lro = is_lro;
190			need_ndev_restart = true;
191		}
192	}
193
194	if ((aq_nic->ndev->features ^ features) & NETIF_F_RXCSUM) {
195		err = aq_nic->aq_hw_ops->hw_set_offload(aq_nic->aq_hw,
196							aq_cfg);
197
198		if (unlikely(err))
199			goto err_exit;
200	}
201
202	if (aq_cfg->is_vlan_rx_strip != is_vlan_rx_strip) {
203		aq_cfg->is_vlan_rx_strip = is_vlan_rx_strip;
204		need_ndev_restart = true;
205	}
206	if (aq_cfg->is_vlan_tx_insert != is_vlan_tx_insert) {
207		aq_cfg->is_vlan_tx_insert = is_vlan_tx_insert;
208		need_ndev_restart = true;
209	}
210
211	if (need_ndev_restart && netif_running(ndev)) {
212		aq_ndev_close(ndev);
213		aq_ndev_open(ndev);
214	}
215
216err_exit:
217	return err;
218}
219
220static netdev_features_t aq_ndev_fix_features(struct net_device *ndev,
221					      netdev_features_t features)
222{
223	struct aq_nic_s *aq_nic = netdev_priv(ndev);
224	struct bpf_prog *prog;
225
226	if (!(features & NETIF_F_RXCSUM))
227		features &= ~NETIF_F_LRO;
228
229	prog = READ_ONCE(aq_nic->xdp_prog);
230	if (prog && !prog->aux->xdp_has_frags &&
231	    aq_nic->xdp_prog && features & NETIF_F_LRO) {
232		netdev_err(ndev, "LRO is not supported with single buffer XDP, disabling\n");
233		features &= ~NETIF_F_LRO;
234	}
235
236	return features;
237}
238
239static int aq_ndev_set_mac_address(struct net_device *ndev, void *addr)
240{
241	struct aq_nic_s *aq_nic = netdev_priv(ndev);
242	int err = 0;
243
244	err = eth_mac_addr(ndev, addr);
245	if (err < 0)
246		goto err_exit;
247	err = aq_nic_set_mac(aq_nic, ndev);
248	if (err < 0)
249		goto err_exit;
250
251err_exit:
252	return err;
253}
254
255static void aq_ndev_set_multicast_settings(struct net_device *ndev)
256{
257	struct aq_nic_s *aq_nic = netdev_priv(ndev);
258
259	(void)aq_nic_set_multicast_list(aq_nic, ndev);
260}
261
262#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
263static int aq_ndev_config_hwtstamp(struct aq_nic_s *aq_nic,
264				   struct hwtstamp_config *config)
265{
266	switch (config->tx_type) {
267	case HWTSTAMP_TX_OFF:
268	case HWTSTAMP_TX_ON:
269		break;
270	default:
271		return -ERANGE;
272	}
273
274	switch (config->rx_filter) {
275	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
276	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
277	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
278	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
279	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
280	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
281	case HWTSTAMP_FILTER_PTP_V2_SYNC:
282	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
283		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
284		break;
285	case HWTSTAMP_FILTER_PTP_V2_EVENT:
286	case HWTSTAMP_FILTER_NONE:
287		break;
288	default:
289		return -ERANGE;
290	}
291
292	return aq_ptp_hwtstamp_config_set(aq_nic->aq_ptp, config);
293}
294#endif
295
296static int aq_ndev_hwtstamp_set(struct aq_nic_s *aq_nic, struct ifreq *ifr)
297{
298	struct hwtstamp_config config;
299#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
300	int ret_val;
301#endif
302
303	if (!aq_nic->aq_ptp)
304		return -EOPNOTSUPP;
305
306	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
307		return -EFAULT;
308#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
309	ret_val = aq_ndev_config_hwtstamp(aq_nic, &config);
310	if (ret_val)
311		return ret_val;
312#endif
313
314	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
315	       -EFAULT : 0;
316}
317
318#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
319static int aq_ndev_hwtstamp_get(struct aq_nic_s *aq_nic, struct ifreq *ifr)
320{
321	struct hwtstamp_config config;
322
323	if (!aq_nic->aq_ptp)
324		return -EOPNOTSUPP;
325
326	aq_ptp_hwtstamp_config_get(aq_nic->aq_ptp, &config);
327	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
328	       -EFAULT : 0;
329}
330#endif
331
332static int aq_ndev_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
333{
334	struct aq_nic_s *aq_nic = netdev_priv(netdev);
335
336	switch (cmd) {
337	case SIOCSHWTSTAMP:
338		return aq_ndev_hwtstamp_set(aq_nic, ifr);
339
340#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
341	case SIOCGHWTSTAMP:
342		return aq_ndev_hwtstamp_get(aq_nic, ifr);
343#endif
344	}
345
346	return -EOPNOTSUPP;
347}
348
349static int aq_ndo_vlan_rx_add_vid(struct net_device *ndev, __be16 proto,
350				  u16 vid)
351{
352	struct aq_nic_s *aq_nic = netdev_priv(ndev);
353
354	if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
355		return -EOPNOTSUPP;
356
357	set_bit(vid, aq_nic->active_vlans);
358
359	return aq_filters_vlans_update(aq_nic);
360}
361
362static int aq_ndo_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto,
363				   u16 vid)
364{
365	struct aq_nic_s *aq_nic = netdev_priv(ndev);
366
367	if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
368		return -EOPNOTSUPP;
369
370	clear_bit(vid, aq_nic->active_vlans);
371
372	if (-ENOENT == aq_del_fvlan_by_vlan(aq_nic, vid))
373		return aq_filters_vlans_update(aq_nic);
374
375	return 0;
376}
377
378static int aq_validate_mqprio_opt(struct aq_nic_s *self,
379				  struct tc_mqprio_qopt_offload *mqprio,
380				  const unsigned int num_tc)
381{
382	const bool has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
383	struct aq_nic_cfg_s *aq_nic_cfg = aq_nic_get_cfg(self);
384	const unsigned int tcs_max = min_t(u8, aq_nic_cfg->aq_hw_caps->tcs_max,
385					   AQ_CFG_TCS_MAX);
386
387	if (num_tc > tcs_max) {
388		netdev_err(self->ndev, "Too many TCs requested\n");
389		return -EOPNOTSUPP;
390	}
391
392	if (num_tc != 0 && !is_power_of_2(num_tc)) {
393		netdev_err(self->ndev, "TC count should be power of 2\n");
394		return -EOPNOTSUPP;
395	}
396
397	if (has_min_rate && !ATL_HW_IS_CHIP_FEATURE(self->aq_hw, ANTIGUA)) {
398		netdev_err(self->ndev, "Min tx rate is not supported\n");
399		return -EOPNOTSUPP;
400	}
401
402	return 0;
403}
404
405static int aq_ndo_setup_tc(struct net_device *dev, enum tc_setup_type type,
406			   void *type_data)
407{
408	struct tc_mqprio_qopt_offload *mqprio = type_data;
409	struct aq_nic_s *aq_nic = netdev_priv(dev);
410	bool has_min_rate;
411	bool has_max_rate;
412	int err;
413	int i;
414
415	if (type != TC_SETUP_QDISC_MQPRIO)
416		return -EOPNOTSUPP;
417
418	has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
419	has_max_rate = !!(mqprio->flags & TC_MQPRIO_F_MAX_RATE);
420
421	err = aq_validate_mqprio_opt(aq_nic, mqprio, mqprio->qopt.num_tc);
422	if (err)
423		return err;
424
425	for (i = 0; i < mqprio->qopt.num_tc; i++) {
426		if (has_max_rate) {
427			u64 max_rate = mqprio->max_rate[i];
428
429			do_div(max_rate, AQ_MBPS_DIVISOR);
430			aq_nic_setup_tc_max_rate(aq_nic, i, (u32)max_rate);
431		}
432
433		if (has_min_rate) {
434			u64 min_rate = mqprio->min_rate[i];
435
436			do_div(min_rate, AQ_MBPS_DIVISOR);
437			aq_nic_setup_tc_min_rate(aq_nic, i, (u32)min_rate);
438		}
439	}
440
441	return aq_nic_setup_tc_mqprio(aq_nic, mqprio->qopt.num_tc,
442				      mqprio->qopt.prio_tc_map);
443}
444
445static int aq_xdp_setup(struct net_device *ndev, struct bpf_prog *prog,
446			struct netlink_ext_ack *extack)
447{
448	bool need_update, running = netif_running(ndev);
449	struct aq_nic_s *aq_nic = netdev_priv(ndev);
450	struct bpf_prog *old_prog;
451
452	if (prog && !prog->aux->xdp_has_frags) {
453		if (ndev->mtu > AQ_CFG_RX_FRAME_MAX) {
454			NL_SET_ERR_MSG_MOD(extack,
455					   "prog does not support XDP frags");
456			return -EOPNOTSUPP;
457		}
458
459		if (prog && ndev->features & NETIF_F_LRO) {
460			netdev_err(ndev,
461				   "LRO is not supported with single buffer XDP, disabling\n");
462			ndev->features &= ~NETIF_F_LRO;
463		}
464	}
465
466	need_update = !!aq_nic->xdp_prog != !!prog;
467	if (running && need_update)
468		aq_ndev_close(ndev);
469
470	old_prog = xchg(&aq_nic->xdp_prog, prog);
471	if (old_prog)
472		bpf_prog_put(old_prog);
473
474	if (!old_prog && prog)
475		static_branch_inc(&aq_xdp_locking_key);
476	else if (old_prog && !prog)
477		static_branch_dec(&aq_xdp_locking_key);
478
479	if (running && need_update)
480		return aq_ndev_open(ndev);
481
482	return 0;
483}
484
485static int aq_xdp(struct net_device *dev, struct netdev_bpf *xdp)
486{
487	switch (xdp->command) {
488	case XDP_SETUP_PROG:
489		return aq_xdp_setup(dev, xdp->prog, xdp->extack);
490	default:
491		return -EINVAL;
492	}
493}
494
495static const struct net_device_ops aq_ndev_ops = {
496	.ndo_open = aq_ndev_open,
497	.ndo_stop = aq_ndev_close,
498	.ndo_start_xmit = aq_ndev_start_xmit,
499	.ndo_set_rx_mode = aq_ndev_set_multicast_settings,
500	.ndo_change_mtu = aq_ndev_change_mtu,
501	.ndo_set_mac_address = aq_ndev_set_mac_address,
502	.ndo_set_features = aq_ndev_set_features,
503	.ndo_fix_features = aq_ndev_fix_features,
504	.ndo_eth_ioctl = aq_ndev_ioctl,
505	.ndo_vlan_rx_add_vid = aq_ndo_vlan_rx_add_vid,
506	.ndo_vlan_rx_kill_vid = aq_ndo_vlan_rx_kill_vid,
507	.ndo_setup_tc = aq_ndo_setup_tc,
508	.ndo_bpf = aq_xdp,
509	.ndo_xdp_xmit = aq_xdp_xmit,
510};
511
512static int __init aq_ndev_init_module(void)
513{
514	int ret;
515
516	aq_ndev_wq = create_singlethread_workqueue(aq_ndev_driver_name);
517	if (!aq_ndev_wq) {
518		pr_err("Failed to create workqueue\n");
519		return -ENOMEM;
520	}
521
522	ret = aq_pci_func_register_driver();
523	if (ret) {
524		destroy_workqueue(aq_ndev_wq);
525		return ret;
526	}
527
528	return 0;
529}
530
531static void __exit aq_ndev_exit_module(void)
532{
533	aq_pci_func_unregister_driver();
534
535	if (aq_ndev_wq) {
536		destroy_workqueue(aq_ndev_wq);
537		aq_ndev_wq = NULL;
538	}
539}
540
541module_init(aq_ndev_init_module);
542module_exit(aq_ndev_exit_module);
543