1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kernel.h>
3#include <linux/init.h>
4#include <linux/module.h>
5#include <linux/netfilter.h>
6#include <linux/rhashtable.h>
7#include <linux/netdevice.h>
8#include <net/ip.h>
9#include <net/ip6_route.h>
10#include <net/netfilter/nf_tables.h>
11#include <net/netfilter/nf_flow_table.h>
12#include <net/netfilter/nf_conntrack.h>
13#include <net/netfilter/nf_conntrack_core.h>
14#include <net/netfilter/nf_conntrack_l4proto.h>
15#include <net/netfilter/nf_conntrack_tuple.h>
16
17static DEFINE_MUTEX(flowtable_lock);
18static LIST_HEAD(flowtables);
19
20static void
21flow_offload_fill_dir(struct flow_offload *flow,
22		      enum flow_offload_tuple_dir dir)
23{
24	struct flow_offload_tuple *ft = &flow->tuplehash[dir].tuple;
25	struct nf_conntrack_tuple *ctt = &flow->ct->tuplehash[dir].tuple;
26
27	ft->dir = dir;
28
29	switch (ctt->src.l3num) {
30	case NFPROTO_IPV4:
31		ft->src_v4 = ctt->src.u3.in;
32		ft->dst_v4 = ctt->dst.u3.in;
33		break;
34	case NFPROTO_IPV6:
35		ft->src_v6 = ctt->src.u3.in6;
36		ft->dst_v6 = ctt->dst.u3.in6;
37		break;
38	}
39
40	ft->l3proto = ctt->src.l3num;
41	ft->l4proto = ctt->dst.protonum;
42
43	switch (ctt->dst.protonum) {
44	case IPPROTO_TCP:
45	case IPPROTO_UDP:
46		ft->src_port = ctt->src.u.tcp.port;
47		ft->dst_port = ctt->dst.u.tcp.port;
48		break;
49	}
50}
51
52struct flow_offload *flow_offload_alloc(struct nf_conn *ct)
53{
54	struct flow_offload *flow;
55
56	if (unlikely(nf_ct_is_dying(ct)))
57		return NULL;
58
59	flow = kzalloc(sizeof(*flow), GFP_ATOMIC);
60	if (!flow)
61		return NULL;
62
63	refcount_inc(&ct->ct_general.use);
64	flow->ct = ct;
65
66	flow_offload_fill_dir(flow, FLOW_OFFLOAD_DIR_ORIGINAL);
67	flow_offload_fill_dir(flow, FLOW_OFFLOAD_DIR_REPLY);
68
69	if (ct->status & IPS_SRC_NAT)
70		__set_bit(NF_FLOW_SNAT, &flow->flags);
71	if (ct->status & IPS_DST_NAT)
72		__set_bit(NF_FLOW_DNAT, &flow->flags);
73
74	return flow;
75}
76EXPORT_SYMBOL_GPL(flow_offload_alloc);
77
78static u32 flow_offload_dst_cookie(struct flow_offload_tuple *flow_tuple)
79{
80	const struct rt6_info *rt;
81
82	if (flow_tuple->l3proto == NFPROTO_IPV6) {
83		rt = (const struct rt6_info *)flow_tuple->dst_cache;
84		return rt6_get_cookie(rt);
85	}
86
87	return 0;
88}
89
90static struct dst_entry *nft_route_dst_fetch(struct nf_flow_route *route,
91					     enum flow_offload_tuple_dir dir)
92{
93	struct dst_entry *dst = route->tuple[dir].dst;
94
95	route->tuple[dir].dst = NULL;
96
97	return dst;
98}
99
100static int flow_offload_fill_route(struct flow_offload *flow,
101				   struct nf_flow_route *route,
102				   enum flow_offload_tuple_dir dir)
103{
104	struct flow_offload_tuple *flow_tuple = &flow->tuplehash[dir].tuple;
105	struct dst_entry *dst = nft_route_dst_fetch(route, dir);
106	int i, j = 0;
107
108	switch (flow_tuple->l3proto) {
109	case NFPROTO_IPV4:
110		flow_tuple->mtu = ip_dst_mtu_maybe_forward(dst, true);
111		break;
112	case NFPROTO_IPV6:
113		flow_tuple->mtu = ip6_dst_mtu_maybe_forward(dst, true);
114		break;
115	}
116
117	flow_tuple->iifidx = route->tuple[dir].in.ifindex;
118	for (i = route->tuple[dir].in.num_encaps - 1; i >= 0; i--) {
119		flow_tuple->encap[j].id = route->tuple[dir].in.encap[i].id;
120		flow_tuple->encap[j].proto = route->tuple[dir].in.encap[i].proto;
121		if (route->tuple[dir].in.ingress_vlans & BIT(i))
122			flow_tuple->in_vlan_ingress |= BIT(j);
123		j++;
124	}
125	flow_tuple->encap_num = route->tuple[dir].in.num_encaps;
126
127	switch (route->tuple[dir].xmit_type) {
128	case FLOW_OFFLOAD_XMIT_DIRECT:
129		memcpy(flow_tuple->out.h_dest, route->tuple[dir].out.h_dest,
130		       ETH_ALEN);
131		memcpy(flow_tuple->out.h_source, route->tuple[dir].out.h_source,
132		       ETH_ALEN);
133		flow_tuple->out.ifidx = route->tuple[dir].out.ifindex;
134		flow_tuple->out.hw_ifidx = route->tuple[dir].out.hw_ifindex;
135		dst_release(dst);
136		break;
137	case FLOW_OFFLOAD_XMIT_XFRM:
138	case FLOW_OFFLOAD_XMIT_NEIGH:
139		flow_tuple->dst_cache = dst;
140		flow_tuple->dst_cookie = flow_offload_dst_cookie(flow_tuple);
141		break;
142	default:
143		WARN_ON_ONCE(1);
144		break;
145	}
146	flow_tuple->xmit_type = route->tuple[dir].xmit_type;
147
148	return 0;
149}
150
151static void nft_flow_dst_release(struct flow_offload *flow,
152				 enum flow_offload_tuple_dir dir)
153{
154	if (flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_NEIGH ||
155	    flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)
156		dst_release(flow->tuplehash[dir].tuple.dst_cache);
157}
158
159void flow_offload_route_init(struct flow_offload *flow,
160			     struct nf_flow_route *route)
161{
162	flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_ORIGINAL);
163	flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_REPLY);
164	flow->type = NF_FLOW_OFFLOAD_ROUTE;
165}
166EXPORT_SYMBOL_GPL(flow_offload_route_init);
167
168static void flow_offload_fixup_tcp(struct ip_ct_tcp *tcp)
169{
170	tcp->seen[0].td_maxwin = 0;
171	tcp->seen[1].td_maxwin = 0;
172}
173
174static void flow_offload_fixup_ct(struct nf_conn *ct)
175{
176	struct net *net = nf_ct_net(ct);
177	int l4num = nf_ct_protonum(ct);
178	s32 timeout;
179
180	if (l4num == IPPROTO_TCP) {
181		struct nf_tcp_net *tn = nf_tcp_pernet(net);
182
183		flow_offload_fixup_tcp(&ct->proto.tcp);
184
185		timeout = tn->timeouts[ct->proto.tcp.state];
186		timeout -= tn->offload_timeout;
187	} else if (l4num == IPPROTO_UDP) {
188		struct nf_udp_net *tn = nf_udp_pernet(net);
189		enum udp_conntrack state =
190			test_bit(IPS_SEEN_REPLY_BIT, &ct->status) ?
191			UDP_CT_REPLIED : UDP_CT_UNREPLIED;
192
193		timeout = tn->timeouts[state];
194		timeout -= tn->offload_timeout;
195	} else {
196		return;
197	}
198
199	if (timeout < 0)
200		timeout = 0;
201
202	if (nf_flow_timeout_delta(READ_ONCE(ct->timeout)) > (__s32)timeout)
203		WRITE_ONCE(ct->timeout, nfct_time_stamp + timeout);
204}
205
206static void flow_offload_route_release(struct flow_offload *flow)
207{
208	nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_ORIGINAL);
209	nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_REPLY);
210}
211
212void flow_offload_free(struct flow_offload *flow)
213{
214	switch (flow->type) {
215	case NF_FLOW_OFFLOAD_ROUTE:
216		flow_offload_route_release(flow);
217		break;
218	default:
219		break;
220	}
221	nf_ct_put(flow->ct);
222	kfree_rcu(flow, rcu_head);
223}
224EXPORT_SYMBOL_GPL(flow_offload_free);
225
226static u32 flow_offload_hash(const void *data, u32 len, u32 seed)
227{
228	const struct flow_offload_tuple *tuple = data;
229
230	return jhash(tuple, offsetof(struct flow_offload_tuple, __hash), seed);
231}
232
233static u32 flow_offload_hash_obj(const void *data, u32 len, u32 seed)
234{
235	const struct flow_offload_tuple_rhash *tuplehash = data;
236
237	return jhash(&tuplehash->tuple, offsetof(struct flow_offload_tuple, __hash), seed);
238}
239
240static int flow_offload_hash_cmp(struct rhashtable_compare_arg *arg,
241					const void *ptr)
242{
243	const struct flow_offload_tuple *tuple = arg->key;
244	const struct flow_offload_tuple_rhash *x = ptr;
245
246	if (memcmp(&x->tuple, tuple, offsetof(struct flow_offload_tuple, __hash)))
247		return 1;
248
249	return 0;
250}
251
252static const struct rhashtable_params nf_flow_offload_rhash_params = {
253	.head_offset		= offsetof(struct flow_offload_tuple_rhash, node),
254	.hashfn			= flow_offload_hash,
255	.obj_hashfn		= flow_offload_hash_obj,
256	.obj_cmpfn		= flow_offload_hash_cmp,
257	.automatic_shrinking	= true,
258};
259
260unsigned long flow_offload_get_timeout(struct flow_offload *flow)
261{
262	unsigned long timeout = NF_FLOW_TIMEOUT;
263	struct net *net = nf_ct_net(flow->ct);
264	int l4num = nf_ct_protonum(flow->ct);
265
266	if (l4num == IPPROTO_TCP) {
267		struct nf_tcp_net *tn = nf_tcp_pernet(net);
268
269		timeout = tn->offload_timeout;
270	} else if (l4num == IPPROTO_UDP) {
271		struct nf_udp_net *tn = nf_udp_pernet(net);
272
273		timeout = tn->offload_timeout;
274	}
275
276	return timeout;
277}
278
279int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
280{
281	int err;
282
283	flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
284
285	err = rhashtable_insert_fast(&flow_table->rhashtable,
286				     &flow->tuplehash[0].node,
287				     nf_flow_offload_rhash_params);
288	if (err < 0)
289		return err;
290
291	err = rhashtable_insert_fast(&flow_table->rhashtable,
292				     &flow->tuplehash[1].node,
293				     nf_flow_offload_rhash_params);
294	if (err < 0) {
295		rhashtable_remove_fast(&flow_table->rhashtable,
296				       &flow->tuplehash[0].node,
297				       nf_flow_offload_rhash_params);
298		return err;
299	}
300
301	nf_ct_offload_timeout(flow->ct);
302
303	if (nf_flowtable_hw_offload(flow_table)) {
304		__set_bit(NF_FLOW_HW, &flow->flags);
305		nf_flow_offload_add(flow_table, flow);
306	}
307
308	return 0;
309}
310EXPORT_SYMBOL_GPL(flow_offload_add);
311
312void flow_offload_refresh(struct nf_flowtable *flow_table,
313			  struct flow_offload *flow, bool force)
314{
315	u32 timeout;
316
317	timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
318	if (force || timeout - READ_ONCE(flow->timeout) > HZ)
319		WRITE_ONCE(flow->timeout, timeout);
320	else
321		return;
322
323	if (likely(!nf_flowtable_hw_offload(flow_table)))
324		return;
325
326	nf_flow_offload_add(flow_table, flow);
327}
328EXPORT_SYMBOL_GPL(flow_offload_refresh);
329
330static inline bool nf_flow_has_expired(const struct flow_offload *flow)
331{
332	return nf_flow_timeout_delta(flow->timeout) <= 0;
333}
334
335static void flow_offload_del(struct nf_flowtable *flow_table,
336			     struct flow_offload *flow)
337{
338	rhashtable_remove_fast(&flow_table->rhashtable,
339			       &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
340			       nf_flow_offload_rhash_params);
341	rhashtable_remove_fast(&flow_table->rhashtable,
342			       &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
343			       nf_flow_offload_rhash_params);
344	flow_offload_free(flow);
345}
346
347void flow_offload_teardown(struct flow_offload *flow)
348{
349	clear_bit(IPS_OFFLOAD_BIT, &flow->ct->status);
350	set_bit(NF_FLOW_TEARDOWN, &flow->flags);
351	flow_offload_fixup_ct(flow->ct);
352}
353EXPORT_SYMBOL_GPL(flow_offload_teardown);
354
355struct flow_offload_tuple_rhash *
356flow_offload_lookup(struct nf_flowtable *flow_table,
357		    struct flow_offload_tuple *tuple)
358{
359	struct flow_offload_tuple_rhash *tuplehash;
360	struct flow_offload *flow;
361	int dir;
362
363	tuplehash = rhashtable_lookup(&flow_table->rhashtable, tuple,
364				      nf_flow_offload_rhash_params);
365	if (!tuplehash)
366		return NULL;
367
368	dir = tuplehash->tuple.dir;
369	flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
370	if (test_bit(NF_FLOW_TEARDOWN, &flow->flags))
371		return NULL;
372
373	if (unlikely(nf_ct_is_dying(flow->ct)))
374		return NULL;
375
376	return tuplehash;
377}
378EXPORT_SYMBOL_GPL(flow_offload_lookup);
379
380static int
381nf_flow_table_iterate(struct nf_flowtable *flow_table,
382		      void (*iter)(struct nf_flowtable *flowtable,
383				   struct flow_offload *flow, void *data),
384		      void *data)
385{
386	struct flow_offload_tuple_rhash *tuplehash;
387	struct rhashtable_iter hti;
388	struct flow_offload *flow;
389	int err = 0;
390
391	rhashtable_walk_enter(&flow_table->rhashtable, &hti);
392	rhashtable_walk_start(&hti);
393
394	while ((tuplehash = rhashtable_walk_next(&hti))) {
395		if (IS_ERR(tuplehash)) {
396			if (PTR_ERR(tuplehash) != -EAGAIN) {
397				err = PTR_ERR(tuplehash);
398				break;
399			}
400			continue;
401		}
402		if (tuplehash->tuple.dir)
403			continue;
404
405		flow = container_of(tuplehash, struct flow_offload, tuplehash[0]);
406
407		iter(flow_table, flow, data);
408	}
409	rhashtable_walk_stop(&hti);
410	rhashtable_walk_exit(&hti);
411
412	return err;
413}
414
415static bool nf_flow_custom_gc(struct nf_flowtable *flow_table,
416			      const struct flow_offload *flow)
417{
418	return flow_table->type->gc && flow_table->type->gc(flow);
419}
420
421static void nf_flow_offload_gc_step(struct nf_flowtable *flow_table,
422				    struct flow_offload *flow, void *data)
423{
424	if (nf_flow_has_expired(flow) ||
425	    nf_ct_is_dying(flow->ct) ||
426	    nf_flow_custom_gc(flow_table, flow))
427		flow_offload_teardown(flow);
428
429	if (test_bit(NF_FLOW_TEARDOWN, &flow->flags)) {
430		if (test_bit(NF_FLOW_HW, &flow->flags)) {
431			if (!test_bit(NF_FLOW_HW_DYING, &flow->flags))
432				nf_flow_offload_del(flow_table, flow);
433			else if (test_bit(NF_FLOW_HW_DEAD, &flow->flags))
434				flow_offload_del(flow_table, flow);
435		} else {
436			flow_offload_del(flow_table, flow);
437		}
438	} else if (test_bit(NF_FLOW_HW, &flow->flags)) {
439		nf_flow_offload_stats(flow_table, flow);
440	}
441}
442
443void nf_flow_table_gc_run(struct nf_flowtable *flow_table)
444{
445	nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL);
446}
447
448static void nf_flow_offload_work_gc(struct work_struct *work)
449{
450	struct nf_flowtable *flow_table;
451
452	flow_table = container_of(work, struct nf_flowtable, gc_work.work);
453	nf_flow_table_gc_run(flow_table);
454	queue_delayed_work(system_power_efficient_wq, &flow_table->gc_work, HZ);
455}
456
457static void nf_flow_nat_port_tcp(struct sk_buff *skb, unsigned int thoff,
458				 __be16 port, __be16 new_port)
459{
460	struct tcphdr *tcph;
461
462	tcph = (void *)(skb_network_header(skb) + thoff);
463	inet_proto_csum_replace2(&tcph->check, skb, port, new_port, false);
464}
465
466static void nf_flow_nat_port_udp(struct sk_buff *skb, unsigned int thoff,
467				 __be16 port, __be16 new_port)
468{
469	struct udphdr *udph;
470
471	udph = (void *)(skb_network_header(skb) + thoff);
472	if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
473		inet_proto_csum_replace2(&udph->check, skb, port,
474					 new_port, false);
475		if (!udph->check)
476			udph->check = CSUM_MANGLED_0;
477	}
478}
479
480static void nf_flow_nat_port(struct sk_buff *skb, unsigned int thoff,
481			     u8 protocol, __be16 port, __be16 new_port)
482{
483	switch (protocol) {
484	case IPPROTO_TCP:
485		nf_flow_nat_port_tcp(skb, thoff, port, new_port);
486		break;
487	case IPPROTO_UDP:
488		nf_flow_nat_port_udp(skb, thoff, port, new_port);
489		break;
490	}
491}
492
493void nf_flow_snat_port(const struct flow_offload *flow,
494		       struct sk_buff *skb, unsigned int thoff,
495		       u8 protocol, enum flow_offload_tuple_dir dir)
496{
497	struct flow_ports *hdr;
498	__be16 port, new_port;
499
500	hdr = (void *)(skb_network_header(skb) + thoff);
501
502	switch (dir) {
503	case FLOW_OFFLOAD_DIR_ORIGINAL:
504		port = hdr->source;
505		new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port;
506		hdr->source = new_port;
507		break;
508	case FLOW_OFFLOAD_DIR_REPLY:
509		port = hdr->dest;
510		new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port;
511		hdr->dest = new_port;
512		break;
513	}
514
515	nf_flow_nat_port(skb, thoff, protocol, port, new_port);
516}
517EXPORT_SYMBOL_GPL(nf_flow_snat_port);
518
519void nf_flow_dnat_port(const struct flow_offload *flow, struct sk_buff *skb,
520		       unsigned int thoff, u8 protocol,
521		       enum flow_offload_tuple_dir dir)
522{
523	struct flow_ports *hdr;
524	__be16 port, new_port;
525
526	hdr = (void *)(skb_network_header(skb) + thoff);
527
528	switch (dir) {
529	case FLOW_OFFLOAD_DIR_ORIGINAL:
530		port = hdr->dest;
531		new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port;
532		hdr->dest = new_port;
533		break;
534	case FLOW_OFFLOAD_DIR_REPLY:
535		port = hdr->source;
536		new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port;
537		hdr->source = new_port;
538		break;
539	}
540
541	nf_flow_nat_port(skb, thoff, protocol, port, new_port);
542}
543EXPORT_SYMBOL_GPL(nf_flow_dnat_port);
544
545int nf_flow_table_init(struct nf_flowtable *flowtable)
546{
547	int err;
548
549	INIT_DELAYED_WORK(&flowtable->gc_work, nf_flow_offload_work_gc);
550	flow_block_init(&flowtable->flow_block);
551	init_rwsem(&flowtable->flow_block_lock);
552
553	err = rhashtable_init(&flowtable->rhashtable,
554			      &nf_flow_offload_rhash_params);
555	if (err < 0)
556		return err;
557
558	queue_delayed_work(system_power_efficient_wq,
559			   &flowtable->gc_work, HZ);
560
561	mutex_lock(&flowtable_lock);
562	list_add(&flowtable->list, &flowtables);
563	mutex_unlock(&flowtable_lock);
564
565	return 0;
566}
567EXPORT_SYMBOL_GPL(nf_flow_table_init);
568
569static void nf_flow_table_do_cleanup(struct nf_flowtable *flow_table,
570				     struct flow_offload *flow, void *data)
571{
572	struct net_device *dev = data;
573
574	if (!dev) {
575		flow_offload_teardown(flow);
576		return;
577	}
578
579	if (net_eq(nf_ct_net(flow->ct), dev_net(dev)) &&
580	    (flow->tuplehash[0].tuple.iifidx == dev->ifindex ||
581	     flow->tuplehash[1].tuple.iifidx == dev->ifindex))
582		flow_offload_teardown(flow);
583}
584
585void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable,
586			      struct net_device *dev)
587{
588	nf_flow_table_iterate(flowtable, nf_flow_table_do_cleanup, dev);
589	flush_delayed_work(&flowtable->gc_work);
590	nf_flow_table_offload_flush(flowtable);
591}
592
593void nf_flow_table_cleanup(struct net_device *dev)
594{
595	struct nf_flowtable *flowtable;
596
597	mutex_lock(&flowtable_lock);
598	list_for_each_entry(flowtable, &flowtables, list)
599		nf_flow_table_gc_cleanup(flowtable, dev);
600	mutex_unlock(&flowtable_lock);
601}
602EXPORT_SYMBOL_GPL(nf_flow_table_cleanup);
603
604void nf_flow_table_free(struct nf_flowtable *flow_table)
605{
606	mutex_lock(&flowtable_lock);
607	list_del(&flow_table->list);
608	mutex_unlock(&flowtable_lock);
609
610	cancel_delayed_work_sync(&flow_table->gc_work);
611	nf_flow_table_offload_flush(flow_table);
612	/* ... no more pending work after this stage ... */
613	nf_flow_table_iterate(flow_table, nf_flow_table_do_cleanup, NULL);
614	nf_flow_table_gc_run(flow_table);
615	nf_flow_table_offload_flush_cleanup(flow_table);
616	rhashtable_destroy(&flow_table->rhashtable);
617}
618EXPORT_SYMBOL_GPL(nf_flow_table_free);
619
620static int nf_flow_table_init_net(struct net *net)
621{
622	net->ft.stat = alloc_percpu(struct nf_flow_table_stat);
623	return net->ft.stat ? 0 : -ENOMEM;
624}
625
626static void nf_flow_table_fini_net(struct net *net)
627{
628	free_percpu(net->ft.stat);
629}
630
631static int nf_flow_table_pernet_init(struct net *net)
632{
633	int ret;
634
635	ret = nf_flow_table_init_net(net);
636	if (ret < 0)
637		return ret;
638
639	ret = nf_flow_table_init_proc(net);
640	if (ret < 0)
641		goto out_proc;
642
643	return 0;
644
645out_proc:
646	nf_flow_table_fini_net(net);
647	return ret;
648}
649
650static void nf_flow_table_pernet_exit(struct list_head *net_exit_list)
651{
652	struct net *net;
653
654	list_for_each_entry(net, net_exit_list, exit_list) {
655		nf_flow_table_fini_proc(net);
656		nf_flow_table_fini_net(net);
657	}
658}
659
660static struct pernet_operations nf_flow_table_net_ops = {
661	.init = nf_flow_table_pernet_init,
662	.exit_batch = nf_flow_table_pernet_exit,
663};
664
665static int __init nf_flow_table_module_init(void)
666{
667	int ret;
668
669	ret = register_pernet_subsys(&nf_flow_table_net_ops);
670	if (ret < 0)
671		return ret;
672
673	ret = nf_flow_table_offload_init();
674	if (ret)
675		goto out_offload;
676
677	return 0;
678
679out_offload:
680	unregister_pernet_subsys(&nf_flow_table_net_ops);
681	return ret;
682}
683
684static void __exit nf_flow_table_module_exit(void)
685{
686	nf_flow_table_offload_exit();
687	unregister_pernet_subsys(&nf_flow_table_net_ops);
688}
689
690module_init(nf_flow_table_module_init);
691module_exit(nf_flow_table_module_exit);
692
693MODULE_LICENSE("GPL");
694MODULE_AUTHOR("Pablo Neira Ayuso <pablo@netfilter.org>");
695MODULE_DESCRIPTION("Netfilter flow table module");
696