1#include <linux/kernel.h> 2#include <linux/init.h> 3#include <linux/module.h> 4#include <linux/netfilter.h> 5#include <linux/rhashtable.h> 6#include <linux/netdevice.h> 7#include <linux/tc_act/tc_csum.h> 8#include <net/flow_offload.h> 9#include <net/netfilter/nf_flow_table.h> 10#include <net/netfilter/nf_tables.h> 11#include <net/netfilter/nf_conntrack.h> 12#include <net/netfilter/nf_conntrack_acct.h> 13#include <net/netfilter/nf_conntrack_core.h> 14#include <net/netfilter/nf_conntrack_tuple.h> 15 16static struct workqueue_struct *nf_flow_offload_add_wq; 17static struct workqueue_struct *nf_flow_offload_del_wq; 18static struct workqueue_struct *nf_flow_offload_stats_wq; 19 20struct flow_offload_work { 21 struct list_head list; 22 enum flow_cls_command cmd; 23 struct nf_flowtable *flowtable; 24 struct flow_offload *flow; 25 struct work_struct work; 26}; 27 28#define NF_FLOW_DISSECTOR(__match, __type, __field) \ 29 (__match)->dissector.offset[__type] = \ 30 offsetof(struct nf_flow_key, __field) 31 32static void nf_flow_rule_lwt_match(struct nf_flow_match *match, 33 struct ip_tunnel_info *tun_info) 34{ 35 struct nf_flow_key *mask = &match->mask; 36 struct nf_flow_key *key = &match->key; 37 unsigned long long enc_keys; 38 39 if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX)) 40 return; 41 42 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control); 43 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 44 key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id); 45 mask->enc_key_id.keyid = 0xffffffff; 46 enc_keys = BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | 47 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL); 48 49 if (ip_tunnel_info_af(tun_info) == AF_INET) { 50 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 51 enc_ipv4); 52 key->enc_ipv4.src = tun_info->key.u.ipv4.dst; 53 key->enc_ipv4.dst = tun_info->key.u.ipv4.src; 54 if (key->enc_ipv4.src) 55 mask->enc_ipv4.src = 0xffffffff; 56 if (key->enc_ipv4.dst) 57 mask->enc_ipv4.dst = 0xffffffff; 58 enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); 59 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 60 } else { 61 memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst, 62 sizeof(struct in6_addr)); 63 memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src, 64 sizeof(struct in6_addr)); 65 if (memcmp(&key->enc_ipv6.src, &in6addr_any, 66 sizeof(struct in6_addr))) 67 memset(&mask->enc_ipv6.src, 0xff, 68 sizeof(struct in6_addr)); 69 if (memcmp(&key->enc_ipv6.dst, &in6addr_any, 70 sizeof(struct in6_addr))) 71 memset(&mask->enc_ipv6.dst, 0xff, 72 sizeof(struct in6_addr)); 73 enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS); 74 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 75 } 76 77 match->dissector.used_keys |= enc_keys; 78} 79 80static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key, 81 struct flow_dissector_key_vlan *mask, 82 u16 vlan_id, __be16 proto) 83{ 84 key->vlan_id = vlan_id; 85 mask->vlan_id = VLAN_VID_MASK; 86 key->vlan_tpid = proto; 87 mask->vlan_tpid = 0xffff; 88} 89 90static int nf_flow_rule_match(struct nf_flow_match *match, 91 const struct flow_offload_tuple *tuple, 92 struct dst_entry *other_dst) 93{ 94 struct nf_flow_key *mask = &match->mask; 95 struct nf_flow_key *key = &match->key; 96 struct ip_tunnel_info *tun_info; 97 bool vlan_encap = false; 98 99 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta); 100 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control); 101 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic); 102 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 103 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 104 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp); 105 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp); 106 107 if (other_dst && other_dst->lwtstate) { 108 tun_info = lwt_tun_info(other_dst->lwtstate); 109 nf_flow_rule_lwt_match(match, tun_info); 110 } 111 112 if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_TC) 113 key->meta.ingress_ifindex = tuple->tc.iifidx; 114 else 115 key->meta.ingress_ifindex = tuple->iifidx; 116 117 mask->meta.ingress_ifindex = 0xffffffff; 118 119 if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) && 120 tuple->encap[0].proto == htons(ETH_P_8021Q)) { 121 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan); 122 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan, 123 tuple->encap[0].id, 124 tuple->encap[0].proto); 125 vlan_encap = true; 126 } 127 128 if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) && 129 tuple->encap[1].proto == htons(ETH_P_8021Q)) { 130 if (vlan_encap) { 131 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN, 132 cvlan); 133 nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan, 134 tuple->encap[1].id, 135 tuple->encap[1].proto); 136 } else { 137 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, 138 vlan); 139 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan, 140 tuple->encap[1].id, 141 tuple->encap[1].proto); 142 } 143 } 144 145 switch (tuple->l3proto) { 146 case AF_INET: 147 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 148 key->basic.n_proto = htons(ETH_P_IP); 149 key->ipv4.src = tuple->src_v4.s_addr; 150 mask->ipv4.src = 0xffffffff; 151 key->ipv4.dst = tuple->dst_v4.s_addr; 152 mask->ipv4.dst = 0xffffffff; 153 break; 154 case AF_INET6: 155 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 156 key->basic.n_proto = htons(ETH_P_IPV6); 157 key->ipv6.src = tuple->src_v6; 158 memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src)); 159 key->ipv6.dst = tuple->dst_v6; 160 memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst)); 161 break; 162 default: 163 return -EOPNOTSUPP; 164 } 165 mask->control.addr_type = 0xffff; 166 match->dissector.used_keys |= BIT_ULL(key->control.addr_type); 167 mask->basic.n_proto = 0xffff; 168 169 switch (tuple->l4proto) { 170 case IPPROTO_TCP: 171 key->tcp.flags = 0; 172 mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16); 173 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_TCP); 174 break; 175 case IPPROTO_UDP: 176 case IPPROTO_GRE: 177 break; 178 default: 179 return -EOPNOTSUPP; 180 } 181 182 key->basic.ip_proto = tuple->l4proto; 183 mask->basic.ip_proto = 0xff; 184 185 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_META) | 186 BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 187 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC); 188 189 switch (tuple->l4proto) { 190 case IPPROTO_TCP: 191 case IPPROTO_UDP: 192 key->tp.src = tuple->src_port; 193 mask->tp.src = 0xffff; 194 key->tp.dst = tuple->dst_port; 195 mask->tp.dst = 0xffff; 196 197 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_PORTS); 198 break; 199 } 200 201 return 0; 202} 203 204static void flow_offload_mangle(struct flow_action_entry *entry, 205 enum flow_action_mangle_base htype, u32 offset, 206 const __be32 *value, const __be32 *mask) 207{ 208 entry->id = FLOW_ACTION_MANGLE; 209 entry->mangle.htype = htype; 210 entry->mangle.offset = offset; 211 memcpy(&entry->mangle.mask, mask, sizeof(u32)); 212 memcpy(&entry->mangle.val, value, sizeof(u32)); 213} 214 215static inline struct flow_action_entry * 216flow_action_entry_next(struct nf_flow_rule *flow_rule) 217{ 218 int i = flow_rule->rule->action.num_entries++; 219 220 return &flow_rule->rule->action.entries[i]; 221} 222 223static int flow_offload_eth_src(struct net *net, 224 const struct flow_offload *flow, 225 enum flow_offload_tuple_dir dir, 226 struct nf_flow_rule *flow_rule) 227{ 228 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule); 229 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule); 230 const struct flow_offload_tuple *other_tuple, *this_tuple; 231 struct net_device *dev = NULL; 232 const unsigned char *addr; 233 u32 mask, val; 234 u16 val16; 235 236 this_tuple = &flow->tuplehash[dir].tuple; 237 238 switch (this_tuple->xmit_type) { 239 case FLOW_OFFLOAD_XMIT_DIRECT: 240 addr = this_tuple->out.h_source; 241 break; 242 case FLOW_OFFLOAD_XMIT_NEIGH: 243 other_tuple = &flow->tuplehash[!dir].tuple; 244 dev = dev_get_by_index(net, other_tuple->iifidx); 245 if (!dev) 246 return -ENOENT; 247 248 addr = dev->dev_addr; 249 break; 250 default: 251 return -EOPNOTSUPP; 252 } 253 254 mask = ~0xffff0000; 255 memcpy(&val16, addr, 2); 256 val = val16 << 16; 257 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4, 258 &val, &mask); 259 260 mask = ~0xffffffff; 261 memcpy(&val, addr + 2, 4); 262 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8, 263 &val, &mask); 264 265 dev_put(dev); 266 267 return 0; 268} 269 270static int flow_offload_eth_dst(struct net *net, 271 const struct flow_offload *flow, 272 enum flow_offload_tuple_dir dir, 273 struct nf_flow_rule *flow_rule) 274{ 275 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule); 276 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule); 277 const struct flow_offload_tuple *other_tuple, *this_tuple; 278 const struct dst_entry *dst_cache; 279 unsigned char ha[ETH_ALEN]; 280 struct neighbour *n; 281 const void *daddr; 282 u32 mask, val; 283 u8 nud_state; 284 u16 val16; 285 286 this_tuple = &flow->tuplehash[dir].tuple; 287 288 switch (this_tuple->xmit_type) { 289 case FLOW_OFFLOAD_XMIT_DIRECT: 290 ether_addr_copy(ha, this_tuple->out.h_dest); 291 break; 292 case FLOW_OFFLOAD_XMIT_NEIGH: 293 other_tuple = &flow->tuplehash[!dir].tuple; 294 daddr = &other_tuple->src_v4; 295 dst_cache = this_tuple->dst_cache; 296 n = dst_neigh_lookup(dst_cache, daddr); 297 if (!n) 298 return -ENOENT; 299 300 read_lock_bh(&n->lock); 301 nud_state = n->nud_state; 302 ether_addr_copy(ha, n->ha); 303 read_unlock_bh(&n->lock); 304 neigh_release(n); 305 306 if (!(nud_state & NUD_VALID)) 307 return -ENOENT; 308 break; 309 default: 310 return -EOPNOTSUPP; 311 } 312 313 mask = ~0xffffffff; 314 memcpy(&val, ha, 4); 315 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0, 316 &val, &mask); 317 318 mask = ~0x0000ffff; 319 memcpy(&val16, ha + 4, 2); 320 val = val16; 321 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4, 322 &val, &mask); 323 324 return 0; 325} 326 327static void flow_offload_ipv4_snat(struct net *net, 328 const struct flow_offload *flow, 329 enum flow_offload_tuple_dir dir, 330 struct nf_flow_rule *flow_rule) 331{ 332 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 333 u32 mask = ~htonl(0xffffffff); 334 __be32 addr; 335 u32 offset; 336 337 switch (dir) { 338 case FLOW_OFFLOAD_DIR_ORIGINAL: 339 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; 340 offset = offsetof(struct iphdr, saddr); 341 break; 342 case FLOW_OFFLOAD_DIR_REPLY: 343 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; 344 offset = offsetof(struct iphdr, daddr); 345 break; 346 default: 347 return; 348 } 349 350 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, 351 &addr, &mask); 352} 353 354static void flow_offload_ipv4_dnat(struct net *net, 355 const struct flow_offload *flow, 356 enum flow_offload_tuple_dir dir, 357 struct nf_flow_rule *flow_rule) 358{ 359 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 360 u32 mask = ~htonl(0xffffffff); 361 __be32 addr; 362 u32 offset; 363 364 switch (dir) { 365 case FLOW_OFFLOAD_DIR_ORIGINAL: 366 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; 367 offset = offsetof(struct iphdr, daddr); 368 break; 369 case FLOW_OFFLOAD_DIR_REPLY: 370 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; 371 offset = offsetof(struct iphdr, saddr); 372 break; 373 default: 374 return; 375 } 376 377 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, 378 &addr, &mask); 379} 380 381static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, 382 unsigned int offset, 383 const __be32 *addr, const __be32 *mask) 384{ 385 struct flow_action_entry *entry; 386 int i; 387 388 for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++) { 389 entry = flow_action_entry_next(flow_rule); 390 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6, 391 offset + i * sizeof(u32), &addr[i], mask); 392 } 393} 394 395static void flow_offload_ipv6_snat(struct net *net, 396 const struct flow_offload *flow, 397 enum flow_offload_tuple_dir dir, 398 struct nf_flow_rule *flow_rule) 399{ 400 u32 mask = ~htonl(0xffffffff); 401 const __be32 *addr; 402 u32 offset; 403 404 switch (dir) { 405 case FLOW_OFFLOAD_DIR_ORIGINAL: 406 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32; 407 offset = offsetof(struct ipv6hdr, saddr); 408 break; 409 case FLOW_OFFLOAD_DIR_REPLY: 410 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32; 411 offset = offsetof(struct ipv6hdr, daddr); 412 break; 413 default: 414 return; 415 } 416 417 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); 418} 419 420static void flow_offload_ipv6_dnat(struct net *net, 421 const struct flow_offload *flow, 422 enum flow_offload_tuple_dir dir, 423 struct nf_flow_rule *flow_rule) 424{ 425 u32 mask = ~htonl(0xffffffff); 426 const __be32 *addr; 427 u32 offset; 428 429 switch (dir) { 430 case FLOW_OFFLOAD_DIR_ORIGINAL: 431 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32; 432 offset = offsetof(struct ipv6hdr, daddr); 433 break; 434 case FLOW_OFFLOAD_DIR_REPLY: 435 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32; 436 offset = offsetof(struct ipv6hdr, saddr); 437 break; 438 default: 439 return; 440 } 441 442 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); 443} 444 445static int flow_offload_l4proto(const struct flow_offload *flow) 446{ 447 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; 448 u8 type = 0; 449 450 switch (protonum) { 451 case IPPROTO_TCP: 452 type = FLOW_ACT_MANGLE_HDR_TYPE_TCP; 453 break; 454 case IPPROTO_UDP: 455 type = FLOW_ACT_MANGLE_HDR_TYPE_UDP; 456 break; 457 default: 458 break; 459 } 460 461 return type; 462} 463 464static void flow_offload_port_snat(struct net *net, 465 const struct flow_offload *flow, 466 enum flow_offload_tuple_dir dir, 467 struct nf_flow_rule *flow_rule) 468{ 469 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 470 u32 mask, port; 471 u32 offset; 472 473 switch (dir) { 474 case FLOW_OFFLOAD_DIR_ORIGINAL: 475 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port); 476 offset = 0; /* offsetof(struct tcphdr, source); */ 477 port = htonl(port << 16); 478 mask = ~htonl(0xffff0000); 479 break; 480 case FLOW_OFFLOAD_DIR_REPLY: 481 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port); 482 offset = 0; /* offsetof(struct tcphdr, dest); */ 483 port = htonl(port); 484 mask = ~htonl(0xffff); 485 break; 486 default: 487 return; 488 } 489 490 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, 491 &port, &mask); 492} 493 494static void flow_offload_port_dnat(struct net *net, 495 const struct flow_offload *flow, 496 enum flow_offload_tuple_dir dir, 497 struct nf_flow_rule *flow_rule) 498{ 499 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 500 u32 mask, port; 501 u32 offset; 502 503 switch (dir) { 504 case FLOW_OFFLOAD_DIR_ORIGINAL: 505 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port); 506 offset = 0; /* offsetof(struct tcphdr, dest); */ 507 port = htonl(port); 508 mask = ~htonl(0xffff); 509 break; 510 case FLOW_OFFLOAD_DIR_REPLY: 511 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port); 512 offset = 0; /* offsetof(struct tcphdr, source); */ 513 port = htonl(port << 16); 514 mask = ~htonl(0xffff0000); 515 break; 516 default: 517 return; 518 } 519 520 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, 521 &port, &mask); 522} 523 524static void flow_offload_ipv4_checksum(struct net *net, 525 const struct flow_offload *flow, 526 struct nf_flow_rule *flow_rule) 527{ 528 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; 529 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 530 531 entry->id = FLOW_ACTION_CSUM; 532 entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR; 533 534 switch (protonum) { 535 case IPPROTO_TCP: 536 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP; 537 break; 538 case IPPROTO_UDP: 539 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP; 540 break; 541 } 542} 543 544static void flow_offload_redirect(struct net *net, 545 const struct flow_offload *flow, 546 enum flow_offload_tuple_dir dir, 547 struct nf_flow_rule *flow_rule) 548{ 549 const struct flow_offload_tuple *this_tuple, *other_tuple; 550 struct flow_action_entry *entry; 551 struct net_device *dev; 552 int ifindex; 553 554 this_tuple = &flow->tuplehash[dir].tuple; 555 switch (this_tuple->xmit_type) { 556 case FLOW_OFFLOAD_XMIT_DIRECT: 557 this_tuple = &flow->tuplehash[dir].tuple; 558 ifindex = this_tuple->out.hw_ifidx; 559 break; 560 case FLOW_OFFLOAD_XMIT_NEIGH: 561 other_tuple = &flow->tuplehash[!dir].tuple; 562 ifindex = other_tuple->iifidx; 563 break; 564 default: 565 return; 566 } 567 568 dev = dev_get_by_index(net, ifindex); 569 if (!dev) 570 return; 571 572 entry = flow_action_entry_next(flow_rule); 573 entry->id = FLOW_ACTION_REDIRECT; 574 entry->dev = dev; 575} 576 577static void flow_offload_encap_tunnel(const struct flow_offload *flow, 578 enum flow_offload_tuple_dir dir, 579 struct nf_flow_rule *flow_rule) 580{ 581 const struct flow_offload_tuple *this_tuple; 582 struct flow_action_entry *entry; 583 struct dst_entry *dst; 584 585 this_tuple = &flow->tuplehash[dir].tuple; 586 if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) 587 return; 588 589 dst = this_tuple->dst_cache; 590 if (dst && dst->lwtstate) { 591 struct ip_tunnel_info *tun_info; 592 593 tun_info = lwt_tun_info(dst->lwtstate); 594 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { 595 entry = flow_action_entry_next(flow_rule); 596 entry->id = FLOW_ACTION_TUNNEL_ENCAP; 597 entry->tunnel = tun_info; 598 } 599 } 600} 601 602static void flow_offload_decap_tunnel(const struct flow_offload *flow, 603 enum flow_offload_tuple_dir dir, 604 struct nf_flow_rule *flow_rule) 605{ 606 const struct flow_offload_tuple *other_tuple; 607 struct flow_action_entry *entry; 608 struct dst_entry *dst; 609 610 other_tuple = &flow->tuplehash[!dir].tuple; 611 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) 612 return; 613 614 dst = other_tuple->dst_cache; 615 if (dst && dst->lwtstate) { 616 struct ip_tunnel_info *tun_info; 617 618 tun_info = lwt_tun_info(dst->lwtstate); 619 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { 620 entry = flow_action_entry_next(flow_rule); 621 entry->id = FLOW_ACTION_TUNNEL_DECAP; 622 } 623 } 624} 625 626static int 627nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, 628 enum flow_offload_tuple_dir dir, 629 struct nf_flow_rule *flow_rule) 630{ 631 const struct flow_offload_tuple *other_tuple; 632 const struct flow_offload_tuple *tuple; 633 int i; 634 635 flow_offload_decap_tunnel(flow, dir, flow_rule); 636 flow_offload_encap_tunnel(flow, dir, flow_rule); 637 638 if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 || 639 flow_offload_eth_dst(net, flow, dir, flow_rule) < 0) 640 return -1; 641 642 tuple = &flow->tuplehash[dir].tuple; 643 644 for (i = 0; i < tuple->encap_num; i++) { 645 struct flow_action_entry *entry; 646 647 if (tuple->in_vlan_ingress & BIT(i)) 648 continue; 649 650 if (tuple->encap[i].proto == htons(ETH_P_8021Q)) { 651 entry = flow_action_entry_next(flow_rule); 652 entry->id = FLOW_ACTION_VLAN_POP; 653 } 654 } 655 656 other_tuple = &flow->tuplehash[!dir].tuple; 657 658 for (i = 0; i < other_tuple->encap_num; i++) { 659 struct flow_action_entry *entry; 660 661 if (other_tuple->in_vlan_ingress & BIT(i)) 662 continue; 663 664 entry = flow_action_entry_next(flow_rule); 665 666 switch (other_tuple->encap[i].proto) { 667 case htons(ETH_P_PPP_SES): 668 entry->id = FLOW_ACTION_PPPOE_PUSH; 669 entry->pppoe.sid = other_tuple->encap[i].id; 670 break; 671 case htons(ETH_P_8021Q): 672 entry->id = FLOW_ACTION_VLAN_PUSH; 673 entry->vlan.vid = other_tuple->encap[i].id; 674 entry->vlan.proto = other_tuple->encap[i].proto; 675 break; 676 } 677 } 678 679 return 0; 680} 681 682int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow, 683 enum flow_offload_tuple_dir dir, 684 struct nf_flow_rule *flow_rule) 685{ 686 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0) 687 return -1; 688 689 if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 690 flow_offload_ipv4_snat(net, flow, dir, flow_rule); 691 flow_offload_port_snat(net, flow, dir, flow_rule); 692 } 693 if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 694 flow_offload_ipv4_dnat(net, flow, dir, flow_rule); 695 flow_offload_port_dnat(net, flow, dir, flow_rule); 696 } 697 if (test_bit(NF_FLOW_SNAT, &flow->flags) || 698 test_bit(NF_FLOW_DNAT, &flow->flags)) 699 flow_offload_ipv4_checksum(net, flow, flow_rule); 700 701 flow_offload_redirect(net, flow, dir, flow_rule); 702 703 return 0; 704} 705EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4); 706 707int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow, 708 enum flow_offload_tuple_dir dir, 709 struct nf_flow_rule *flow_rule) 710{ 711 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0) 712 return -1; 713 714 if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 715 flow_offload_ipv6_snat(net, flow, dir, flow_rule); 716 flow_offload_port_snat(net, flow, dir, flow_rule); 717 } 718 if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 719 flow_offload_ipv6_dnat(net, flow, dir, flow_rule); 720 flow_offload_port_dnat(net, flow, dir, flow_rule); 721 } 722 723 flow_offload_redirect(net, flow, dir, flow_rule); 724 725 return 0; 726} 727EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6); 728 729#define NF_FLOW_RULE_ACTION_MAX 16 730 731static struct nf_flow_rule * 732nf_flow_offload_rule_alloc(struct net *net, 733 const struct flow_offload_work *offload, 734 enum flow_offload_tuple_dir dir) 735{ 736 const struct nf_flowtable *flowtable = offload->flowtable; 737 const struct flow_offload_tuple *tuple, *other_tuple; 738 struct flow_offload *flow = offload->flow; 739 struct dst_entry *other_dst = NULL; 740 struct nf_flow_rule *flow_rule; 741 int err = -ENOMEM; 742 743 flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL); 744 if (!flow_rule) 745 goto err_flow; 746 747 flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX); 748 if (!flow_rule->rule) 749 goto err_flow_rule; 750 751 flow_rule->rule->match.dissector = &flow_rule->match.dissector; 752 flow_rule->rule->match.mask = &flow_rule->match.mask; 753 flow_rule->rule->match.key = &flow_rule->match.key; 754 755 tuple = &flow->tuplehash[dir].tuple; 756 other_tuple = &flow->tuplehash[!dir].tuple; 757 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) 758 other_dst = other_tuple->dst_cache; 759 760 err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst); 761 if (err < 0) 762 goto err_flow_match; 763 764 flow_rule->rule->action.num_entries = 0; 765 if (flowtable->type->action(net, flow, dir, flow_rule) < 0) 766 goto err_flow_match; 767 768 return flow_rule; 769 770err_flow_match: 771 kfree(flow_rule->rule); 772err_flow_rule: 773 kfree(flow_rule); 774err_flow: 775 return NULL; 776} 777 778static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule) 779{ 780 struct flow_action_entry *entry; 781 int i; 782 783 for (i = 0; i < flow_rule->rule->action.num_entries; i++) { 784 entry = &flow_rule->rule->action.entries[i]; 785 if (entry->id != FLOW_ACTION_REDIRECT) 786 continue; 787 788 dev_put(entry->dev); 789 } 790 kfree(flow_rule->rule); 791 kfree(flow_rule); 792} 793 794static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[]) 795{ 796 int i; 797 798 for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++) 799 __nf_flow_offload_destroy(flow_rule[i]); 800} 801 802static int nf_flow_offload_alloc(const struct flow_offload_work *offload, 803 struct nf_flow_rule *flow_rule[]) 804{ 805 struct net *net = read_pnet(&offload->flowtable->net); 806 807 flow_rule[0] = nf_flow_offload_rule_alloc(net, offload, 808 FLOW_OFFLOAD_DIR_ORIGINAL); 809 if (!flow_rule[0]) 810 return -ENOMEM; 811 812 flow_rule[1] = nf_flow_offload_rule_alloc(net, offload, 813 FLOW_OFFLOAD_DIR_REPLY); 814 if (!flow_rule[1]) { 815 __nf_flow_offload_destroy(flow_rule[0]); 816 return -ENOMEM; 817 } 818 819 return 0; 820} 821 822static void nf_flow_offload_init(struct flow_cls_offload *cls_flow, 823 __be16 proto, int priority, 824 enum flow_cls_command cmd, 825 const struct flow_offload_tuple *tuple, 826 struct netlink_ext_ack *extack) 827{ 828 cls_flow->common.protocol = proto; 829 cls_flow->common.prio = priority; 830 cls_flow->common.extack = extack; 831 cls_flow->command = cmd; 832 cls_flow->cookie = (unsigned long)tuple; 833} 834 835static int nf_flow_offload_tuple(struct nf_flowtable *flowtable, 836 struct flow_offload *flow, 837 struct nf_flow_rule *flow_rule, 838 enum flow_offload_tuple_dir dir, 839 int priority, int cmd, 840 struct flow_stats *stats, 841 struct list_head *block_cb_list) 842{ 843 struct flow_cls_offload cls_flow = {}; 844 struct flow_block_cb *block_cb; 845 struct netlink_ext_ack extack; 846 __be16 proto = ETH_P_ALL; 847 int err, i = 0; 848 849 nf_flow_offload_init(&cls_flow, proto, priority, cmd, 850 &flow->tuplehash[dir].tuple, &extack); 851 if (cmd == FLOW_CLS_REPLACE) 852 cls_flow.rule = flow_rule->rule; 853 854 down_read(&flowtable->flow_block_lock); 855 list_for_each_entry(block_cb, block_cb_list, list) { 856 err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow, 857 block_cb->cb_priv); 858 if (err < 0) 859 continue; 860 861 i++; 862 } 863 up_read(&flowtable->flow_block_lock); 864 865 if (cmd == FLOW_CLS_STATS) 866 memcpy(stats, &cls_flow.stats, sizeof(*stats)); 867 868 return i; 869} 870 871static int flow_offload_tuple_add(struct flow_offload_work *offload, 872 struct nf_flow_rule *flow_rule, 873 enum flow_offload_tuple_dir dir) 874{ 875 return nf_flow_offload_tuple(offload->flowtable, offload->flow, 876 flow_rule, dir, 877 offload->flowtable->priority, 878 FLOW_CLS_REPLACE, NULL, 879 &offload->flowtable->flow_block.cb_list); 880} 881 882static void flow_offload_tuple_del(struct flow_offload_work *offload, 883 enum flow_offload_tuple_dir dir) 884{ 885 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir, 886 offload->flowtable->priority, 887 FLOW_CLS_DESTROY, NULL, 888 &offload->flowtable->flow_block.cb_list); 889} 890 891static int flow_offload_rule_add(struct flow_offload_work *offload, 892 struct nf_flow_rule *flow_rule[]) 893{ 894 int ok_count = 0; 895 896 ok_count += flow_offload_tuple_add(offload, flow_rule[0], 897 FLOW_OFFLOAD_DIR_ORIGINAL); 898 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags)) 899 ok_count += flow_offload_tuple_add(offload, flow_rule[1], 900 FLOW_OFFLOAD_DIR_REPLY); 901 if (ok_count == 0) 902 return -ENOENT; 903 904 return 0; 905} 906 907static void flow_offload_work_add(struct flow_offload_work *offload) 908{ 909 struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX]; 910 int err; 911 912 err = nf_flow_offload_alloc(offload, flow_rule); 913 if (err < 0) 914 return; 915 916 err = flow_offload_rule_add(offload, flow_rule); 917 if (err < 0) 918 goto out; 919 920 set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status); 921 922out: 923 nf_flow_offload_destroy(flow_rule); 924} 925 926static void flow_offload_work_del(struct flow_offload_work *offload) 927{ 928 clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status); 929 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL); 930 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags)) 931 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY); 932 set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags); 933} 934 935static void flow_offload_tuple_stats(struct flow_offload_work *offload, 936 enum flow_offload_tuple_dir dir, 937 struct flow_stats *stats) 938{ 939 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir, 940 offload->flowtable->priority, 941 FLOW_CLS_STATS, stats, 942 &offload->flowtable->flow_block.cb_list); 943} 944 945static void flow_offload_work_stats(struct flow_offload_work *offload) 946{ 947 struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {}; 948 u64 lastused; 949 950 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]); 951 if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags)) 952 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY, 953 &stats[1]); 954 955 lastused = max_t(u64, stats[0].lastused, stats[1].lastused); 956 offload->flow->timeout = max_t(u64, offload->flow->timeout, 957 lastused + flow_offload_get_timeout(offload->flow)); 958 959 if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) { 960 if (stats[0].pkts) 961 nf_ct_acct_add(offload->flow->ct, 962 FLOW_OFFLOAD_DIR_ORIGINAL, 963 stats[0].pkts, stats[0].bytes); 964 if (stats[1].pkts) 965 nf_ct_acct_add(offload->flow->ct, 966 FLOW_OFFLOAD_DIR_REPLY, 967 stats[1].pkts, stats[1].bytes); 968 } 969} 970 971static void flow_offload_work_handler(struct work_struct *work) 972{ 973 struct flow_offload_work *offload; 974 struct net *net; 975 976 offload = container_of(work, struct flow_offload_work, work); 977 net = read_pnet(&offload->flowtable->net); 978 switch (offload->cmd) { 979 case FLOW_CLS_REPLACE: 980 flow_offload_work_add(offload); 981 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add); 982 break; 983 case FLOW_CLS_DESTROY: 984 flow_offload_work_del(offload); 985 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del); 986 break; 987 case FLOW_CLS_STATS: 988 flow_offload_work_stats(offload); 989 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats); 990 break; 991 default: 992 WARN_ON_ONCE(1); 993 } 994 995 clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags); 996 kfree(offload); 997} 998 999static void flow_offload_queue_work(struct flow_offload_work *offload) 1000{ 1001 struct net *net = read_pnet(&offload->flowtable->net); 1002 1003 if (offload->cmd == FLOW_CLS_REPLACE) { 1004 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_add); 1005 queue_work(nf_flow_offload_add_wq, &offload->work); 1006 } else if (offload->cmd == FLOW_CLS_DESTROY) { 1007 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_del); 1008 queue_work(nf_flow_offload_del_wq, &offload->work); 1009 } else { 1010 NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_stats); 1011 queue_work(nf_flow_offload_stats_wq, &offload->work); 1012 } 1013} 1014 1015static struct flow_offload_work * 1016nf_flow_offload_work_alloc(struct nf_flowtable *flowtable, 1017 struct flow_offload *flow, unsigned int cmd) 1018{ 1019 struct flow_offload_work *offload; 1020 1021 if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags)) 1022 return NULL; 1023 1024 offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC); 1025 if (!offload) { 1026 clear_bit(NF_FLOW_HW_PENDING, &flow->flags); 1027 return NULL; 1028 } 1029 1030 offload->cmd = cmd; 1031 offload->flow = flow; 1032 offload->flowtable = flowtable; 1033 INIT_WORK(&offload->work, flow_offload_work_handler); 1034 1035 return offload; 1036} 1037 1038 1039void nf_flow_offload_add(struct nf_flowtable *flowtable, 1040 struct flow_offload *flow) 1041{ 1042 struct flow_offload_work *offload; 1043 1044 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE); 1045 if (!offload) 1046 return; 1047 1048 flow_offload_queue_work(offload); 1049} 1050 1051void nf_flow_offload_del(struct nf_flowtable *flowtable, 1052 struct flow_offload *flow) 1053{ 1054 struct flow_offload_work *offload; 1055 1056 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY); 1057 if (!offload) 1058 return; 1059 1060 set_bit(NF_FLOW_HW_DYING, &flow->flags); 1061 flow_offload_queue_work(offload); 1062} 1063 1064void nf_flow_offload_stats(struct nf_flowtable *flowtable, 1065 struct flow_offload *flow) 1066{ 1067 struct flow_offload_work *offload; 1068 __s32 delta; 1069 1070 delta = nf_flow_timeout_delta(flow->timeout); 1071 if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10)) 1072 return; 1073 1074 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS); 1075 if (!offload) 1076 return; 1077 1078 flow_offload_queue_work(offload); 1079} 1080 1081void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable) 1082{ 1083 if (nf_flowtable_hw_offload(flowtable)) { 1084 flush_workqueue(nf_flow_offload_del_wq); 1085 nf_flow_table_gc_run(flowtable); 1086 } 1087} 1088 1089void nf_flow_table_offload_flush(struct nf_flowtable *flowtable) 1090{ 1091 if (nf_flowtable_hw_offload(flowtable)) { 1092 flush_workqueue(nf_flow_offload_add_wq); 1093 flush_workqueue(nf_flow_offload_del_wq); 1094 flush_workqueue(nf_flow_offload_stats_wq); 1095 } 1096} 1097 1098static int nf_flow_table_block_setup(struct nf_flowtable *flowtable, 1099 struct flow_block_offload *bo, 1100 enum flow_block_command cmd) 1101{ 1102 struct flow_block_cb *block_cb, *next; 1103 int err = 0; 1104 1105 down_write(&flowtable->flow_block_lock); 1106 switch (cmd) { 1107 case FLOW_BLOCK_BIND: 1108 list_splice(&bo->cb_list, &flowtable->flow_block.cb_list); 1109 break; 1110 case FLOW_BLOCK_UNBIND: 1111 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 1112 list_del(&block_cb->list); 1113 flow_block_cb_free(block_cb); 1114 } 1115 break; 1116 default: 1117 WARN_ON_ONCE(1); 1118 err = -EOPNOTSUPP; 1119 } 1120 up_write(&flowtable->flow_block_lock); 1121 1122 return err; 1123} 1124 1125static void nf_flow_table_block_offload_init(struct flow_block_offload *bo, 1126 struct net *net, 1127 enum flow_block_command cmd, 1128 struct nf_flowtable *flowtable, 1129 struct netlink_ext_ack *extack) 1130{ 1131 memset(bo, 0, sizeof(*bo)); 1132 bo->net = net; 1133 bo->block = &flowtable->flow_block; 1134 bo->command = cmd; 1135 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; 1136 bo->extack = extack; 1137 bo->cb_list_head = &flowtable->flow_block.cb_list; 1138 INIT_LIST_HEAD(&bo->cb_list); 1139} 1140 1141static void nf_flow_table_indr_cleanup(struct flow_block_cb *block_cb) 1142{ 1143 struct nf_flowtable *flowtable = block_cb->indr.data; 1144 struct net_device *dev = block_cb->indr.dev; 1145 1146 nf_flow_table_gc_cleanup(flowtable, dev); 1147 down_write(&flowtable->flow_block_lock); 1148 list_del(&block_cb->list); 1149 list_del(&block_cb->driver_list); 1150 flow_block_cb_free(block_cb); 1151 up_write(&flowtable->flow_block_lock); 1152} 1153 1154static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo, 1155 struct nf_flowtable *flowtable, 1156 struct net_device *dev, 1157 enum flow_block_command cmd, 1158 struct netlink_ext_ack *extack) 1159{ 1160 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable, 1161 extack); 1162 1163 return flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_FT, flowtable, bo, 1164 nf_flow_table_indr_cleanup); 1165} 1166 1167static int nf_flow_table_offload_cmd(struct flow_block_offload *bo, 1168 struct nf_flowtable *flowtable, 1169 struct net_device *dev, 1170 enum flow_block_command cmd, 1171 struct netlink_ext_ack *extack) 1172{ 1173 int err; 1174 1175 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable, 1176 extack); 1177 down_write(&flowtable->flow_block_lock); 1178 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo); 1179 up_write(&flowtable->flow_block_lock); 1180 if (err < 0) 1181 return err; 1182 1183 return 0; 1184} 1185 1186int nf_flow_table_offload_setup(struct nf_flowtable *flowtable, 1187 struct net_device *dev, 1188 enum flow_block_command cmd) 1189{ 1190 struct netlink_ext_ack extack = {}; 1191 struct flow_block_offload bo; 1192 int err; 1193 1194 if (!nf_flowtable_hw_offload(flowtable)) 1195 return 0; 1196 1197 if (dev->netdev_ops->ndo_setup_tc) 1198 err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd, 1199 &extack); 1200 else 1201 err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd, 1202 &extack); 1203 if (err < 0) 1204 return err; 1205 1206 return nf_flow_table_block_setup(flowtable, &bo, cmd); 1207} 1208EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup); 1209 1210int nf_flow_table_offload_init(void) 1211{ 1212 nf_flow_offload_add_wq = alloc_workqueue("nf_ft_offload_add", 1213 WQ_UNBOUND | WQ_SYSFS, 0); 1214 if (!nf_flow_offload_add_wq) 1215 return -ENOMEM; 1216 1217 nf_flow_offload_del_wq = alloc_workqueue("nf_ft_offload_del", 1218 WQ_UNBOUND | WQ_SYSFS, 0); 1219 if (!nf_flow_offload_del_wq) 1220 goto err_del_wq; 1221 1222 nf_flow_offload_stats_wq = alloc_workqueue("nf_ft_offload_stats", 1223 WQ_UNBOUND | WQ_SYSFS, 0); 1224 if (!nf_flow_offload_stats_wq) 1225 goto err_stats_wq; 1226 1227 return 0; 1228 1229err_stats_wq: 1230 destroy_workqueue(nf_flow_offload_del_wq); 1231err_del_wq: 1232 destroy_workqueue(nf_flow_offload_add_wq); 1233 return -ENOMEM; 1234} 1235 1236void nf_flow_table_offload_exit(void) 1237{ 1238 destroy_workqueue(nf_flow_offload_add_wq); 1239 destroy_workqueue(nf_flow_offload_del_wq); 1240 destroy_workqueue(nf_flow_offload_stats_wq); 1241} 1242