1#ifndef _NET_FLOW_OFFLOAD_H 2#define _NET_FLOW_OFFLOAD_H 3 4#include <linux/kernel.h> 5#include <linux/list.h> 6#include <linux/netlink.h> 7#include <net/flow_dissector.h> 8 9struct flow_match { 10 struct flow_dissector *dissector; 11 void *mask; 12 void *key; 13}; 14 15struct flow_match_meta { 16 struct flow_dissector_key_meta *key, *mask; 17}; 18 19struct flow_match_basic { 20 struct flow_dissector_key_basic *key, *mask; 21}; 22 23struct flow_match_control { 24 struct flow_dissector_key_control *key, *mask; 25}; 26 27struct flow_match_eth_addrs { 28 struct flow_dissector_key_eth_addrs *key, *mask; 29}; 30 31struct flow_match_vlan { 32 struct flow_dissector_key_vlan *key, *mask; 33}; 34 35struct flow_match_arp { 36 struct flow_dissector_key_arp *key, *mask; 37}; 38 39struct flow_match_ipv4_addrs { 40 struct flow_dissector_key_ipv4_addrs *key, *mask; 41}; 42 43struct flow_match_ipv6_addrs { 44 struct flow_dissector_key_ipv6_addrs *key, *mask; 45}; 46 47struct flow_match_ip { 48 struct flow_dissector_key_ip *key, *mask; 49}; 50 51struct flow_match_ports { 52 struct flow_dissector_key_ports *key, *mask; 53}; 54 55struct flow_match_ports_range { 56 struct flow_dissector_key_ports_range *key, *mask; 57}; 58 59struct flow_match_icmp { 60 struct flow_dissector_key_icmp *key, *mask; 61}; 62 63struct flow_match_tcp { 64 struct flow_dissector_key_tcp *key, *mask; 65}; 66 67struct flow_match_ipsec { 68 struct flow_dissector_key_ipsec *key, *mask; 69}; 70 71struct flow_match_mpls { 72 struct flow_dissector_key_mpls *key, *mask; 73}; 74 75struct flow_match_enc_keyid { 76 struct flow_dissector_key_keyid *key, *mask; 77}; 78 79struct flow_match_enc_opts { 80 struct flow_dissector_key_enc_opts *key, *mask; 81}; 82 83struct flow_match_ct { 84 struct flow_dissector_key_ct *key, *mask; 85}; 86 87struct flow_match_pppoe { 88 struct flow_dissector_key_pppoe *key, *mask; 89}; 90 91struct flow_match_l2tpv3 { 92 struct flow_dissector_key_l2tpv3 *key, *mask; 93}; 94 95struct flow_rule; 96 97void flow_rule_match_meta(const struct flow_rule *rule, 98 struct flow_match_meta *out); 99void flow_rule_match_basic(const struct flow_rule *rule, 100 struct flow_match_basic *out); 101void flow_rule_match_control(const struct flow_rule *rule, 102 struct flow_match_control *out); 103void flow_rule_match_eth_addrs(const struct flow_rule *rule, 104 struct flow_match_eth_addrs *out); 105void flow_rule_match_vlan(const struct flow_rule *rule, 106 struct flow_match_vlan *out); 107void flow_rule_match_cvlan(const struct flow_rule *rule, 108 struct flow_match_vlan *out); 109void flow_rule_match_arp(const struct flow_rule *rule, 110 struct flow_match_arp *out); 111void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 112 struct flow_match_ipv4_addrs *out); 113void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 114 struct flow_match_ipv6_addrs *out); 115void flow_rule_match_ip(const struct flow_rule *rule, 116 struct flow_match_ip *out); 117void flow_rule_match_ports(const struct flow_rule *rule, 118 struct flow_match_ports *out); 119void flow_rule_match_ports_range(const struct flow_rule *rule, 120 struct flow_match_ports_range *out); 121void flow_rule_match_tcp(const struct flow_rule *rule, 122 struct flow_match_tcp *out); 123void flow_rule_match_ipsec(const struct flow_rule *rule, 124 struct flow_match_ipsec *out); 125void flow_rule_match_icmp(const struct flow_rule *rule, 126 struct flow_match_icmp *out); 127void flow_rule_match_mpls(const struct flow_rule *rule, 128 struct flow_match_mpls *out); 129void flow_rule_match_enc_control(const struct flow_rule *rule, 130 struct flow_match_control *out); 131void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 132 struct flow_match_ipv4_addrs *out); 133void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 134 struct flow_match_ipv6_addrs *out); 135void flow_rule_match_enc_ip(const struct flow_rule *rule, 136 struct flow_match_ip *out); 137void flow_rule_match_enc_ports(const struct flow_rule *rule, 138 struct flow_match_ports *out); 139void flow_rule_match_enc_keyid(const struct flow_rule *rule, 140 struct flow_match_enc_keyid *out); 141void flow_rule_match_enc_opts(const struct flow_rule *rule, 142 struct flow_match_enc_opts *out); 143void flow_rule_match_ct(const struct flow_rule *rule, 144 struct flow_match_ct *out); 145void flow_rule_match_pppoe(const struct flow_rule *rule, 146 struct flow_match_pppoe *out); 147void flow_rule_match_l2tpv3(const struct flow_rule *rule, 148 struct flow_match_l2tpv3 *out); 149 150enum flow_action_id { 151 FLOW_ACTION_ACCEPT = 0, 152 FLOW_ACTION_DROP, 153 FLOW_ACTION_TRAP, 154 FLOW_ACTION_GOTO, 155 FLOW_ACTION_REDIRECT, 156 FLOW_ACTION_MIRRED, 157 FLOW_ACTION_REDIRECT_INGRESS, 158 FLOW_ACTION_MIRRED_INGRESS, 159 FLOW_ACTION_VLAN_PUSH, 160 FLOW_ACTION_VLAN_POP, 161 FLOW_ACTION_VLAN_MANGLE, 162 FLOW_ACTION_TUNNEL_ENCAP, 163 FLOW_ACTION_TUNNEL_DECAP, 164 FLOW_ACTION_MANGLE, 165 FLOW_ACTION_ADD, 166 FLOW_ACTION_CSUM, 167 FLOW_ACTION_MARK, 168 FLOW_ACTION_PTYPE, 169 FLOW_ACTION_PRIORITY, 170 FLOW_ACTION_RX_QUEUE_MAPPING, 171 FLOW_ACTION_WAKE, 172 FLOW_ACTION_QUEUE, 173 FLOW_ACTION_SAMPLE, 174 FLOW_ACTION_POLICE, 175 FLOW_ACTION_CT, 176 FLOW_ACTION_CT_METADATA, 177 FLOW_ACTION_MPLS_PUSH, 178 FLOW_ACTION_MPLS_POP, 179 FLOW_ACTION_MPLS_MANGLE, 180 FLOW_ACTION_GATE, 181 FLOW_ACTION_PPPOE_PUSH, 182 FLOW_ACTION_JUMP, 183 FLOW_ACTION_PIPE, 184 FLOW_ACTION_VLAN_PUSH_ETH, 185 FLOW_ACTION_VLAN_POP_ETH, 186 FLOW_ACTION_CONTINUE, 187 NUM_FLOW_ACTIONS, 188}; 189 190/* This is mirroring enum pedit_header_type definition for easy mapping between 191 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 192 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 193 */ 194enum flow_action_mangle_base { 195 FLOW_ACT_MANGLE_UNSPEC = 0, 196 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 197 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 198 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 199 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 200 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 201}; 202 203enum flow_action_hw_stats_bit { 204 FLOW_ACTION_HW_STATS_IMMEDIATE_BIT, 205 FLOW_ACTION_HW_STATS_DELAYED_BIT, 206 FLOW_ACTION_HW_STATS_DISABLED_BIT, 207 208 FLOW_ACTION_HW_STATS_NUM_BITS 209}; 210 211enum flow_action_hw_stats { 212 FLOW_ACTION_HW_STATS_IMMEDIATE = 213 BIT(FLOW_ACTION_HW_STATS_IMMEDIATE_BIT), 214 FLOW_ACTION_HW_STATS_DELAYED = BIT(FLOW_ACTION_HW_STATS_DELAYED_BIT), 215 FLOW_ACTION_HW_STATS_ANY = FLOW_ACTION_HW_STATS_IMMEDIATE | 216 FLOW_ACTION_HW_STATS_DELAYED, 217 FLOW_ACTION_HW_STATS_DISABLED = 218 BIT(FLOW_ACTION_HW_STATS_DISABLED_BIT), 219 FLOW_ACTION_HW_STATS_DONT_CARE = BIT(FLOW_ACTION_HW_STATS_NUM_BITS) - 1, 220}; 221 222typedef void (*action_destr)(void *priv); 223 224struct flow_action_cookie { 225 u32 cookie_len; 226 u8 cookie[]; 227}; 228 229struct flow_action_cookie *flow_action_cookie_create(void *data, 230 unsigned int len, 231 gfp_t gfp); 232void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 233 234struct flow_action_entry { 235 enum flow_action_id id; 236 u32 hw_index; 237 unsigned long cookie; 238 u64 miss_cookie; 239 enum flow_action_hw_stats hw_stats; 240 action_destr destructor; 241 void *destructor_priv; 242 union { 243 u32 chain_index; /* FLOW_ACTION_GOTO */ 244 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 245 struct { /* FLOW_ACTION_VLAN */ 246 u16 vid; 247 __be16 proto; 248 u8 prio; 249 } vlan; 250 struct { /* FLOW_ACTION_VLAN_PUSH_ETH */ 251 unsigned char dst[ETH_ALEN]; 252 unsigned char src[ETH_ALEN]; 253 } vlan_push_eth; 254 struct { /* FLOW_ACTION_MANGLE */ 255 /* FLOW_ACTION_ADD */ 256 enum flow_action_mangle_base htype; 257 u32 offset; 258 u32 mask; 259 u32 val; 260 } mangle; 261 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 262 u32 csum_flags; /* FLOW_ACTION_CSUM */ 263 u32 mark; /* FLOW_ACTION_MARK */ 264 u16 ptype; /* FLOW_ACTION_PTYPE */ 265 u16 rx_queue; /* FLOW_ACTION_RX_QUEUE_MAPPING */ 266 u32 priority; /* FLOW_ACTION_PRIORITY */ 267 struct { /* FLOW_ACTION_QUEUE */ 268 u32 ctx; 269 u32 index; 270 u8 vf; 271 } queue; 272 struct { /* FLOW_ACTION_SAMPLE */ 273 struct psample_group *psample_group; 274 u32 rate; 275 u32 trunc_size; 276 bool truncate; 277 } sample; 278 struct { /* FLOW_ACTION_POLICE */ 279 u32 burst; 280 u64 rate_bytes_ps; 281 u64 peakrate_bytes_ps; 282 u32 avrate; 283 u16 overhead; 284 u64 burst_pkt; 285 u64 rate_pkt_ps; 286 u32 mtu; 287 struct { 288 enum flow_action_id act_id; 289 u32 extval; 290 } exceed, notexceed; 291 } police; 292 struct { /* FLOW_ACTION_CT */ 293 int action; 294 u16 zone; 295 struct nf_flowtable *flow_table; 296 } ct; 297 struct { 298 unsigned long cookie; 299 u32 mark; 300 u32 labels[4]; 301 bool orig_dir; 302 } ct_metadata; 303 struct { /* FLOW_ACTION_MPLS_PUSH */ 304 u32 label; 305 __be16 proto; 306 u8 tc; 307 u8 bos; 308 u8 ttl; 309 } mpls_push; 310 struct { /* FLOW_ACTION_MPLS_POP */ 311 __be16 proto; 312 } mpls_pop; 313 struct { /* FLOW_ACTION_MPLS_MANGLE */ 314 u32 label; 315 u8 tc; 316 u8 bos; 317 u8 ttl; 318 } mpls_mangle; 319 struct { 320 s32 prio; 321 u64 basetime; 322 u64 cycletime; 323 u64 cycletimeext; 324 u32 num_entries; 325 struct action_gate_entry *entries; 326 } gate; 327 struct { /* FLOW_ACTION_PPPOE_PUSH */ 328 u16 sid; 329 } pppoe; 330 }; 331 struct flow_action_cookie *user_cookie; /* user defined action cookie */ 332}; 333 334struct flow_action { 335 unsigned int num_entries; 336 struct flow_action_entry entries[] __counted_by(num_entries); 337}; 338 339static inline bool flow_action_has_entries(const struct flow_action *action) 340{ 341 return action->num_entries; 342} 343 344/** 345 * flow_offload_has_one_action() - check if exactly one action is present 346 * @action: tc filter flow offload action 347 * 348 * Returns true if exactly one action is present. 349 */ 350static inline bool flow_offload_has_one_action(const struct flow_action *action) 351{ 352 return action->num_entries == 1; 353} 354 355static inline bool flow_action_is_last_entry(const struct flow_action *action, 356 const struct flow_action_entry *entry) 357{ 358 return entry == &action->entries[action->num_entries - 1]; 359} 360 361#define flow_action_for_each(__i, __act, __actions) \ 362 for (__i = 0, __act = &(__actions)->entries[0]; \ 363 __i < (__actions)->num_entries; \ 364 __act = &(__actions)->entries[++__i]) 365 366static inline bool 367flow_action_mixed_hw_stats_check(const struct flow_action *action, 368 struct netlink_ext_ack *extack) 369{ 370 const struct flow_action_entry *action_entry; 371 u8 last_hw_stats; 372 int i; 373 374 if (flow_offload_has_one_action(action)) 375 return true; 376 377 flow_action_for_each(i, action_entry, action) { 378 if (i && action_entry->hw_stats != last_hw_stats) { 379 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 380 return false; 381 } 382 last_hw_stats = action_entry->hw_stats; 383 } 384 return true; 385} 386 387static inline const struct flow_action_entry * 388flow_action_first_entry_get(const struct flow_action *action) 389{ 390 WARN_ON(!flow_action_has_entries(action)); 391 return &action->entries[0]; 392} 393 394static inline bool 395__flow_action_hw_stats_check(const struct flow_action *action, 396 struct netlink_ext_ack *extack, 397 bool check_allow_bit, 398 enum flow_action_hw_stats_bit allow_bit) 399{ 400 const struct flow_action_entry *action_entry; 401 402 if (!flow_action_has_entries(action)) 403 return true; 404 if (!flow_action_mixed_hw_stats_check(action, extack)) 405 return false; 406 407 action_entry = flow_action_first_entry_get(action); 408 409 /* Zero is not a legal value for hw_stats, catch anyone passing it */ 410 WARN_ON_ONCE(!action_entry->hw_stats); 411 412 if (!check_allow_bit && 413 ~action_entry->hw_stats & FLOW_ACTION_HW_STATS_ANY) { 414 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 415 return false; 416 } else if (check_allow_bit && 417 !(action_entry->hw_stats & BIT(allow_bit))) { 418 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 419 return false; 420 } 421 return true; 422} 423 424static inline bool 425flow_action_hw_stats_check(const struct flow_action *action, 426 struct netlink_ext_ack *extack, 427 enum flow_action_hw_stats_bit allow_bit) 428{ 429 return __flow_action_hw_stats_check(action, extack, true, allow_bit); 430} 431 432static inline bool 433flow_action_basic_hw_stats_check(const struct flow_action *action, 434 struct netlink_ext_ack *extack) 435{ 436 return __flow_action_hw_stats_check(action, extack, false, 0); 437} 438 439struct flow_rule { 440 struct flow_match match; 441 struct flow_action action; 442}; 443 444struct flow_rule *flow_rule_alloc(unsigned int num_actions); 445 446static inline bool flow_rule_match_key(const struct flow_rule *rule, 447 enum flow_dissector_key_id key) 448{ 449 return dissector_uses_key(rule->match.dissector, key); 450} 451 452struct flow_stats { 453 u64 pkts; 454 u64 bytes; 455 u64 drops; 456 u64 lastused; 457 enum flow_action_hw_stats used_hw_stats; 458 bool used_hw_stats_valid; 459}; 460 461static inline void flow_stats_update(struct flow_stats *flow_stats, 462 u64 bytes, u64 pkts, 463 u64 drops, u64 lastused, 464 enum flow_action_hw_stats used_hw_stats) 465{ 466 flow_stats->pkts += pkts; 467 flow_stats->bytes += bytes; 468 flow_stats->drops += drops; 469 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 470 471 /* The driver should pass value with a maximum of one bit set. 472 * Passing FLOW_ACTION_HW_STATS_ANY is invalid. 473 */ 474 WARN_ON(used_hw_stats == FLOW_ACTION_HW_STATS_ANY); 475 flow_stats->used_hw_stats |= used_hw_stats; 476 flow_stats->used_hw_stats_valid = true; 477} 478 479enum flow_block_command { 480 FLOW_BLOCK_BIND, 481 FLOW_BLOCK_UNBIND, 482}; 483 484enum flow_block_binder_type { 485 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 486 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 487 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 488 FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP, 489 FLOW_BLOCK_BINDER_TYPE_RED_MARK, 490}; 491 492struct flow_block { 493 struct list_head cb_list; 494}; 495 496struct netlink_ext_ack; 497 498struct flow_block_offload { 499 enum flow_block_command command; 500 enum flow_block_binder_type binder_type; 501 bool block_shared; 502 bool unlocked_driver_cb; 503 struct net *net; 504 struct flow_block *block; 505 struct list_head cb_list; 506 struct list_head *driver_block_list; 507 struct netlink_ext_ack *extack; 508 struct Qdisc *sch; 509 struct list_head *cb_list_head; 510}; 511 512enum tc_setup_type; 513typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 514 void *cb_priv); 515 516struct flow_block_cb; 517 518struct flow_block_indr { 519 struct list_head list; 520 struct net_device *dev; 521 struct Qdisc *sch; 522 enum flow_block_binder_type binder_type; 523 void *data; 524 void *cb_priv; 525 void (*cleanup)(struct flow_block_cb *block_cb); 526}; 527 528struct flow_block_cb { 529 struct list_head driver_list; 530 struct list_head list; 531 flow_setup_cb_t *cb; 532 void *cb_ident; 533 void *cb_priv; 534 void (*release)(void *cb_priv); 535 struct flow_block_indr indr; 536 unsigned int refcnt; 537}; 538 539struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 540 void *cb_ident, void *cb_priv, 541 void (*release)(void *cb_priv)); 542struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb, 543 void *cb_ident, void *cb_priv, 544 void (*release)(void *cb_priv), 545 struct flow_block_offload *bo, 546 struct net_device *dev, 547 struct Qdisc *sch, void *data, 548 void *indr_cb_priv, 549 void (*cleanup)(struct flow_block_cb *block_cb)); 550void flow_block_cb_free(struct flow_block_cb *block_cb); 551 552struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 553 flow_setup_cb_t *cb, void *cb_ident); 554 555void *flow_block_cb_priv(struct flow_block_cb *block_cb); 556void flow_block_cb_incref(struct flow_block_cb *block_cb); 557unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 558 559static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 560 struct flow_block_offload *offload) 561{ 562 list_add_tail(&block_cb->list, &offload->cb_list); 563} 564 565static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 566 struct flow_block_offload *offload) 567{ 568 list_move(&block_cb->list, &offload->cb_list); 569} 570 571static inline void flow_indr_block_cb_remove(struct flow_block_cb *block_cb, 572 struct flow_block_offload *offload) 573{ 574 list_del(&block_cb->indr.list); 575 list_move(&block_cb->list, &offload->cb_list); 576} 577 578bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 579 struct list_head *driver_block_list); 580 581int flow_block_cb_setup_simple(struct flow_block_offload *f, 582 struct list_head *driver_list, 583 flow_setup_cb_t *cb, 584 void *cb_ident, void *cb_priv, bool ingress_only); 585 586enum flow_cls_command { 587 FLOW_CLS_REPLACE, 588 FLOW_CLS_DESTROY, 589 FLOW_CLS_STATS, 590 FLOW_CLS_TMPLT_CREATE, 591 FLOW_CLS_TMPLT_DESTROY, 592}; 593 594struct flow_cls_common_offload { 595 u32 chain_index; 596 __be16 protocol; 597 u32 prio; 598 struct netlink_ext_ack *extack; 599}; 600 601struct flow_cls_offload { 602 struct flow_cls_common_offload common; 603 enum flow_cls_command command; 604 bool use_act_stats; 605 unsigned long cookie; 606 struct flow_rule *rule; 607 struct flow_stats stats; 608 u32 classid; 609}; 610 611enum offload_act_command { 612 FLOW_ACT_REPLACE, 613 FLOW_ACT_DESTROY, 614 FLOW_ACT_STATS, 615}; 616 617struct flow_offload_action { 618 struct netlink_ext_ack *extack; /* NULL in FLOW_ACT_STATS process*/ 619 enum offload_act_command command; 620 enum flow_action_id id; 621 u32 index; 622 unsigned long cookie; 623 struct flow_stats stats; 624 struct flow_action action; 625}; 626 627struct flow_offload_action *offload_action_alloc(unsigned int num_actions); 628 629static inline struct flow_rule * 630flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 631{ 632 return flow_cmd->rule; 633} 634 635static inline void flow_block_init(struct flow_block *flow_block) 636{ 637 INIT_LIST_HEAD(&flow_block->cb_list); 638} 639 640typedef int flow_indr_block_bind_cb_t(struct net_device *dev, struct Qdisc *sch, void *cb_priv, 641 enum tc_setup_type type, void *type_data, 642 void *data, 643 void (*cleanup)(struct flow_block_cb *block_cb)); 644 645int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv); 646void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv, 647 void (*release)(void *cb_priv)); 648int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch, 649 enum tc_setup_type type, void *data, 650 struct flow_block_offload *bo, 651 void (*cleanup)(struct flow_block_cb *block_cb)); 652bool flow_indr_dev_exists(void); 653 654#endif /* _NET_FLOW_OFFLOAD_H */ 655