net/sched/cls_api.c

55 	const struct tcf_proto *tp;
77 tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
84 	if (WARN_ON(!handle || !tp->ops->get_exts))
91 	n->chain_index = tp->chain->index;
92 	n->chain = tp->chain;
93 	n->tp_prio = tp->prio;
94 	n->tp = tp;
133 tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
172 static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
174 	return jhash_3words(tp->chain->index, tp->prio,
175 			    (__force __u32)tp->protocol, 0);
179 					struct tcf_proto *tp)
184 	hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
185 		     destroy_obj_hashfn(tp));
198 					struct tcf_proto *tp)
200 	u32 hash = destroy_obj_hashfn(tp);
207 		if (tcf_proto_cmp(tp, iter)) {
218 tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
223 	if (hash_hashed(&tp->destroy_ht_node))
224 		hash_del_rcu(&tp->destroy_ht_node);
334 static inline u32 tcf_auto_prio(struct tcf_proto *tp)
338 	if (tp)
339 		first = tp->prio - 1;
377 	struct tcf_proto *tp;
380 	tp = kzalloc(sizeof(*tp), GFP_KERNEL);
381 	if (!tp)
384 	tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
385 	if (IS_ERR(tp->ops)) {
386 		err = PTR_ERR(tp->ops);
389 	tp->classify = tp->ops->classify;
390 	tp->protocol = protocol;
391 	tp->prio = prio;
392 	tp->chain = chain;
393 	spin_lock_init(&tp->lock);
394 	refcount_set(&tp->refcnt, 1);
396 	err = tp->ops->init(tp);
398 		module_put(tp->ops->owner);
401 	return tp;
404 	kfree(tp);
408 static void tcf_proto_get(struct tcf_proto *tp)
410 	refcount_inc(&tp->refcnt);
450 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
453 	tp->ops->destroy(tp, rtnl_held, extack);
454 	tcf_block_filter_cnt_update(tp->chain->block, &tp->counted, false);
456 		tcf_proto_signal_destroyed(tp->chain, tp);
457 	tcf_chain_put(tp->chain);
458 	module_put(tp->ops->owner);
459 	kfree_rcu(tp, rcu);
462 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
465 	if (refcount_dec_and_test(&tp->refcnt))
466 		tcf_proto_destroy(tp, rtnl_held, true, extack);
469 static bool tcf_proto_check_delete(struct tcf_proto *tp)
471 	if (tp->ops->delete_empty)
472 		return tp->ops->delete_empty(tp);
474 	tp->deleting = true;
475 	return tp->deleting;
478 static void tcf_proto_mark_delete(struct tcf_proto *tp)
480 	spin_lock(&tp->lock);
481 	tp->deleting = true;
482 	spin_unlock(&tp->lock);
485 static bool tcf_proto_is_deleting(struct tcf_proto *tp)
489 	spin_lock(&tp->lock);
490 	deleting = tp->deleting;
491 	spin_unlock(&tp->lock);
758 	struct tcf_proto *tp, *tp_next;
761 	tp = tcf_chain_dereference(chain->filter_chain, chain);
762 	while (tp) {
763 		tp_next = rcu_dereference_protected(tp->next, 1);
764 		tcf_proto_signal_destroying(chain, tp);
765 		tp = tp_next;
767 	tp = tcf_chain_dereference(chain->filter_chain, chain);
773 	while (tp) {
774 		tp_next = rcu_dereference_protected(tp->next, 1);
775 		tcf_proto_put(tp, rtnl_held, NULL);
776 		tp = tp_next;
1116 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1123 	if (!tp) {
1124 		tp = tcf_chain_dereference(chain->filter_chain, chain);
1125 	} else if (tcf_proto_is_deleting(tp)) {
1130 		prio = tp->prio + 1;
1131 		tp = tcf_chain_dereference(chain->filter_chain, chain);
1133 		for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1134 			if (!tp->deleting && tp->prio >= prio)
1137 		tp = tcf_chain_dereference(tp->next, chain);
1140 	if (tp)
1141 		tcf_proto_get(tp);
1145 	return tp;
1148 /* Function to be used by all clients that want to iterate over all tp's on
1149  * chain. Users of this function must be tolerant to concurrent tp
1157 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1159 	struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1161 	if (tp)
1162 		tcf_proto_put(tp, true, NULL);
1589 	struct tcf_proto *tp, *tp_prev;
1602 		for (tp = __tcf_get_next_proto(chain, NULL); tp;
1603 		     tp_prev = tp,
1604 			     tp = __tcf_get_next_proto(chain, tp),
1606 			if (tp->ops->reoffload) {
1607 				err = tp->ops->reoffload(tp, add, cb, cb_priv,
1625 	tcf_proto_put(tp, true, NULL);
1719 				 const struct tcf_proto *tp,
1734 	for (; tp; tp = rcu_dereference_bh(tp->next)) {
1741 			if (n->tp_prio != tp->prio)
1744 			/* We re-lookup the tp and chain based on index instead
1746 			 * check if any of tp,chain,exts was replaced by the
1749 			if (unlikely(n->tp != tp || n->tp->chain != n->chain ||
1750 				     !tp->ops->get_exts)) {
1756 			exts = tp->ops->get_exts(tp, n->handle);
1766 			if (tp->protocol != protocol &&
1767 			    tp->protocol != htons(ETH_P_ALL))
1770 			err = tc_classify(skb, tp, res);
1798 				       tp->chain->block->index,
1799 				       tp->prio & 0xffff,
1800 				       ntohs(tp->protocol));
1806 	tp = first_tp;
1813 		 const struct tcf_proto *tp,
1819 	return __tcf_classify(skb, tp, tp, res, compat_mode, NULL, 0,
1822 	u32 last_executed_chain = tp ? tp->chain->index : 0;
1824 	const struct tcf_proto *orig_tp = tp;
1861 			tp = rcu_dereference_bh(fchain->filter_chain);
1866 	ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode, n, act_index,
1906 			       struct tcf_proto *tp)
1911 	RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1913 		tcf_chain0_head_change(chain, tp);
1914 	tcf_proto_get(tp);
1915 	rcu_assign_pointer(*chain_info->pprev, tp);
1922 				struct tcf_proto *tp)
1926 	tcf_proto_mark_delete(tp);
1927 	if (tp == chain->filter_chain)
1948 	struct tcf_proto *tp;
1959 	tp = tcf_chain_tp_find(chain, &chain_info,
1961 	if (!tp)
1965 	if (tp) {
1967 		tp_new = tp;
1977 				      struct tcf_proto *tp, bool rtnl_held,
1987 	/* Atomically find and remove tp from chain. */
1991 		if (tp_iter == tp) {
1998 	/* Verify that tp still exists and no new filters were inserted
2000 	 * Mark tp for deletion if it is empty.
2002 	if (!tp_iter || !tcf_proto_check_delete(tp)) {
2007 	tcf_proto_signal_destroying(chain, tp);
2009 	if (tp == chain->filter_chain)
2014 	tcf_proto_put(tp, rtnl_held, extack);
2023 	struct tcf_proto *tp;
2027 	     (tp = tcf_chain_dereference(*pprev, chain));
2028 	     pprev = &tp->next) {
2029 		if (tp->prio >= prio) {
2030 			if (tp->prio == prio) {
2032 				    (tp->protocol != protocol && protocol))
2035 				tp = NULL;
2041 	if (tp) {
2042 		chain_info->next = tp->next;
2043 		tcf_proto_get(tp);
2047 	return tp;
2051 			 struct tcf_proto *tp, struct tcf_block *block,
2075 	tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
2076 	if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
2078 	if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
2083 		if (tp->ops->terse_dump) {
2084 			if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
2091 		if (tp->ops->dump &&
2092 		    tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
2112 			  struct nlmsghdr *n, struct tcf_proto *tp,
2128 	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
2144 			      struct nlmsghdr *n, struct tcf_proto *tp,
2154 		return tp->ops->delete(tp, fh, last, rtnl_held, extack);
2160 	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
2168 	err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
2188 	struct tcf_proto *tp;
2190 	for (tp = tcf_get_next_proto(chain, NULL);
2191 	     tp; tp = tcf_get_next_proto(chain, tp))
2192 		tfilter_notify(net, oskb, n, tp, block, q, parent, NULL,
2196 static void tfilter_put(struct tcf_proto *tp, void *fh)
2198 	if (tp->ops->put && fh)
2199 		tp->ops->put(tp, fh);
2223 	struct tcf_proto *tp;
2244 	tp = NULL;
2313 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2315 	if (IS_ERR(tp)) {
2317 		err = PTR_ERR(tp);
2321 	if (tp == NULL) {
2356 		tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2358 		if (IS_ERR(tp)) {
2359 			err = PTR_ERR(tp);
2366 	if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2372 	fh = tp->ops->get(tp, t->tcm_handle);
2381 		tfilter_put(tp, fh);
2387 	if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2388 		tfilter_put(tp, fh);
2400 	err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
2403 		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2405 		tfilter_put(tp, fh);
2406 		tcf_block_filter_cnt_update(block, &tp->counted, true);
2414 		tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
2417 		if (tp && !IS_ERR(tp))
2418 			tcf_proto_put(tp, rtnl_held, NULL);
2457 	struct tcf_proto *tp = NULL;
2540 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2542 	if (!tp || IS_ERR(tp)) {
2544 		err = tp ? PTR_ERR(tp) : -ENOENT;
2546 	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2551 		tcf_proto_signal_destroying(chain, tp);
2552 		tcf_chain_tp_remove(chain, &chain_info, tp);
2555 		tcf_proto_put(tp, rtnl_held, NULL);
2556 		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2563 	fh = tp->ops->get(tp, t->tcm_handle);
2571 		err = tfilter_del_notify(net, skb, n, tp, block, q, parent, fh,
2577 			tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
2582 		if (tp && !IS_ERR(tp))
2583 			tcf_proto_put(tp, rtnl_held, NULL);
2613 	struct tcf_proto *tp = NULL;
2680 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2683 	if (!tp || IS_ERR(tp)) {
2685 		err = tp ? PTR_ERR(tp) : -ENOENT;
2687 	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2693 	fh = tp->ops->get(tp, t->tcm_handle);
2699 		err = tfilter_notify(net, skb, n, tp, block, q, parent,
2705 	tfilter_put(tp, fh);
2708 		if (tp && !IS_ERR(tp))
2709 			tcf_proto_put(tp, rtnl_held, NULL);
2730 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2735 	return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
2748 	struct tcf_proto *tp, *tp_prev;
2751 	for (tp = __tcf_get_next_proto(chain, NULL);
2752 	     tp;
2753 	     tp_prev = tp,
2754 		     tp = __tcf_get_next_proto(chain, tp),
2760 		    TC_H_MAJ(tcm->tcm_info) != tp->prio)
2763 		    TC_H_MIN(tcm->tcm_info) != tp->protocol)
2769 			if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2776 		if (!tp->ops->walk)
2789 		tp->ops->walk(tp, &arg.w, true);
2798 	tcf_proto_put(tp, true, NULL);
3302 		     int police, struct tcf_proto *tp, u32 handle,
3327 	err = tcf_exts_miss_cookie_base_alloc(exts, tp, handle);
3356 int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3374 			act = tcf_action_init_1(net, tp, tb[exts->police],
3389 			err = tcf_action_init(net, tp, tb[exts->action],
3410 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3414 	return tcf_exts_validate_ex(net, tp, tb, rate_tlv, exts,
3539 				      struct tcf_proto *tp, u32 *cnt,
3544 	spin_lock(&tp->lock);
3554 	spin_unlock(&tp->lock);
3558 tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3563 	spin_lock(&tp->lock);
3566 	spin_unlock(&tp->lock);
3624 int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3655 	if (tp->ops->hw_add)
3656 		tp->ops->hw_add(tp, type_data);
3658 		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
3674 int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3703 	tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
3704 	if (tp->ops->hw_del)
3705 		tp->ops->hw_del(tp, type_data);
3711 	if (tp->ops->hw_add)
3712 		tp->ops->hw_add(tp, type_data);
3714 		tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
3728 int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3751 	tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
3752 	if (tp->ops->hw_del)
3753 		tp->ops->hw_del(tp, type_data);
3762 int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3773 		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,