net/sched/cls_api.c

54 	const struct tcf_chain *chain;
91 	n->chain_index = tp->chain->index;
92 	n->chain = tp->chain;
174 	return jhash_3words(tp->chain->index, tp->prio,
178 static void tcf_proto_signal_destroying(struct tcf_chain *chain,
181 	struct tcf_block *block = chain->block;
192 	return tp1->chain->index == tp2->chain->index &&
197 static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
205 	hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
218 tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
220 	struct tcf_block *block = chain->block;
373 					  u32 prio, struct tcf_chain *chain,
392 	tp->chain = chain;
413 static void tcf_chain_put(struct tcf_chain *chain);
420 		tcf_proto_signal_destroyed(tp->chain, tp);
421 	tcf_chain_put(tp->chain);
472 	struct tcf_chain *chain;
476 	chain = kzalloc(sizeof(*chain), GFP_KERNEL);
477 	if (!chain)
479 	list_add_tail_rcu(&chain->list, &block->chain_list);
480 	mutex_init(&chain->filter_chain_lock);
481 	chain->block = block;
482 	chain->index = chain_index;
483 	chain->refcnt = 1;
484 	if (!chain->index)
485 		block->chain0.chain = chain;
486 	return chain;
496 static void tcf_chain0_head_change(struct tcf_chain *chain,
500 	struct tcf_block *block = chain->block;
502 	if (chain->index)
513 static bool tcf_chain_detach(struct tcf_chain *chain)
515 	struct tcf_block *block = chain->block;
519 	list_del_rcu(&chain->list);
520 	if (!chain->index)
521 		block->chain0.chain = NULL;
538 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
540 	struct tcf_block *block = chain->block;
542 	mutex_destroy(&chain->filter_chain_lock);
543 	kfree_rcu(chain, rcu);
548 static void tcf_chain_hold(struct tcf_chain *chain)
550 	ASSERT_BLOCK_LOCKED(chain->block);
552 	++chain->refcnt;
555 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
557 	ASSERT_BLOCK_LOCKED(chain->block);
560 	 * chain should not be shown to the user.
562 	return chain->refcnt == chain->action_refcnt;
568 	struct tcf_chain *chain;
572 	list_for_each_entry(chain, &block->chain_list, list) {
573 		if (chain->index == chain_index)
574 			return chain;
583 	struct tcf_chain *chain;
585 	list_for_each_entry_rcu(chain, &block->chain_list, list) {
586 		if (chain->index == chain_index)
587 			return chain;
593 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
601 	struct tcf_chain *chain = NULL;
605 	chain = tcf_chain_lookup(block, chain_index);
606 	if (chain) {
607 		tcf_chain_hold(chain);
611 		chain = tcf_chain_create(block, chain_index);
612 		if (!chain)
617 		++chain->action_refcnt;
618 	is_first_reference = chain->refcnt - chain->action_refcnt == 1;
622 	 * non-action reference. Until then, the chain acts only as
627 		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
630 	return chain;
634 	return chain;
656 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
659 	struct tcf_block *block = chain->block;
667 		if (!chain->explicitly_created) {
671 		chain->explicitly_created = false;
675 		chain->action_refcnt--;
678 	 * However, when block is unlocked chain can be changed concurrently, so
681 	refcnt = --chain->refcnt;
682 	non_act_refcnt = refcnt - chain->action_refcnt;
683 	tmplt_ops = chain->tmplt_ops;
684 	tmplt_priv = chain->tmplt_priv;
686 	if (non_act_refcnt == chain->explicitly_created && !by_act) {
689 					       chain->index, block, NULL, 0, 0);
690 		/* Last reference to chain, no need to lock. */
691 		chain->flushing = false;
695 		free_block = tcf_chain_detach(chain);
700 		tcf_chain_destroy(chain, free_block);
704 static void tcf_chain_put(struct tcf_chain *chain)
706 	__tcf_chain_put(chain, false, false);
709 void tcf_chain_put_by_act(struct tcf_chain *chain)
711 	__tcf_chain_put(chain, true, false);
715 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
717 	__tcf_chain_put(chain, false, true);
720 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
724 	mutex_lock(&chain->filter_chain_lock);
725 	tp = tcf_chain_dereference(chain->filter_chain, chain);
728 		tcf_proto_signal_destroying(chain, tp);
731 	tp = tcf_chain_dereference(chain->filter_chain, chain);
732 	RCU_INIT_POINTER(chain->filter_chain, NULL);
733 	tcf_chain0_head_change(chain, NULL);
734 	chain->flushing = true;
735 	mutex_unlock(&chain->filter_chain_lock);
901 	chain0 = block->chain0.chain;
939 			if (block->chain0.chain)
1036 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
1039 	if (chain)
1040 		chain = list_is_last(&chain->list, &block->chain_list) ?
1041 			NULL : list_next_entry(chain, list);
1043 		chain = list_first_entry_or_null(&block->chain_list,
1047 	while (chain && tcf_chain_held_by_acts_only(chain))
1048 		chain = list_is_last(&chain->list, &block->chain_list) ?
1049 			NULL : list_next_entry(chain, list);
1051 	if (chain)
1052 		tcf_chain_hold(chain);
1055 	return chain;
1059  * block. It properly obtains block->lock and takes reference to chain before
1060  * returning it. Users of this function must be tolerant to concurrent chain
1061  * insertion/deletion or ensure that no concurrent chain modification is
1068 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
1070 	struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
1072 	if (chain)
1073 		tcf_chain_put(chain);
1080 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1085 	mutex_lock(&chain->filter_chain_lock);
1088 		tp = tcf_chain_dereference(chain->filter_chain, chain);
1090 		/* 'deleting' flag is set and chain->filter_chain_lock was
1095 		tp = tcf_chain_dereference(chain->filter_chain, chain);
1097 		for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1101 		tp = tcf_chain_dereference(tp->next, chain);
1107 	mutex_unlock(&chain->filter_chain_lock);
1113  * chain. Users of this function must be tolerant to concurrent tp
1114  * insertion/deletion or ensure that no concurrent chain modification is
1121 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1123 	struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1134 	struct tcf_chain *chain;
1139 	for (chain = tcf_get_next_chain(block, NULL);
1140 	     chain;
1141 	     chain = tcf_get_next_chain(block, chain)) {
1142 		tcf_chain_put_explicitly_created(chain);
1143 		tcf_chain_flush(chain, rtnl_held);
1288 		 * deallocated when last chain is freed. However, if chain_list
1517 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
1552 	struct tcf_chain *chain, *chain_prev;
1558 	for (chain = __tcf_get_next_chain(block, NULL);
1559 	     chain;
1560 	     chain_prev = chain,
1561 		     chain = __tcf_get_next_chain(block, chain),
1563 		if (chain->tmplt_ops && add)
1564 			chain->tmplt_ops->tmplt_reoffload(chain, true, cb,
1566 		for (tp = __tcf_get_next_proto(chain, NULL); tp;
1568 			     tp = __tcf_get_next_proto(chain, tp),
1581 		if (chain->tmplt_ops && !add)
1582 			chain->tmplt_ops->tmplt_reoffload(chain, false, cb,
1590 	tcf_chain_put(chain);
1678 /* Main classifier routine: scans classifier chain attached
1708 			/* We re-lookup the tp and chain based on index instead
1710 			 * check if any of tp,chain,exts was replaced by the
1713 			if (unlikely(n->tp != tp || n->tp->chain != n->chain ||
1739 			*last_executed_chain = first_tp->chain->index;
1762 				       tp->chain->block->index,
1786 	u32 last_executed_chain = tp ? tp->chain->index : 0;
1796 		if (ext && (ext->chain || ext->act_miss)) {
1798 			u32 chain;
1809 				chain = n->chain_index;
1811 				chain = ext->chain;
1814 			fchain = tcf_chain_lookup_rcu(block, chain);
1834 		/* If we missed on some chain */
1843 			ext->chain = last_executed_chain;
1862 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1865 	return tcf_chain_dereference(*chain_info->pprev, chain);
1868 static int tcf_chain_tp_insert(struct tcf_chain *chain,
1872 	if (chain->flushing)
1875 	RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1876 	if (*chain_info->pprev == chain->filter_chain)
1877 		tcf_chain0_head_change(chain, tp);
1884 static void tcf_chain_tp_remove(struct tcf_chain *chain,
1888 	struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1891 	if (tp == chain->filter_chain)
1892 		tcf_chain0_head_change(chain, next);
1896 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1906 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1915 	mutex_lock(&chain->filter_chain_lock);
1917 	if (tcf_proto_exists_destroying(chain, tp_new)) {
1918 		mutex_unlock(&chain->filter_chain_lock);
1923 	tp = tcf_chain_tp_find(chain, &chain_info,
1926 		err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
1927 	mutex_unlock(&chain->filter_chain_lock);
1940 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
1949 	mutex_lock(&chain->filter_chain_lock);
1951 	/* Atomically find and remove tp from chain. */
1952 	for (pprev = &chain->filter_chain;
1953 	     (tp_iter = tcf_chain_dereference(*pprev, chain));
1967 		mutex_unlock(&chain->filter_chain_lock);
1971 	tcf_proto_signal_destroying(chain, tp);
1972 	next = tcf_chain_dereference(chain_info.next, chain);
1973 	if (tp == chain->filter_chain)
1974 		tcf_chain0_head_change(chain, next);
1976 	mutex_unlock(&chain->filter_chain_lock);
1981 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1989 	/* Check the chain for existence of proto-tcf with this priority */
1990 	for (pprev = &chain->filter_chain;
1991 	     (tp = tcf_chain_dereference(*pprev, chain));
2042 	if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
2149 				 struct tcf_chain *chain, int event,
2154 	for (tp = tcf_get_next_proto(chain, NULL);
2155 	     tp; tp = tcf_get_next_proto(chain, tp))
2185 	struct tcf_chain *chain;
2212 	chain = NULL;
2228 	/* Find head of filter chain. */
2265 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2269 	chain = tcf_chain_get(block, chain_index, true);
2270 	if (!chain) {
2271 		NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
2276 	mutex_lock(&chain->filter_chain_lock);
2277 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2288 		if (chain->flushing) {
2308 			prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2311 		mutex_unlock(&chain->filter_chain_lock);
2312 		tp_new = tcf_proto_create(name, protocol, prio, chain,
2320 		tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2327 		mutex_unlock(&chain->filter_chain_lock);
2351 	if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2377 		tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
2379 	if (chain) {
2383 			tcf_chain_put(chain);
2392 		 * of target chain.
2401 	mutex_unlock(&chain->filter_chain_lock);
2418 	struct tcf_chain *chain = NULL;
2441 	/* Find head of filter chain. */
2452 	/* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2476 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2480 	chain = tcf_chain_get(block, chain_index, false);
2481 	if (!chain) {
2482 		/* User requested flush on non-existent chain. Nothing to do,
2489 		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2496 				     chain, RTM_DELTFILTER, extack);
2497 		tcf_chain_flush(chain, rtnl_held);
2502 	mutex_lock(&chain->filter_chain_lock);
2503 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2514 		tcf_proto_signal_destroying(chain, tp);
2515 		tcf_chain_tp_remove(chain, &chain_info, tp);
2516 		mutex_unlock(&chain->filter_chain_lock);
2524 	mutex_unlock(&chain->filter_chain_lock);
2540 			tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
2544 	if (chain) {
2547 		tcf_chain_put(chain);
2557 	mutex_unlock(&chain->filter_chain_lock);
2574 	struct tcf_chain *chain = NULL;
2597 	/* Find head of filter chain. */
2631 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2635 	chain = tcf_chain_get(block, chain_index, false);
2636 	if (!chain) {
2637 		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2642 	mutex_lock(&chain->filter_chain_lock);
2643 	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2645 	mutex_unlock(&chain->filter_chain_lock);
2670 	if (chain) {
2673 		tcf_chain_put(chain);
2704 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2709 	struct tcf_block *block = chain->block;
2714 	for (tp = __tcf_get_next_proto(chain, NULL);
2717 		     tp = __tcf_get_next_proto(chain, tp),
2773 	struct tcf_chain *chain, *chain_prev;
2848 	for (chain = __tcf_get_next_chain(block, NULL);
2849 	     chain;
2850 	     chain_prev = chain,
2851 		     chain = __tcf_get_next_chain(block, chain),
2854 		    nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2856 		if (!tcf_chain_dump(chain, q, parent, skb, cb,
2858 			tcf_chain_put(chain);
2931 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2936 	struct tcf_block *block = chain->block;
2948 	if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2949 			       chain->index, net, skb, block, portid,
2989 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
3002 		NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
3016 	tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
3021 	chain->tmplt_ops = ops;
3022 	chain->tmplt_priv = tmplt_priv;
3037 /* Add/delete/get a chain */
3048 	struct tcf_chain *chain;
3071 		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
3077 	chain = tcf_chain_lookup(block, chain_index);
3079 		if (chain) {
3080 			if (tcf_chain_held_by_acts_only(chain)) {
3081 				/* The chain exists only because there is
3084 				tcf_chain_hold(chain);
3086 				NL_SET_ERR_MSG(extack, "Filter chain already exists");
3092 				NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
3096 			chain = tcf_chain_create(block, chain_index);
3097 			if (!chain) {
3098 				NL_SET_ERR_MSG(extack, "Failed to create filter chain");
3104 		if (!chain || tcf_chain_held_by_acts_only(chain)) {
3105 			NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
3109 		tcf_chain_hold(chain);
3113 		/* Modifying chain requires holding parent block lock. In case
3114 		 * the chain was successfully added, take a reference to the
3115 		 * chain. This ensures that an empty chain does not disappear at
3118 		tcf_chain_hold(chain);
3119 		chain->explicitly_created = true;
3125 		err = tc_chain_tmplt_add(chain, net, tca, extack);
3127 			tcf_chain_put_explicitly_created(chain);
3131 		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
3136 				     chain, RTM_DELTFILTER, extack);
3137 		/* Flush the chain first as the user requested chain removal. */
3138 		tcf_chain_flush(chain, true);
3139 		/* In case the chain was successfully deleted, put a reference
3140 		 * to the chain previously taken during addition.
3142 		tcf_chain_put_explicitly_created(chain);
3145 		err = tc_chain_notify(chain, skb, n->nlmsg_seq,
3148 			NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
3157 	tcf_chain_put(chain);
3178 	struct tcf_chain *chain;
3232 	list_for_each_entry(chain, &block->chain_list, list) {
3234 		     nla_get_u32(tca[TCA_CHAIN]) != chain->index))
3240 		if (tcf_chain_held_by_acts_only(chain))
3242 		err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
3243 					 chain->index, net, skb, block,