net/sched/sch_generic.c

31 #include <trace/events/qdisc.h>
63  * qdisc_lock(qdisc) spinlock.
66  * - enqueue, dequeue are serialized via qdisc root lock
67  * - ingress filtering is also serialized via qdisc root lock
308  * required. Owning qdisc running bit guarantees that only one CPU
322 	/* And release qdisc */
374  * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
633 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
640 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
655 	RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
682 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
688 	qdisc->enqueue = NULL;
727 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
731 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
739 		if (qdisc_is_percpu_stats(qdisc))
740 			return qdisc_drop_cpu(skb, qdisc, to_free);
742 			return qdisc_drop(skb, qdisc, to_free);
745 	qdisc_update_stats_at_enqueue(qdisc, pkt_len);
749 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
751 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
766 		qdisc_update_stats_at_dequeue(qdisc, skb);
768 		   READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) {
774 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
775 		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
790 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
792 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
805 static void pfifo_fast_reset(struct Qdisc *qdisc)
808 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
824 	if (qdisc_is_percpu_stats(qdisc)) {
828 			q = per_cpu_ptr(qdisc->cpu_qstats, i);
835 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
848 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
851 	unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
852 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
869 	qdisc->flags |= TCQ_F_CAN_BYPASS;
972 	/* seqlock has the same scope of busylock, for NOLOCK qdisc */
1023 /* Under qdisc_lock(qdisc) and BH! */
1025 void qdisc_reset(struct Qdisc *qdisc)
1027 	const struct Qdisc_ops *ops = qdisc->ops;
1029 	trace_qdisc_reset(qdisc);
1032 		ops->reset(qdisc);
1034 	__skb_queue_purge(&qdisc->gso_skb);
1035 	__skb_queue_purge(&qdisc->skb_bad_txq);
1037 	qdisc->q.qlen = 0;
1038 	qdisc->qstats.backlog = 0;
1042 void qdisc_free(struct Qdisc *qdisc)
1044 	if (qdisc_is_percpu_stats(qdisc)) {
1045 		free_percpu(qdisc->cpu_bstats);
1046 		free_percpu(qdisc->cpu_qstats);
1049 	kfree(qdisc);
1059 static void __qdisc_destroy(struct Qdisc *qdisc)
1061 	const struct Qdisc_ops  *ops = qdisc->ops;
1062 	struct net_device *dev = qdisc_dev(qdisc);
1065 	qdisc_hash_del(qdisc);
1067 	qdisc_put_stab(rtnl_dereference(qdisc->stab));
1069 	gen_kill_estimator(&qdisc->rate_est);
1071 	qdisc_reset(qdisc);
1075 		ops->destroy(qdisc);
1077 	lockdep_unregister_key(&qdisc->root_lock_key);
1079 	netdev_put(dev, &qdisc->dev_tracker);
1081 	trace_qdisc_destroy(qdisc);
1083 	call_rcu(&qdisc->rcu, qdisc_free_cb);
1086 void qdisc_destroy(struct Qdisc *qdisc)
1088 	if (qdisc->flags & TCQ_F_BUILTIN)
1091 	__qdisc_destroy(qdisc);
1094 void qdisc_put(struct Qdisc *qdisc)
1096 	if (!qdisc)
1099 	if (qdisc->flags & TCQ_F_BUILTIN ||
1100 	    !refcount_dec_and_test(&qdisc->refcnt))
1103 	__qdisc_destroy(qdisc);
1109  * qdisc reference counter reached zero.
1112 void qdisc_put_unlocked(struct Qdisc *qdisc)
1114 	if (qdisc->flags & TCQ_F_BUILTIN ||
1115 	    !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
1118 	__qdisc_destroy(qdisc);
1123 /* Attach toplevel qdisc to device queue. */
1125 			      struct Qdisc *qdisc)
1134 	if (qdisc == NULL)
1135 		qdisc = &noop_qdisc;
1136 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1137 	rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1149 	struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1152 	if (qdisc) {
1153 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1156 		qdisc_put(qdisc);
1164 	struct Qdisc *qdisc;
1172 	qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
1173 	if (!qdisc)
1177 		qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
1178 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1184 	struct Qdisc *qdisc;
1191 		qdisc = rtnl_dereference(txq->qdisc_sleeping);
1192 		rcu_assign_pointer(dev->qdisc, qdisc);
1193 		qdisc_refcount_inc(qdisc);
1195 		qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
1196 		if (qdisc) {
1197 			rcu_assign_pointer(dev->qdisc, qdisc);
1198 			qdisc->ops->attach(qdisc);
1201 	qdisc = rtnl_dereference(dev->qdisc);
1203 	/* Detect default qdisc setup/init failed and fallback to "noqueue" */
1204 	if (qdisc == &noop_qdisc) {
1205 		netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
1210 		qdisc = rtnl_dereference(txq->qdisc_sleeping);
1211 		rcu_assign_pointer(dev->qdisc, qdisc);
1212 		qdisc_refcount_inc(qdisc);
1217 	if (qdisc != &noop_qdisc)
1218 		qdisc_hash_add(qdisc, false);
1232 	rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
1248 	if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
1267 static void qdisc_deactivate(struct Qdisc *qdisc)
1269 	if (qdisc->flags & TCQ_F_BUILTIN)
1272 	set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
1280 	struct Qdisc *qdisc;
1282 	qdisc = rtnl_dereference(dev_queue->qdisc);
1283 	if (qdisc) {
1284 		qdisc_deactivate(qdisc);
1285 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1293 	struct Qdisc *qdisc;
1296 	qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1297 	if (!qdisc)
1300 	nolock = qdisc->flags & TCQ_F_NOLOCK;
1303 		spin_lock_bh(&qdisc->seqlock);
1304 	spin_lock_bh(qdisc_lock(qdisc));
1306 	qdisc_reset(qdisc);
1308 	spin_unlock_bh(qdisc_lock(qdisc));
1310 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
1311 		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
1312 		spin_unlock_bh(&qdisc->seqlock);
1364 	/* Wait for outstanding qdisc-less dev_queue_xmit calls or
1365 	 * outstanding qdisc enqueuing calls.
1403 	struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1404 	const struct Qdisc_ops *ops = qdisc->ops;
1407 		return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
1414 	struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);
1416 	if (qdisc->ops->change_real_num_tx)
1417 		qdisc->ops->change_real_num_tx(qdisc, new_real_tx);
1424 	struct Qdisc *qdisc;
1428 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1432 		if (qdisc != &noop_qdisc && !qdisc->handle)
1433 			qdisc_hash_del(qdisc);
1436 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1437 		if (qdisc != &noop_qdisc && !qdisc->handle)
1438 			qdisc_hash_add(qdisc, false);
1470 	struct Qdisc *qdisc = _qdisc;
1472 	rcu_assign_pointer(dev_queue->qdisc, qdisc);
1473 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1478 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1491 	qdisc_put(rtnl_dereference(dev->qdisc));
1492 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1606 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1609 	miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
1610 	miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
1611 	miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
1612 	miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;