ifq.c revision 1.9
1/* $OpenBSD: ifq.c,v 1.9 2017/03/07 15:42:02 mikeb Exp $ */ 2 3/* 4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19#include <sys/param.h> 20#include <sys/systm.h> 21#include <sys/socket.h> 22#include <sys/mbuf.h> 23#include <sys/proc.h> 24 25#include <net/if.h> 26#include <net/if_var.h> 27 28/* 29 * priq glue 30 */ 31unsigned int priq_idx(unsigned int, const struct mbuf *); 32struct mbuf *priq_enq(struct ifqueue *, struct mbuf *); 33struct mbuf *priq_deq_begin(struct ifqueue *, void **); 34void priq_deq_commit(struct ifqueue *, struct mbuf *, void *); 35void priq_purge(struct ifqueue *, struct mbuf_list *); 36 37void *priq_alloc(unsigned int, void *); 38void priq_free(unsigned int, void *); 39 40const struct ifq_ops priq_ops = { 41 priq_idx, 42 priq_enq, 43 priq_deq_begin, 44 priq_deq_commit, 45 priq_purge, 46 priq_alloc, 47 priq_free, 48}; 49 50const struct ifq_ops * const ifq_priq_ops = &priq_ops; 51 52/* 53 * priq internal structures 54 */ 55 56struct priq { 57 struct mbuf_list pq_lists[IFQ_NQUEUES]; 58}; 59 60/* 61 * ifqueue serialiser 62 */ 63 64void ifq_start_task(void *); 65void ifq_restart_task(void *); 66void ifq_barrier_task(void *); 67 68#define TASK_ONQUEUE 0x1 69 70void 71ifq_serialize(struct ifqueue *ifq, struct task *t) 72{ 73 struct task work; 74 75 if (ISSET(t->t_flags, TASK_ONQUEUE)) 76 return; 77 78 mtx_enter(&ifq->ifq_task_mtx); 79 if (!ISSET(t->t_flags, TASK_ONQUEUE)) { 80 SET(t->t_flags, TASK_ONQUEUE); 81 TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry); 82 } 83 84 if (ifq->ifq_serializer == NULL) { 85 ifq->ifq_serializer = curcpu(); 86 87 while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) { 88 TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry); 89 CLR(t->t_flags, TASK_ONQUEUE); 90 work = *t; /* copy to caller to avoid races */ 91 92 mtx_leave(&ifq->ifq_task_mtx); 93 94 (*work.t_func)(work.t_arg); 95 96 mtx_enter(&ifq->ifq_task_mtx); 97 } 98 99 ifq->ifq_serializer = NULL; 100 } 101 mtx_leave(&ifq->ifq_task_mtx); 102} 103 104int 105ifq_is_serialized(struct ifqueue *ifq) 106{ 107 return (ifq->ifq_serializer == curcpu()); 108} 109 110void 111ifq_start_task(void *p) 112{ 113 struct ifqueue *ifq = p; 114 struct ifnet *ifp = ifq->ifq_if; 115 116 if (!ISSET(ifp->if_flags, IFF_RUNNING) || 117 ifq_empty(ifq) || ifq_is_oactive(ifq)) 118 return; 119 120 ifp->if_qstart(ifq); 121} 122 123void 124ifq_restart_task(void *p) 125{ 126 struct ifqueue *ifq = p; 127 struct ifnet *ifp = ifq->ifq_if; 128 129 ifq_clr_oactive(ifq); 130 ifp->if_qstart(ifq); 131} 132 133void 134ifq_barrier(struct ifqueue *ifq) 135{ 136 struct sleep_state sls; 137 unsigned int notdone = 1; 138 struct task t = TASK_INITIALIZER(ifq_barrier_task, ¬done); 139 140 /* this should only be called from converted drivers */ 141 KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE)); 142 143 if (ifq->ifq_serializer == NULL) 144 return; 145 146 ifq_serialize(ifq, &t); 147 148 while (notdone) { 149 sleep_setup(&sls, ¬done, PWAIT, "ifqbar"); 150 sleep_finish(&sls, notdone); 151 } 152} 153 154void 155ifq_barrier_task(void *p) 156{ 157 unsigned int *notdone = p; 158 159 *notdone = 0; 160 wakeup_one(notdone); 161} 162 163/* 164 * ifqueue mbuf queue API 165 */ 166 167void 168ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx) 169{ 170 ifq->ifq_if = ifp; 171 ifq->ifq_softc = NULL; 172 173 mtx_init(&ifq->ifq_mtx, IPL_NET); 174 ifq->ifq_qdrops = 0; 175 176 /* default to priq */ 177 ifq->ifq_ops = &priq_ops; 178 ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL); 179 180 ifq->ifq_len = 0; 181 182 ifq->ifq_packets = 0; 183 ifq->ifq_bytes = 0; 184 ifq->ifq_qdrops = 0; 185 ifq->ifq_errors = 0; 186 ifq->ifq_mcasts = 0; 187 188 mtx_init(&ifq->ifq_task_mtx, IPL_NET); 189 TAILQ_INIT(&ifq->ifq_task_list); 190 ifq->ifq_serializer = NULL; 191 192 task_set(&ifq->ifq_start, ifq_start_task, ifq); 193 task_set(&ifq->ifq_restart, ifq_restart_task, ifq); 194 195 if (ifq->ifq_maxlen == 0) 196 ifq_set_maxlen(ifq, IFQ_MAXLEN); 197 198 ifq->ifq_idx = idx; 199} 200 201void 202ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg) 203{ 204 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 205 struct mbuf_list free_ml = MBUF_LIST_INITIALIZER(); 206 struct mbuf *m; 207 const struct ifq_ops *oldops; 208 void *newq, *oldq; 209 210 newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg); 211 212 mtx_enter(&ifq->ifq_mtx); 213 ifq->ifq_ops->ifqop_purge(ifq, &ml); 214 ifq->ifq_len = 0; 215 216 oldops = ifq->ifq_ops; 217 oldq = ifq->ifq_q; 218 219 ifq->ifq_ops = newops; 220 ifq->ifq_q = newq; 221 222 while ((m = ml_dequeue(&ml)) != NULL) { 223 m = ifq->ifq_ops->ifqop_enq(ifq, m); 224 if (m != NULL) { 225 ifq->ifq_qdrops++; 226 ml_enqueue(&free_ml, m); 227 } else 228 ifq->ifq_len++; 229 } 230 mtx_leave(&ifq->ifq_mtx); 231 232 oldops->ifqop_free(ifq->ifq_idx, oldq); 233 234 ml_purge(&free_ml); 235} 236 237void 238ifq_destroy(struct ifqueue *ifq) 239{ 240 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 241 242 /* don't need to lock because this is the last use of the ifq */ 243 244 ifq->ifq_ops->ifqop_purge(ifq, &ml); 245 ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q); 246 247 ml_purge(&ml); 248} 249 250int 251ifq_enqueue(struct ifqueue *ifq, struct mbuf *m) 252{ 253 struct mbuf *dm; 254 255 mtx_enter(&ifq->ifq_mtx); 256 dm = ifq->ifq_ops->ifqop_enq(ifq, m); 257 if (dm != m) { 258 ifq->ifq_packets++; 259 ifq->ifq_bytes += m->m_pkthdr.len; 260 if (ISSET(m->m_flags, M_MCAST)) 261 ifq->ifq_mcasts++; 262 } 263 264 if (dm == NULL) 265 ifq->ifq_len++; 266 else 267 ifq->ifq_qdrops++; 268 mtx_leave(&ifq->ifq_mtx); 269 270 if (dm != NULL) 271 m_freem(dm); 272 273 return (dm == m ? ENOBUFS : 0); 274} 275 276struct mbuf * 277ifq_deq_begin(struct ifqueue *ifq) 278{ 279 struct mbuf *m = NULL; 280 void *cookie; 281 282 mtx_enter(&ifq->ifq_mtx); 283 if (ifq->ifq_len == 0 || 284 (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) { 285 mtx_leave(&ifq->ifq_mtx); 286 return (NULL); 287 } 288 289 m->m_pkthdr.ph_cookie = cookie; 290 291 return (m); 292} 293 294void 295ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m) 296{ 297 void *cookie; 298 299 KASSERT(m != NULL); 300 cookie = m->m_pkthdr.ph_cookie; 301 302 ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie); 303 ifq->ifq_len--; 304 mtx_leave(&ifq->ifq_mtx); 305} 306 307void 308ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m) 309{ 310 KASSERT(m != NULL); 311 312 mtx_leave(&ifq->ifq_mtx); 313} 314 315struct mbuf * 316ifq_dequeue(struct ifqueue *ifq) 317{ 318 struct mbuf *m; 319 320 m = ifq_deq_begin(ifq); 321 if (m == NULL) 322 return (NULL); 323 324 ifq_deq_commit(ifq, m); 325 326 return (m); 327} 328 329unsigned int 330ifq_purge(struct ifqueue *ifq) 331{ 332 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 333 unsigned int rv; 334 335 mtx_enter(&ifq->ifq_mtx); 336 ifq->ifq_ops->ifqop_purge(ifq, &ml); 337 rv = ifq->ifq_len; 338 ifq->ifq_len = 0; 339 ifq->ifq_qdrops += rv; 340 mtx_leave(&ifq->ifq_mtx); 341 342 KASSERT(rv == ml_len(&ml)); 343 344 ml_purge(&ml); 345 346 return (rv); 347} 348 349void * 350ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops) 351{ 352 mtx_enter(&ifq->ifq_mtx); 353 if (ifq->ifq_ops == ops) 354 return (ifq->ifq_q); 355 356 mtx_leave(&ifq->ifq_mtx); 357 358 return (NULL); 359} 360 361void 362ifq_q_leave(struct ifqueue *ifq, void *q) 363{ 364 KASSERT(q == ifq->ifq_q); 365 mtx_leave(&ifq->ifq_mtx); 366} 367 368/* 369 * priq implementation 370 */ 371 372unsigned int 373priq_idx(unsigned int nqueues, const struct mbuf *m) 374{ 375 unsigned int flow = 0; 376 377 if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID)) 378 flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK; 379 380 return (flow % nqueues); 381} 382 383void * 384priq_alloc(unsigned int idx, void *null) 385{ 386 struct priq *pq; 387 int i; 388 389 pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK); 390 for (i = 0; i < IFQ_NQUEUES; i++) 391 ml_init(&pq->pq_lists[i]); 392 return (pq); 393} 394 395void 396priq_free(unsigned int idx, void *pq) 397{ 398 free(pq, M_DEVBUF, sizeof(struct priq)); 399} 400 401struct mbuf * 402priq_enq(struct ifqueue *ifq, struct mbuf *m) 403{ 404 struct priq *pq; 405 struct mbuf_list *pl; 406 struct mbuf *n = NULL; 407 unsigned int prio; 408 409 pq = ifq->ifq_q; 410 KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO); 411 412 /* Find a lower priority queue to drop from */ 413 if (ifq_len(ifq) >= ifq->ifq_maxlen) { 414 for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) { 415 pl = &pq->pq_lists[prio]; 416 if (ml_len(pl) > 0) { 417 n = ml_dequeue(pl); 418 goto enqueue; 419 } 420 } 421 /* 422 * There's no lower priority queue that we can 423 * drop from so don't enqueue this one. 424 */ 425 return (m); 426 } 427 428 enqueue: 429 pl = &pq->pq_lists[m->m_pkthdr.pf.prio]; 430 ml_enqueue(pl, m); 431 432 return (n); 433} 434 435struct mbuf * 436priq_deq_begin(struct ifqueue *ifq, void **cookiep) 437{ 438 struct priq *pq = ifq->ifq_q; 439 struct mbuf_list *pl; 440 unsigned int prio = nitems(pq->pq_lists); 441 struct mbuf *m; 442 443 do { 444 pl = &pq->pq_lists[--prio]; 445 m = MBUF_LIST_FIRST(pl); 446 if (m != NULL) { 447 *cookiep = pl; 448 return (m); 449 } 450 } while (prio > 0); 451 452 return (NULL); 453} 454 455void 456priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie) 457{ 458 struct mbuf_list *pl = cookie; 459 460 KASSERT(MBUF_LIST_FIRST(pl) == m); 461 462 ml_dequeue(pl); 463} 464 465void 466priq_purge(struct ifqueue *ifq, struct mbuf_list *ml) 467{ 468 struct priq *pq = ifq->ifq_q; 469 struct mbuf_list *pl; 470 unsigned int prio = nitems(pq->pq_lists); 471 472 do { 473 pl = &pq->pq_lists[--prio]; 474 ml_enlist(ml, pl); 475 } while (prio > 0); 476} 477