ifq.c revision 1.11
1/* $OpenBSD: ifq.c,v 1.11 2017/05/03 20:55:29 mikeb Exp $ */ 2 3/* 4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19#include <sys/param.h> 20#include <sys/systm.h> 21#include <sys/socket.h> 22#include <sys/mbuf.h> 23#include <sys/proc.h> 24 25#include <net/if.h> 26#include <net/if_var.h> 27 28/* 29 * priq glue 30 */ 31unsigned int priq_idx(unsigned int, const struct mbuf *); 32struct mbuf *priq_enq(struct ifqueue *, struct mbuf *); 33struct mbuf *priq_deq_begin(struct ifqueue *, void **); 34void priq_deq_commit(struct ifqueue *, struct mbuf *, void *); 35void priq_purge(struct ifqueue *, struct mbuf_list *); 36 37void *priq_alloc(unsigned int, void *); 38void priq_free(unsigned int, void *); 39 40const struct ifq_ops priq_ops = { 41 priq_idx, 42 priq_enq, 43 priq_deq_begin, 44 priq_deq_commit, 45 priq_purge, 46 priq_alloc, 47 priq_free, 48}; 49 50const struct ifq_ops * const ifq_priq_ops = &priq_ops; 51 52/* 53 * priq internal structures 54 */ 55 56struct priq { 57 struct mbuf_list pq_lists[IFQ_NQUEUES]; 58}; 59 60/* 61 * ifqueue serialiser 62 */ 63 64void ifq_start_task(void *); 65void ifq_restart_task(void *); 66void ifq_barrier_task(void *); 67 68#define TASK_ONQUEUE 0x1 69 70void 71ifq_serialize(struct ifqueue *ifq, struct task *t) 72{ 73 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 74 struct task work; 75 76 if (ISSET(t->t_flags, TASK_ONQUEUE)) 77 return; 78 79 mtx_enter(&ifq->ifq_task_mtx); 80 if (!ISSET(t->t_flags, TASK_ONQUEUE)) { 81 SET(t->t_flags, TASK_ONQUEUE); 82 TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry); 83 } 84 85 if (ifq->ifq_serializer == NULL) { 86 ifq->ifq_serializer = curcpu(); 87 88 while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) { 89 TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry); 90 CLR(t->t_flags, TASK_ONQUEUE); 91 work = *t; /* copy to caller to avoid races */ 92 93 mtx_leave(&ifq->ifq_task_mtx); 94 95 (*work.t_func)(work.t_arg); 96 97 mtx_enter(&ifq->ifq_task_mtx); 98 } 99 100 /* 101 * ifq->ifq_free is only modified by dequeue, which 102 * is only called from within this serialization 103 * context. it is therefore safe to access and modify 104 * here without taking ifq->ifq_mtx. 105 */ 106 ml = ifq->ifq_free; 107 ml_init(&ifq->ifq_free); 108 109 ifq->ifq_serializer = NULL; 110 } 111 mtx_leave(&ifq->ifq_task_mtx); 112 113 ml_purge(&ml); 114} 115 116int 117ifq_is_serialized(struct ifqueue *ifq) 118{ 119 return (ifq->ifq_serializer == curcpu()); 120} 121 122void 123ifq_start_task(void *p) 124{ 125 struct ifqueue *ifq = p; 126 struct ifnet *ifp = ifq->ifq_if; 127 128 if (!ISSET(ifp->if_flags, IFF_RUNNING) || 129 ifq_empty(ifq) || ifq_is_oactive(ifq)) 130 return; 131 132 ifp->if_qstart(ifq); 133} 134 135void 136ifq_restart_task(void *p) 137{ 138 struct ifqueue *ifq = p; 139 struct ifnet *ifp = ifq->ifq_if; 140 141 ifq_clr_oactive(ifq); 142 ifp->if_qstart(ifq); 143} 144 145void 146ifq_barrier(struct ifqueue *ifq) 147{ 148 struct sleep_state sls; 149 unsigned int notdone = 1; 150 struct task t = TASK_INITIALIZER(ifq_barrier_task, ¬done); 151 152 /* this should only be called from converted drivers */ 153 KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE)); 154 155 if (ifq->ifq_serializer == NULL) 156 return; 157 158 ifq_serialize(ifq, &t); 159 160 while (notdone) { 161 sleep_setup(&sls, ¬done, PWAIT, "ifqbar"); 162 sleep_finish(&sls, notdone); 163 } 164} 165 166void 167ifq_barrier_task(void *p) 168{ 169 unsigned int *notdone = p; 170 171 *notdone = 0; 172 wakeup_one(notdone); 173} 174 175/* 176 * ifqueue mbuf queue API 177 */ 178 179void 180ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx) 181{ 182 ifq->ifq_if = ifp; 183 ifq->ifq_softc = NULL; 184 185 mtx_init(&ifq->ifq_mtx, IPL_NET); 186 ifq->ifq_qdrops = 0; 187 188 /* default to priq */ 189 ifq->ifq_ops = &priq_ops; 190 ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL); 191 192 ml_init(&ifq->ifq_free); 193 ifq->ifq_len = 0; 194 195 ifq->ifq_packets = 0; 196 ifq->ifq_bytes = 0; 197 ifq->ifq_qdrops = 0; 198 ifq->ifq_errors = 0; 199 ifq->ifq_mcasts = 0; 200 201 mtx_init(&ifq->ifq_task_mtx, IPL_NET); 202 TAILQ_INIT(&ifq->ifq_task_list); 203 ifq->ifq_serializer = NULL; 204 205 task_set(&ifq->ifq_start, ifq_start_task, ifq); 206 task_set(&ifq->ifq_restart, ifq_restart_task, ifq); 207 208 if (ifq->ifq_maxlen == 0) 209 ifq_set_maxlen(ifq, IFQ_MAXLEN); 210 211 ifq->ifq_idx = idx; 212} 213 214void 215ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg) 216{ 217 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 218 struct mbuf_list free_ml = MBUF_LIST_INITIALIZER(); 219 struct mbuf *m; 220 const struct ifq_ops *oldops; 221 void *newq, *oldq; 222 223 newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg); 224 225 mtx_enter(&ifq->ifq_mtx); 226 ifq->ifq_ops->ifqop_purge(ifq, &ml); 227 ifq->ifq_len = 0; 228 229 oldops = ifq->ifq_ops; 230 oldq = ifq->ifq_q; 231 232 ifq->ifq_ops = newops; 233 ifq->ifq_q = newq; 234 235 while ((m = ml_dequeue(&ml)) != NULL) { 236 m = ifq->ifq_ops->ifqop_enq(ifq, m); 237 if (m != NULL) { 238 ifq->ifq_qdrops++; 239 ml_enqueue(&free_ml, m); 240 } else 241 ifq->ifq_len++; 242 } 243 mtx_leave(&ifq->ifq_mtx); 244 245 oldops->ifqop_free(ifq->ifq_idx, oldq); 246 247 ml_purge(&free_ml); 248} 249 250void 251ifq_destroy(struct ifqueue *ifq) 252{ 253 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 254 255 /* don't need to lock because this is the last use of the ifq */ 256 257 ifq->ifq_ops->ifqop_purge(ifq, &ml); 258 ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q); 259 260 ml_purge(&ml); 261} 262 263int 264ifq_enqueue(struct ifqueue *ifq, struct mbuf *m) 265{ 266 struct mbuf *dm; 267 268 mtx_enter(&ifq->ifq_mtx); 269 dm = ifq->ifq_ops->ifqop_enq(ifq, m); 270 if (dm != m) { 271 ifq->ifq_packets++; 272 ifq->ifq_bytes += m->m_pkthdr.len; 273 if (ISSET(m->m_flags, M_MCAST)) 274 ifq->ifq_mcasts++; 275 } 276 277 if (dm == NULL) 278 ifq->ifq_len++; 279 else 280 ifq->ifq_qdrops++; 281 mtx_leave(&ifq->ifq_mtx); 282 283 if (dm != NULL) 284 m_freem(dm); 285 286 return (dm == m ? ENOBUFS : 0); 287} 288 289struct mbuf * 290ifq_deq_begin(struct ifqueue *ifq) 291{ 292 struct mbuf *m = NULL; 293 void *cookie; 294 295 mtx_enter(&ifq->ifq_mtx); 296 if (ifq->ifq_len == 0 || 297 (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) { 298 mtx_leave(&ifq->ifq_mtx); 299 return (NULL); 300 } 301 302 m->m_pkthdr.ph_cookie = cookie; 303 304 return (m); 305} 306 307void 308ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m) 309{ 310 void *cookie; 311 312 KASSERT(m != NULL); 313 cookie = m->m_pkthdr.ph_cookie; 314 315 ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie); 316 ifq->ifq_len--; 317 mtx_leave(&ifq->ifq_mtx); 318} 319 320void 321ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m) 322{ 323 KASSERT(m != NULL); 324 325 mtx_leave(&ifq->ifq_mtx); 326} 327 328struct mbuf * 329ifq_dequeue(struct ifqueue *ifq) 330{ 331 struct mbuf *m; 332 333 m = ifq_deq_begin(ifq); 334 if (m == NULL) 335 return (NULL); 336 337 ifq_deq_commit(ifq, m); 338 339 return (m); 340} 341 342unsigned int 343ifq_purge(struct ifqueue *ifq) 344{ 345 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 346 unsigned int rv; 347 348 mtx_enter(&ifq->ifq_mtx); 349 ifq->ifq_ops->ifqop_purge(ifq, &ml); 350 rv = ifq->ifq_len; 351 ifq->ifq_len = 0; 352 ifq->ifq_qdrops += rv; 353 mtx_leave(&ifq->ifq_mtx); 354 355 KASSERT(rv == ml_len(&ml)); 356 357 ml_purge(&ml); 358 359 return (rv); 360} 361 362void * 363ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops) 364{ 365 mtx_enter(&ifq->ifq_mtx); 366 if (ifq->ifq_ops == ops) 367 return (ifq->ifq_q); 368 369 mtx_leave(&ifq->ifq_mtx); 370 371 return (NULL); 372} 373 374void 375ifq_q_leave(struct ifqueue *ifq, void *q) 376{ 377 KASSERT(q == ifq->ifq_q); 378 mtx_leave(&ifq->ifq_mtx); 379} 380 381void 382ifq_mfreem(struct ifqueue *ifq, struct mbuf *m) 383{ 384 IFQ_ASSERT_SERIALIZED(ifq); 385 386 ifq->ifq_len--; 387 ifq->ifq_qdrops++; 388 ml_enqueue(&ifq->ifq_free, m); 389} 390 391void 392ifq_mfreeml(struct ifqueue *ifq, struct mbuf_list *ml) 393{ 394 IFQ_ASSERT_SERIALIZED(ifq); 395 396 ifq->ifq_len -= ml_len(ml); 397 ifq->ifq_qdrops += ml_len(ml); 398 ml_enlist(&ifq->ifq_free, ml); 399} 400 401/* 402 * priq implementation 403 */ 404 405unsigned int 406priq_idx(unsigned int nqueues, const struct mbuf *m) 407{ 408 unsigned int flow = 0; 409 410 if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID)) 411 flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK; 412 413 return (flow % nqueues); 414} 415 416void * 417priq_alloc(unsigned int idx, void *null) 418{ 419 struct priq *pq; 420 int i; 421 422 pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK); 423 for (i = 0; i < IFQ_NQUEUES; i++) 424 ml_init(&pq->pq_lists[i]); 425 return (pq); 426} 427 428void 429priq_free(unsigned int idx, void *pq) 430{ 431 free(pq, M_DEVBUF, sizeof(struct priq)); 432} 433 434struct mbuf * 435priq_enq(struct ifqueue *ifq, struct mbuf *m) 436{ 437 struct priq *pq; 438 struct mbuf_list *pl; 439 struct mbuf *n = NULL; 440 unsigned int prio; 441 442 pq = ifq->ifq_q; 443 KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO); 444 445 /* Find a lower priority queue to drop from */ 446 if (ifq_len(ifq) >= ifq->ifq_maxlen) { 447 for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) { 448 pl = &pq->pq_lists[prio]; 449 if (ml_len(pl) > 0) { 450 n = ml_dequeue(pl); 451 goto enqueue; 452 } 453 } 454 /* 455 * There's no lower priority queue that we can 456 * drop from so don't enqueue this one. 457 */ 458 return (m); 459 } 460 461 enqueue: 462 pl = &pq->pq_lists[m->m_pkthdr.pf.prio]; 463 ml_enqueue(pl, m); 464 465 return (n); 466} 467 468struct mbuf * 469priq_deq_begin(struct ifqueue *ifq, void **cookiep) 470{ 471 struct priq *pq = ifq->ifq_q; 472 struct mbuf_list *pl; 473 unsigned int prio = nitems(pq->pq_lists); 474 struct mbuf *m; 475 476 do { 477 pl = &pq->pq_lists[--prio]; 478 m = MBUF_LIST_FIRST(pl); 479 if (m != NULL) { 480 *cookiep = pl; 481 return (m); 482 } 483 } while (prio > 0); 484 485 return (NULL); 486} 487 488void 489priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie) 490{ 491 struct mbuf_list *pl = cookie; 492 493 KASSERT(MBUF_LIST_FIRST(pl) == m); 494 495 ml_dequeue(pl); 496} 497 498void 499priq_purge(struct ifqueue *ifq, struct mbuf_list *ml) 500{ 501 struct priq *pq = ifq->ifq_q; 502 struct mbuf_list *pl; 503 unsigned int prio = nitems(pq->pq_lists); 504 505 do { 506 pl = &pq->pq_lists[--prio]; 507 ml_enlist(ml, pl); 508 } while (prio > 0); 509} 510