netisr.c revision 256281
1/*- 2 * Copyright (c) 2007-2009 Robert N. M. Watson 3 * Copyright (c) 2010-2011 Juniper Networks, Inc. 4 * All rights reserved. 5 * 6 * This software was developed by Robert N. M. Watson under contract 7 * to Juniper Networks, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: stable/10/sys/net/netisr.c 255329 2013-09-06 21:02:43Z davide $"); 33 34/* 35 * netisr is a packet dispatch service, allowing synchronous (directly 36 * dispatched) and asynchronous (deferred dispatch) processing of packets by 37 * registered protocol handlers. Callers pass a protocol identifier and 38 * packet to netisr, along with a direct dispatch hint, and work will either 39 * be immediately processed by the registered handler, or passed to a 40 * software interrupt (SWI) thread for deferred dispatch. Callers will 41 * generally select one or the other based on: 42 * 43 * - Whether directly dispatching a netisr handler lead to code reentrance or 44 * lock recursion, such as entering the socket code from the socket code. 45 * - Whether directly dispatching a netisr handler lead to recursive 46 * processing, such as when decapsulating several wrapped layers of tunnel 47 * information (IPSEC within IPSEC within ...). 48 * 49 * Maintaining ordering for protocol streams is a critical design concern. 50 * Enforcing ordering limits the opportunity for concurrency, but maintains 51 * the strong ordering requirements found in some protocols, such as TCP. Of 52 * related concern is CPU affinity--it is desirable to process all data 53 * associated with a particular stream on the same CPU over time in order to 54 * avoid acquiring locks associated with the connection on different CPUs, 55 * keep connection data in one cache, and to generally encourage associated 56 * user threads to live on the same CPU as the stream. It's also desirable 57 * to avoid lock migration and contention where locks are associated with 58 * more than one flow. 59 * 60 * netisr supports several policy variations, represented by the 61 * NETISR_POLICY_* constants, allowing protocols to play various roles in 62 * identifying flows, assigning work to CPUs, etc. These are described in 63 * netisr.h. 64 */ 65 66#include "opt_ddb.h" 67#include "opt_device_polling.h" 68 69#include <sys/param.h> 70#include <sys/bus.h> 71#include <sys/kernel.h> 72#include <sys/kthread.h> 73#include <sys/interrupt.h> 74#include <sys/lock.h> 75#include <sys/mbuf.h> 76#include <sys/mutex.h> 77#include <sys/pcpu.h> 78#include <sys/proc.h> 79#include <sys/rmlock.h> 80#include <sys/sched.h> 81#include <sys/smp.h> 82#include <sys/socket.h> 83#include <sys/sysctl.h> 84#include <sys/systm.h> 85 86#ifdef DDB 87#include <ddb/ddb.h> 88#endif 89 90#define _WANT_NETISR_INTERNAL /* Enable definitions from netisr_internal.h */ 91#include <net/if.h> 92#include <net/if_var.h> 93#include <net/netisr.h> 94#include <net/netisr_internal.h> 95#include <net/vnet.h> 96 97/*- 98 * Synchronize use and modification of the registered netisr data structures; 99 * acquire a read lock while modifying the set of registered protocols to 100 * prevent partially registered or unregistered protocols from being run. 101 * 102 * The following data structures and fields are protected by this lock: 103 * 104 * - The netisr_proto array, including all fields of struct netisr_proto. 105 * - The nws array, including all fields of struct netisr_worker. 106 * - The nws_array array. 107 * 108 * Note: the NETISR_LOCKING define controls whether read locks are acquired 109 * in packet processing paths requiring netisr registration stability. This 110 * is disabled by default as it can lead to measurable performance 111 * degradation even with rmlocks (3%-6% for loopback ping-pong traffic), and 112 * because netisr registration and unregistration is extremely rare at 113 * runtime. If it becomes more common, this decision should be revisited. 114 * 115 * XXXRW: rmlocks don't support assertions. 116 */ 117static struct rmlock netisr_rmlock; 118#define NETISR_LOCK_INIT() rm_init_flags(&netisr_rmlock, "netisr", \ 119 RM_NOWITNESS) 120#define NETISR_LOCK_ASSERT() 121#define NETISR_RLOCK(tracker) rm_rlock(&netisr_rmlock, (tracker)) 122#define NETISR_RUNLOCK(tracker) rm_runlock(&netisr_rmlock, (tracker)) 123#define NETISR_WLOCK() rm_wlock(&netisr_rmlock) 124#define NETISR_WUNLOCK() rm_wunlock(&netisr_rmlock) 125/* #define NETISR_LOCKING */ 126 127static SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr"); 128 129/*- 130 * Three global direct dispatch policies are supported: 131 * 132 * NETISR_DISPATCH_QUEUED: All work is deferred for a netisr, regardless of 133 * context (may be overriden by protocols). 134 * 135 * NETISR_DISPATCH_HYBRID: If the executing context allows direct dispatch, 136 * and we're running on the CPU the work would be performed on, then direct 137 * dispatch it if it wouldn't violate ordering constraints on the workstream. 138 * 139 * NETISR_DISPATCH_DIRECT: If the executing context allows direct dispatch, 140 * always direct dispatch. (The default.) 141 * 142 * Notice that changing the global policy could lead to short periods of 143 * misordered processing, but this is considered acceptable as compared to 144 * the complexity of enforcing ordering during policy changes. Protocols can 145 * override the global policy (when they're not doing that, they select 146 * NETISR_DISPATCH_DEFAULT). 147 */ 148#define NETISR_DISPATCH_POLICY_DEFAULT NETISR_DISPATCH_DIRECT 149#define NETISR_DISPATCH_POLICY_MAXSTR 20 /* Used for temporary buffers. */ 150static u_int netisr_dispatch_policy = NETISR_DISPATCH_POLICY_DEFAULT; 151static int sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS); 152SYSCTL_PROC(_net_isr, OID_AUTO, dispatch, CTLTYPE_STRING | CTLFLAG_RW | 153 CTLFLAG_TUN, 0, 0, sysctl_netisr_dispatch_policy, "A", 154 "netisr dispatch policy"); 155 156/* 157 * Allow the administrator to limit the number of threads (CPUs) to use for 158 * netisr. We don't check netisr_maxthreads before creating the thread for 159 * CPU 0, so in practice we ignore values <= 1. This must be set at boot. 160 * We will create at most one thread per CPU. 161 */ 162static int netisr_maxthreads = -1; /* Max number of threads. */ 163TUNABLE_INT("net.isr.maxthreads", &netisr_maxthreads); 164SYSCTL_INT(_net_isr, OID_AUTO, maxthreads, CTLFLAG_RDTUN, 165 &netisr_maxthreads, 0, 166 "Use at most this many CPUs for netisr processing"); 167 168static int netisr_bindthreads = 0; /* Bind threads to CPUs. */ 169TUNABLE_INT("net.isr.bindthreads", &netisr_bindthreads); 170SYSCTL_INT(_net_isr, OID_AUTO, bindthreads, CTLFLAG_RDTUN, 171 &netisr_bindthreads, 0, "Bind netisr threads to CPUs."); 172 173/* 174 * Limit per-workstream mbuf queue limits s to at most net.isr.maxqlimit, 175 * both for initial configuration and later modification using 176 * netisr_setqlimit(). 177 */ 178#define NETISR_DEFAULT_MAXQLIMIT 10240 179static u_int netisr_maxqlimit = NETISR_DEFAULT_MAXQLIMIT; 180TUNABLE_INT("net.isr.maxqlimit", &netisr_maxqlimit); 181SYSCTL_UINT(_net_isr, OID_AUTO, maxqlimit, CTLFLAG_RDTUN, 182 &netisr_maxqlimit, 0, 183 "Maximum netisr per-protocol, per-CPU queue depth."); 184 185/* 186 * The default per-workstream mbuf queue limit for protocols that don't 187 * initialize the nh_qlimit field of their struct netisr_handler. If this is 188 * set above netisr_maxqlimit, we truncate it to the maximum during boot. 189 */ 190#define NETISR_DEFAULT_DEFAULTQLIMIT 256 191static u_int netisr_defaultqlimit = NETISR_DEFAULT_DEFAULTQLIMIT; 192TUNABLE_INT("net.isr.defaultqlimit", &netisr_defaultqlimit); 193SYSCTL_UINT(_net_isr, OID_AUTO, defaultqlimit, CTLFLAG_RDTUN, 194 &netisr_defaultqlimit, 0, 195 "Default netisr per-protocol, per-CPU queue limit if not set by protocol"); 196 197/* 198 * Store and export the compile-time constant NETISR_MAXPROT limit on the 199 * number of protocols that can register with netisr at a time. This is 200 * required for crashdump analysis, as it sizes netisr_proto[]. 201 */ 202static u_int netisr_maxprot = NETISR_MAXPROT; 203SYSCTL_UINT(_net_isr, OID_AUTO, maxprot, CTLFLAG_RD, 204 &netisr_maxprot, 0, 205 "Compile-time limit on the number of protocols supported by netisr."); 206 207/* 208 * The netisr_proto array describes all registered protocols, indexed by 209 * protocol number. See netisr_internal.h for more details. 210 */ 211static struct netisr_proto netisr_proto[NETISR_MAXPROT]; 212 213/* 214 * Per-CPU workstream data. See netisr_internal.h for more details. 215 */ 216DPCPU_DEFINE(struct netisr_workstream, nws); 217 218/* 219 * Map contiguous values between 0 and nws_count into CPU IDs appropriate for 220 * accessing workstreams. This allows constructions of the form 221 * DPCPU_ID_GET(nws_array[arbitraryvalue % nws_count], nws). 222 */ 223static u_int nws_array[MAXCPU]; 224 225/* 226 * Number of registered workstreams. Will be at most the number of running 227 * CPUs once fully started. 228 */ 229static u_int nws_count; 230SYSCTL_UINT(_net_isr, OID_AUTO, numthreads, CTLFLAG_RD, 231 &nws_count, 0, "Number of extant netisr threads."); 232 233/* 234 * Synchronization for each workstream: a mutex protects all mutable fields 235 * in each stream, including per-protocol state (mbuf queues). The SWI is 236 * woken up if asynchronous dispatch is required. 237 */ 238#define NWS_LOCK(s) mtx_lock(&(s)->nws_mtx) 239#define NWS_LOCK_ASSERT(s) mtx_assert(&(s)->nws_mtx, MA_OWNED) 240#define NWS_UNLOCK(s) mtx_unlock(&(s)->nws_mtx) 241#define NWS_SIGNAL(s) swi_sched((s)->nws_swi_cookie, 0) 242 243/* 244 * Utility routines for protocols that implement their own mapping of flows 245 * to CPUs. 246 */ 247u_int 248netisr_get_cpucount(void) 249{ 250 251 return (nws_count); 252} 253 254u_int 255netisr_get_cpuid(u_int cpunumber) 256{ 257 258 KASSERT(cpunumber < nws_count, ("%s: %u > %u", __func__, cpunumber, 259 nws_count)); 260 261 return (nws_array[cpunumber]); 262} 263 264/* 265 * The default implementation of flow -> CPU ID mapping. 266 * 267 * Non-static so that protocols can use it to map their own work to specific 268 * CPUs in a manner consistent to netisr for affinity purposes. 269 */ 270u_int 271netisr_default_flow2cpu(u_int flowid) 272{ 273 274 return (nws_array[flowid % nws_count]); 275} 276 277/* 278 * Dispatch tunable and sysctl configuration. 279 */ 280struct netisr_dispatch_table_entry { 281 u_int ndte_policy; 282 const char *ndte_policy_str; 283}; 284static const struct netisr_dispatch_table_entry netisr_dispatch_table[] = { 285 { NETISR_DISPATCH_DEFAULT, "default" }, 286 { NETISR_DISPATCH_DEFERRED, "deferred" }, 287 { NETISR_DISPATCH_HYBRID, "hybrid" }, 288 { NETISR_DISPATCH_DIRECT, "direct" }, 289}; 290static const u_int netisr_dispatch_table_len = 291 (sizeof(netisr_dispatch_table) / sizeof(netisr_dispatch_table[0])); 292 293static void 294netisr_dispatch_policy_to_str(u_int dispatch_policy, char *buffer, 295 u_int buflen) 296{ 297 const struct netisr_dispatch_table_entry *ndtep; 298 const char *str; 299 u_int i; 300 301 str = "unknown"; 302 for (i = 0; i < netisr_dispatch_table_len; i++) { 303 ndtep = &netisr_dispatch_table[i]; 304 if (ndtep->ndte_policy == dispatch_policy) { 305 str = ndtep->ndte_policy_str; 306 break; 307 } 308 } 309 snprintf(buffer, buflen, "%s", str); 310} 311 312static int 313netisr_dispatch_policy_from_str(const char *str, u_int *dispatch_policyp) 314{ 315 const struct netisr_dispatch_table_entry *ndtep; 316 u_int i; 317 318 for (i = 0; i < netisr_dispatch_table_len; i++) { 319 ndtep = &netisr_dispatch_table[i]; 320 if (strcmp(ndtep->ndte_policy_str, str) == 0) { 321 *dispatch_policyp = ndtep->ndte_policy; 322 return (0); 323 } 324 } 325 return (EINVAL); 326} 327 328static int 329sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS) 330{ 331 char tmp[NETISR_DISPATCH_POLICY_MAXSTR]; 332 u_int dispatch_policy; 333 int error; 334 335 netisr_dispatch_policy_to_str(netisr_dispatch_policy, tmp, 336 sizeof(tmp)); 337 error = sysctl_handle_string(oidp, tmp, sizeof(tmp), req); 338 if (error == 0 && req->newptr != NULL) { 339 error = netisr_dispatch_policy_from_str(tmp, 340 &dispatch_policy); 341 if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT) 342 error = EINVAL; 343 if (error == 0) 344 netisr_dispatch_policy = dispatch_policy; 345 } 346 return (error); 347} 348 349/* 350 * Register a new netisr handler, which requires initializing per-protocol 351 * fields for each workstream. All netisr work is briefly suspended while 352 * the protocol is installed. 353 */ 354void 355netisr_register(const struct netisr_handler *nhp) 356{ 357 struct netisr_work *npwp; 358 const char *name; 359 u_int i, proto; 360 361 proto = nhp->nh_proto; 362 name = nhp->nh_name; 363 364 /* 365 * Test that the requested registration is valid. 366 */ 367 KASSERT(nhp->nh_name != NULL, 368 ("%s: nh_name NULL for %u", __func__, proto)); 369 KASSERT(nhp->nh_handler != NULL, 370 ("%s: nh_handler NULL for %s", __func__, name)); 371 KASSERT(nhp->nh_policy == NETISR_POLICY_SOURCE || 372 nhp->nh_policy == NETISR_POLICY_FLOW || 373 nhp->nh_policy == NETISR_POLICY_CPU, 374 ("%s: unsupported nh_policy %u for %s", __func__, 375 nhp->nh_policy, name)); 376 KASSERT(nhp->nh_policy == NETISR_POLICY_FLOW || 377 nhp->nh_m2flow == NULL, 378 ("%s: nh_policy != FLOW but m2flow defined for %s", __func__, 379 name)); 380 KASSERT(nhp->nh_policy == NETISR_POLICY_CPU || nhp->nh_m2cpuid == NULL, 381 ("%s: nh_policy != CPU but m2cpuid defined for %s", __func__, 382 name)); 383 KASSERT(nhp->nh_policy != NETISR_POLICY_CPU || nhp->nh_m2cpuid != NULL, 384 ("%s: nh_policy == CPU but m2cpuid not defined for %s", __func__, 385 name)); 386 KASSERT(nhp->nh_dispatch == NETISR_DISPATCH_DEFAULT || 387 nhp->nh_dispatch == NETISR_DISPATCH_DEFERRED || 388 nhp->nh_dispatch == NETISR_DISPATCH_HYBRID || 389 nhp->nh_dispatch == NETISR_DISPATCH_DIRECT, 390 ("%s: invalid nh_dispatch (%u)", __func__, nhp->nh_dispatch)); 391 392 KASSERT(proto < NETISR_MAXPROT, 393 ("%s(%u, %s): protocol too big", __func__, proto, name)); 394 395 /* 396 * Test that no existing registration exists for this protocol. 397 */ 398 NETISR_WLOCK(); 399 KASSERT(netisr_proto[proto].np_name == NULL, 400 ("%s(%u, %s): name present", __func__, proto, name)); 401 KASSERT(netisr_proto[proto].np_handler == NULL, 402 ("%s(%u, %s): handler present", __func__, proto, name)); 403 404 netisr_proto[proto].np_name = name; 405 netisr_proto[proto].np_handler = nhp->nh_handler; 406 netisr_proto[proto].np_m2flow = nhp->nh_m2flow; 407 netisr_proto[proto].np_m2cpuid = nhp->nh_m2cpuid; 408 netisr_proto[proto].np_drainedcpu = nhp->nh_drainedcpu; 409 if (nhp->nh_qlimit == 0) 410 netisr_proto[proto].np_qlimit = netisr_defaultqlimit; 411 else if (nhp->nh_qlimit > netisr_maxqlimit) { 412 printf("%s: %s requested queue limit %u capped to " 413 "net.isr.maxqlimit %u\n", __func__, name, nhp->nh_qlimit, 414 netisr_maxqlimit); 415 netisr_proto[proto].np_qlimit = netisr_maxqlimit; 416 } else 417 netisr_proto[proto].np_qlimit = nhp->nh_qlimit; 418 netisr_proto[proto].np_policy = nhp->nh_policy; 419 netisr_proto[proto].np_dispatch = nhp->nh_dispatch; 420 CPU_FOREACH(i) { 421 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 422 bzero(npwp, sizeof(*npwp)); 423 npwp->nw_qlimit = netisr_proto[proto].np_qlimit; 424 } 425 NETISR_WUNLOCK(); 426} 427 428/* 429 * Clear drop counters across all workstreams for a protocol. 430 */ 431void 432netisr_clearqdrops(const struct netisr_handler *nhp) 433{ 434 struct netisr_work *npwp; 435#ifdef INVARIANTS 436 const char *name; 437#endif 438 u_int i, proto; 439 440 proto = nhp->nh_proto; 441#ifdef INVARIANTS 442 name = nhp->nh_name; 443#endif 444 KASSERT(proto < NETISR_MAXPROT, 445 ("%s(%u): protocol too big for %s", __func__, proto, name)); 446 447 NETISR_WLOCK(); 448 KASSERT(netisr_proto[proto].np_handler != NULL, 449 ("%s(%u): protocol not registered for %s", __func__, proto, 450 name)); 451 452 CPU_FOREACH(i) { 453 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 454 npwp->nw_qdrops = 0; 455 } 456 NETISR_WUNLOCK(); 457} 458 459/* 460 * Query current drop counters across all workstreams for a protocol. 461 */ 462void 463netisr_getqdrops(const struct netisr_handler *nhp, u_int64_t *qdropp) 464{ 465 struct netisr_work *npwp; 466 struct rm_priotracker tracker; 467#ifdef INVARIANTS 468 const char *name; 469#endif 470 u_int i, proto; 471 472 *qdropp = 0; 473 proto = nhp->nh_proto; 474#ifdef INVARIANTS 475 name = nhp->nh_name; 476#endif 477 KASSERT(proto < NETISR_MAXPROT, 478 ("%s(%u): protocol too big for %s", __func__, proto, name)); 479 480 NETISR_RLOCK(&tracker); 481 KASSERT(netisr_proto[proto].np_handler != NULL, 482 ("%s(%u): protocol not registered for %s", __func__, proto, 483 name)); 484 485 CPU_FOREACH(i) { 486 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 487 *qdropp += npwp->nw_qdrops; 488 } 489 NETISR_RUNLOCK(&tracker); 490} 491 492/* 493 * Query current per-workstream queue limit for a protocol. 494 */ 495void 496netisr_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp) 497{ 498 struct rm_priotracker tracker; 499#ifdef INVARIANTS 500 const char *name; 501#endif 502 u_int proto; 503 504 proto = nhp->nh_proto; 505#ifdef INVARIANTS 506 name = nhp->nh_name; 507#endif 508 KASSERT(proto < NETISR_MAXPROT, 509 ("%s(%u): protocol too big for %s", __func__, proto, name)); 510 511 NETISR_RLOCK(&tracker); 512 KASSERT(netisr_proto[proto].np_handler != NULL, 513 ("%s(%u): protocol not registered for %s", __func__, proto, 514 name)); 515 *qlimitp = netisr_proto[proto].np_qlimit; 516 NETISR_RUNLOCK(&tracker); 517} 518 519/* 520 * Update the queue limit across per-workstream queues for a protocol. We 521 * simply change the limits, and don't drain overflowed packets as they will 522 * (hopefully) take care of themselves shortly. 523 */ 524int 525netisr_setqlimit(const struct netisr_handler *nhp, u_int qlimit) 526{ 527 struct netisr_work *npwp; 528#ifdef INVARIANTS 529 const char *name; 530#endif 531 u_int i, proto; 532 533 if (qlimit > netisr_maxqlimit) 534 return (EINVAL); 535 536 proto = nhp->nh_proto; 537#ifdef INVARIANTS 538 name = nhp->nh_name; 539#endif 540 KASSERT(proto < NETISR_MAXPROT, 541 ("%s(%u): protocol too big for %s", __func__, proto, name)); 542 543 NETISR_WLOCK(); 544 KASSERT(netisr_proto[proto].np_handler != NULL, 545 ("%s(%u): protocol not registered for %s", __func__, proto, 546 name)); 547 548 netisr_proto[proto].np_qlimit = qlimit; 549 CPU_FOREACH(i) { 550 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 551 npwp->nw_qlimit = qlimit; 552 } 553 NETISR_WUNLOCK(); 554 return (0); 555} 556 557/* 558 * Drain all packets currently held in a particular protocol work queue. 559 */ 560static void 561netisr_drain_proto(struct netisr_work *npwp) 562{ 563 struct mbuf *m; 564 565 /* 566 * We would assert the lock on the workstream but it's not passed in. 567 */ 568 while ((m = npwp->nw_head) != NULL) { 569 npwp->nw_head = m->m_nextpkt; 570 m->m_nextpkt = NULL; 571 if (npwp->nw_head == NULL) 572 npwp->nw_tail = NULL; 573 npwp->nw_len--; 574 m_freem(m); 575 } 576 KASSERT(npwp->nw_tail == NULL, ("%s: tail", __func__)); 577 KASSERT(npwp->nw_len == 0, ("%s: len", __func__)); 578} 579 580/* 581 * Remove the registration of a network protocol, which requires clearing 582 * per-protocol fields across all workstreams, including freeing all mbufs in 583 * the queues at time of unregister. All work in netisr is briefly suspended 584 * while this takes place. 585 */ 586void 587netisr_unregister(const struct netisr_handler *nhp) 588{ 589 struct netisr_work *npwp; 590#ifdef INVARIANTS 591 const char *name; 592#endif 593 u_int i, proto; 594 595 proto = nhp->nh_proto; 596#ifdef INVARIANTS 597 name = nhp->nh_name; 598#endif 599 KASSERT(proto < NETISR_MAXPROT, 600 ("%s(%u): protocol too big for %s", __func__, proto, name)); 601 602 NETISR_WLOCK(); 603 KASSERT(netisr_proto[proto].np_handler != NULL, 604 ("%s(%u): protocol not registered for %s", __func__, proto, 605 name)); 606 607 netisr_proto[proto].np_name = NULL; 608 netisr_proto[proto].np_handler = NULL; 609 netisr_proto[proto].np_m2flow = NULL; 610 netisr_proto[proto].np_m2cpuid = NULL; 611 netisr_proto[proto].np_qlimit = 0; 612 netisr_proto[proto].np_policy = 0; 613 CPU_FOREACH(i) { 614 npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 615 netisr_drain_proto(npwp); 616 bzero(npwp, sizeof(*npwp)); 617 } 618 NETISR_WUNLOCK(); 619} 620 621/* 622 * Compose the global and per-protocol policies on dispatch, and return the 623 * dispatch policy to use. 624 */ 625static u_int 626netisr_get_dispatch(struct netisr_proto *npp) 627{ 628 629 /* 630 * Protocol-specific configuration overrides the global default. 631 */ 632 if (npp->np_dispatch != NETISR_DISPATCH_DEFAULT) 633 return (npp->np_dispatch); 634 return (netisr_dispatch_policy); 635} 636 637/* 638 * Look up the workstream given a packet and source identifier. Do this by 639 * checking the protocol's policy, and optionally call out to the protocol 640 * for assistance if required. 641 */ 642static struct mbuf * 643netisr_select_cpuid(struct netisr_proto *npp, u_int dispatch_policy, 644 uintptr_t source, struct mbuf *m, u_int *cpuidp) 645{ 646 struct ifnet *ifp; 647 u_int policy; 648 649 NETISR_LOCK_ASSERT(); 650 651 /* 652 * In the event we have only one worker, shortcut and deliver to it 653 * without further ado. 654 */ 655 if (nws_count == 1) { 656 *cpuidp = nws_array[0]; 657 return (m); 658 } 659 660 /* 661 * What happens next depends on the policy selected by the protocol. 662 * If we want to support per-interface policies, we should do that 663 * here first. 664 */ 665 policy = npp->np_policy; 666 if (policy == NETISR_POLICY_CPU) { 667 m = npp->np_m2cpuid(m, source, cpuidp); 668 if (m == NULL) 669 return (NULL); 670 671 /* 672 * It's possible for a protocol not to have a good idea about 673 * where to process a packet, in which case we fall back on 674 * the netisr code to decide. In the hybrid case, return the 675 * current CPU ID, which will force an immediate direct 676 * dispatch. In the queued case, fall back on the SOURCE 677 * policy. 678 */ 679 if (*cpuidp != NETISR_CPUID_NONE) 680 return (m); 681 if (dispatch_policy == NETISR_DISPATCH_HYBRID) { 682 *cpuidp = curcpu; 683 return (m); 684 } 685 policy = NETISR_POLICY_SOURCE; 686 } 687 688 if (policy == NETISR_POLICY_FLOW) { 689 if (!(m->m_flags & M_FLOWID) && npp->np_m2flow != NULL) { 690 m = npp->np_m2flow(m, source); 691 if (m == NULL) 692 return (NULL); 693 } 694 if (m->m_flags & M_FLOWID) { 695 *cpuidp = 696 netisr_default_flow2cpu(m->m_pkthdr.flowid); 697 return (m); 698 } 699 policy = NETISR_POLICY_SOURCE; 700 } 701 702 KASSERT(policy == NETISR_POLICY_SOURCE, 703 ("%s: invalid policy %u for %s", __func__, npp->np_policy, 704 npp->np_name)); 705 706 ifp = m->m_pkthdr.rcvif; 707 if (ifp != NULL) 708 *cpuidp = nws_array[(ifp->if_index + source) % nws_count]; 709 else 710 *cpuidp = nws_array[source % nws_count]; 711 return (m); 712} 713 714/* 715 * Process packets associated with a workstream and protocol. For reasons of 716 * fairness, we process up to one complete netisr queue at a time, moving the 717 * queue to a stack-local queue for processing, but do not loop refreshing 718 * from the global queue. The caller is responsible for deciding whether to 719 * loop, and for setting the NWS_RUNNING flag. The passed workstream will be 720 * locked on entry and relocked before return, but will be released while 721 * processing. The number of packets processed is returned. 722 */ 723static u_int 724netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) 725{ 726 struct netisr_work local_npw, *npwp; 727 u_int handled; 728 struct mbuf *m; 729 730 NETISR_LOCK_ASSERT(); 731 NWS_LOCK_ASSERT(nwsp); 732 733 KASSERT(nwsp->nws_flags & NWS_RUNNING, 734 ("%s(%u): not running", __func__, proto)); 735 KASSERT(proto >= 0 && proto < NETISR_MAXPROT, 736 ("%s(%u): invalid proto\n", __func__, proto)); 737 738 npwp = &nwsp->nws_work[proto]; 739 if (npwp->nw_len == 0) 740 return (0); 741 742 /* 743 * Move the global work queue to a thread-local work queue. 744 * 745 * Notice that this means the effective maximum length of the queue 746 * is actually twice that of the maximum queue length specified in 747 * the protocol registration call. 748 */ 749 handled = npwp->nw_len; 750 local_npw = *npwp; 751 npwp->nw_head = NULL; 752 npwp->nw_tail = NULL; 753 npwp->nw_len = 0; 754 nwsp->nws_pendingbits &= ~(1 << proto); 755 NWS_UNLOCK(nwsp); 756 while ((m = local_npw.nw_head) != NULL) { 757 local_npw.nw_head = m->m_nextpkt; 758 m->m_nextpkt = NULL; 759 if (local_npw.nw_head == NULL) 760 local_npw.nw_tail = NULL; 761 local_npw.nw_len--; 762 VNET_ASSERT(m->m_pkthdr.rcvif != NULL, 763 ("%s:%d rcvif == NULL: m=%p", __func__, __LINE__, m)); 764 CURVNET_SET(m->m_pkthdr.rcvif->if_vnet); 765 netisr_proto[proto].np_handler(m); 766 CURVNET_RESTORE(); 767 } 768 KASSERT(local_npw.nw_len == 0, 769 ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); 770 if (netisr_proto[proto].np_drainedcpu) 771 netisr_proto[proto].np_drainedcpu(nwsp->nws_cpu); 772 NWS_LOCK(nwsp); 773 npwp->nw_handled += handled; 774 return (handled); 775} 776 777/* 778 * SWI handler for netisr -- processes packets in a set of workstreams that 779 * it owns, woken up by calls to NWS_SIGNAL(). If this workstream is already 780 * being direct dispatched, go back to sleep and wait for the dispatching 781 * thread to wake us up again. 782 */ 783static void 784swi_net(void *arg) 785{ 786#ifdef NETISR_LOCKING 787 struct rm_priotracker tracker; 788#endif 789 struct netisr_workstream *nwsp; 790 u_int bits, prot; 791 792 nwsp = arg; 793 794#ifdef DEVICE_POLLING 795 KASSERT(nws_count == 1, 796 ("%s: device_polling but nws_count != 1", __func__)); 797 netisr_poll(); 798#endif 799#ifdef NETISR_LOCKING 800 NETISR_RLOCK(&tracker); 801#endif 802 NWS_LOCK(nwsp); 803 KASSERT(!(nwsp->nws_flags & NWS_RUNNING), ("swi_net: running")); 804 if (nwsp->nws_flags & NWS_DISPATCHING) 805 goto out; 806 nwsp->nws_flags |= NWS_RUNNING; 807 nwsp->nws_flags &= ~NWS_SCHEDULED; 808 while ((bits = nwsp->nws_pendingbits) != 0) { 809 while ((prot = ffs(bits)) != 0) { 810 prot--; 811 bits &= ~(1 << prot); 812 (void)netisr_process_workstream_proto(nwsp, prot); 813 } 814 } 815 nwsp->nws_flags &= ~NWS_RUNNING; 816out: 817 NWS_UNLOCK(nwsp); 818#ifdef NETISR_LOCKING 819 NETISR_RUNLOCK(&tracker); 820#endif 821#ifdef DEVICE_POLLING 822 netisr_pollmore(); 823#endif 824} 825 826static int 827netisr_queue_workstream(struct netisr_workstream *nwsp, u_int proto, 828 struct netisr_work *npwp, struct mbuf *m, int *dosignalp) 829{ 830 831 NWS_LOCK_ASSERT(nwsp); 832 833 *dosignalp = 0; 834 if (npwp->nw_len < npwp->nw_qlimit) { 835 m->m_nextpkt = NULL; 836 if (npwp->nw_head == NULL) { 837 npwp->nw_head = m; 838 npwp->nw_tail = m; 839 } else { 840 npwp->nw_tail->m_nextpkt = m; 841 npwp->nw_tail = m; 842 } 843 npwp->nw_len++; 844 if (npwp->nw_len > npwp->nw_watermark) 845 npwp->nw_watermark = npwp->nw_len; 846 847 /* 848 * We must set the bit regardless of NWS_RUNNING, so that 849 * swi_net() keeps calling netisr_process_workstream_proto(). 850 */ 851 nwsp->nws_pendingbits |= (1 << proto); 852 if (!(nwsp->nws_flags & 853 (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED))) { 854 nwsp->nws_flags |= NWS_SCHEDULED; 855 *dosignalp = 1; /* Defer until unlocked. */ 856 } 857 npwp->nw_queued++; 858 return (0); 859 } else { 860 m_freem(m); 861 npwp->nw_qdrops++; 862 return (ENOBUFS); 863 } 864} 865 866static int 867netisr_queue_internal(u_int proto, struct mbuf *m, u_int cpuid) 868{ 869 struct netisr_workstream *nwsp; 870 struct netisr_work *npwp; 871 int dosignal, error; 872 873#ifdef NETISR_LOCKING 874 NETISR_LOCK_ASSERT(); 875#endif 876 KASSERT(cpuid <= mp_maxid, ("%s: cpuid too big (%u, %u)", __func__, 877 cpuid, mp_maxid)); 878 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 879 880 dosignal = 0; 881 error = 0; 882 nwsp = DPCPU_ID_PTR(cpuid, nws); 883 npwp = &nwsp->nws_work[proto]; 884 NWS_LOCK(nwsp); 885 error = netisr_queue_workstream(nwsp, proto, npwp, m, &dosignal); 886 NWS_UNLOCK(nwsp); 887 if (dosignal) 888 NWS_SIGNAL(nwsp); 889 return (error); 890} 891 892int 893netisr_queue_src(u_int proto, uintptr_t source, struct mbuf *m) 894{ 895#ifdef NETISR_LOCKING 896 struct rm_priotracker tracker; 897#endif 898 u_int cpuid; 899 int error; 900 901 KASSERT(proto < NETISR_MAXPROT, 902 ("%s: invalid proto %u", __func__, proto)); 903 904#ifdef NETISR_LOCKING 905 NETISR_RLOCK(&tracker); 906#endif 907 KASSERT(netisr_proto[proto].np_handler != NULL, 908 ("%s: invalid proto %u", __func__, proto)); 909 910 m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_DEFERRED, 911 source, m, &cpuid); 912 if (m != NULL) { 913 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, 914 cpuid)); 915 error = netisr_queue_internal(proto, m, cpuid); 916 } else 917 error = ENOBUFS; 918#ifdef NETISR_LOCKING 919 NETISR_RUNLOCK(&tracker); 920#endif 921 return (error); 922} 923 924int 925netisr_queue(u_int proto, struct mbuf *m) 926{ 927 928 return (netisr_queue_src(proto, 0, m)); 929} 930 931/* 932 * Dispatch a packet for netisr processing; direct dispatch is permitted by 933 * calling context. 934 */ 935int 936netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) 937{ 938#ifdef NETISR_LOCKING 939 struct rm_priotracker tracker; 940#endif 941 struct netisr_workstream *nwsp; 942 struct netisr_proto *npp; 943 struct netisr_work *npwp; 944 int dosignal, error; 945 u_int cpuid, dispatch_policy; 946 947 KASSERT(proto < NETISR_MAXPROT, 948 ("%s: invalid proto %u", __func__, proto)); 949#ifdef NETISR_LOCKING 950 NETISR_RLOCK(&tracker); 951#endif 952 npp = &netisr_proto[proto]; 953 KASSERT(npp->np_handler != NULL, ("%s: invalid proto %u", __func__, 954 proto)); 955 956 dispatch_policy = netisr_get_dispatch(npp); 957 if (dispatch_policy == NETISR_DISPATCH_DEFERRED) 958 return (netisr_queue_src(proto, source, m)); 959 960 /* 961 * If direct dispatch is forced, then unconditionally dispatch 962 * without a formal CPU selection. Borrow the current CPU's stats, 963 * even if there's no worker on it. In this case we don't update 964 * nws_flags because all netisr processing will be source ordered due 965 * to always being forced to directly dispatch. 966 */ 967 if (dispatch_policy == NETISR_DISPATCH_DIRECT) { 968 nwsp = DPCPU_PTR(nws); 969 npwp = &nwsp->nws_work[proto]; 970 npwp->nw_dispatched++; 971 npwp->nw_handled++; 972 netisr_proto[proto].np_handler(m); 973 error = 0; 974 goto out_unlock; 975 } 976 977 KASSERT(dispatch_policy == NETISR_DISPATCH_HYBRID, 978 ("%s: unknown dispatch policy (%u)", __func__, dispatch_policy)); 979 980 /* 981 * Otherwise, we execute in a hybrid mode where we will try to direct 982 * dispatch if we're on the right CPU and the netisr worker isn't 983 * already running. 984 */ 985 sched_pin(); 986 m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_HYBRID, 987 source, m, &cpuid); 988 if (m == NULL) { 989 error = ENOBUFS; 990 goto out_unpin; 991 } 992 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 993 if (cpuid != curcpu) 994 goto queue_fallback; 995 nwsp = DPCPU_PTR(nws); 996 npwp = &nwsp->nws_work[proto]; 997 998 /*- 999 * We are willing to direct dispatch only if three conditions hold: 1000 * 1001 * (1) The netisr worker isn't already running, 1002 * (2) Another thread isn't already directly dispatching, and 1003 * (3) The netisr hasn't already been woken up. 1004 */ 1005 NWS_LOCK(nwsp); 1006 if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) { 1007 error = netisr_queue_workstream(nwsp, proto, npwp, m, 1008 &dosignal); 1009 NWS_UNLOCK(nwsp); 1010 if (dosignal) 1011 NWS_SIGNAL(nwsp); 1012 goto out_unpin; 1013 } 1014 1015 /* 1016 * The current thread is now effectively the netisr worker, so set 1017 * the dispatching flag to prevent concurrent processing of the 1018 * stream from another thread (even the netisr worker), which could 1019 * otherwise lead to effective misordering of the stream. 1020 */ 1021 nwsp->nws_flags |= NWS_DISPATCHING; 1022 NWS_UNLOCK(nwsp); 1023 netisr_proto[proto].np_handler(m); 1024 NWS_LOCK(nwsp); 1025 nwsp->nws_flags &= ~NWS_DISPATCHING; 1026 npwp->nw_handled++; 1027 npwp->nw_hybrid_dispatched++; 1028 1029 /* 1030 * If other work was enqueued by another thread while we were direct 1031 * dispatching, we need to signal the netisr worker to do that work. 1032 * In the future, we might want to do some of that work in the 1033 * current thread, rather than trigger further context switches. If 1034 * so, we'll want to establish a reasonable bound on the work done in 1035 * the "borrowed" context. 1036 */ 1037 if (nwsp->nws_pendingbits != 0) { 1038 nwsp->nws_flags |= NWS_SCHEDULED; 1039 dosignal = 1; 1040 } else 1041 dosignal = 0; 1042 NWS_UNLOCK(nwsp); 1043 if (dosignal) 1044 NWS_SIGNAL(nwsp); 1045 error = 0; 1046 goto out_unpin; 1047 1048queue_fallback: 1049 error = netisr_queue_internal(proto, m, cpuid); 1050out_unpin: 1051 sched_unpin(); 1052out_unlock: 1053#ifdef NETISR_LOCKING 1054 NETISR_RUNLOCK(&tracker); 1055#endif 1056 return (error); 1057} 1058 1059int 1060netisr_dispatch(u_int proto, struct mbuf *m) 1061{ 1062 1063 return (netisr_dispatch_src(proto, 0, m)); 1064} 1065 1066#ifdef DEVICE_POLLING 1067/* 1068 * Kernel polling borrows a netisr thread to run interface polling in; this 1069 * function allows kernel polling to request that the netisr thread be 1070 * scheduled even if no packets are pending for protocols. 1071 */ 1072void 1073netisr_sched_poll(void) 1074{ 1075 struct netisr_workstream *nwsp; 1076 1077 nwsp = DPCPU_ID_PTR(nws_array[0], nws); 1078 NWS_SIGNAL(nwsp); 1079} 1080#endif 1081 1082static void 1083netisr_start_swi(u_int cpuid, struct pcpu *pc) 1084{ 1085 char swiname[12]; 1086 struct netisr_workstream *nwsp; 1087 int error; 1088 1089 KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 1090 1091 nwsp = DPCPU_ID_PTR(cpuid, nws); 1092 mtx_init(&nwsp->nws_mtx, "netisr_mtx", NULL, MTX_DEF); 1093 nwsp->nws_cpu = cpuid; 1094 snprintf(swiname, sizeof(swiname), "netisr %u", cpuid); 1095 error = swi_add(&nwsp->nws_intr_event, swiname, swi_net, nwsp, 1096 SWI_NET, INTR_MPSAFE, &nwsp->nws_swi_cookie); 1097 if (error) 1098 panic("%s: swi_add %d", __func__, error); 1099 pc->pc_netisr = nwsp->nws_intr_event; 1100 if (netisr_bindthreads) { 1101 error = intr_event_bind(nwsp->nws_intr_event, cpuid); 1102 if (error != 0) 1103 printf("%s: cpu %u: intr_event_bind: %d", __func__, 1104 cpuid, error); 1105 } 1106 NETISR_WLOCK(); 1107 nws_array[nws_count] = nwsp->nws_cpu; 1108 nws_count++; 1109 NETISR_WUNLOCK(); 1110} 1111 1112/* 1113 * Initialize the netisr subsystem. We rely on BSS and static initialization 1114 * of most fields in global data structures. 1115 * 1116 * Start a worker thread for the boot CPU so that we can support network 1117 * traffic immediately in case the network stack is used before additional 1118 * CPUs are started (for example, diskless boot). 1119 */ 1120static void 1121netisr_init(void *arg) 1122{ 1123 char tmp[NETISR_DISPATCH_POLICY_MAXSTR]; 1124 u_int dispatch_policy; 1125 int error; 1126 1127 KASSERT(curcpu == 0, ("%s: not on CPU 0", __func__)); 1128 1129 NETISR_LOCK_INIT(); 1130 if (netisr_maxthreads < 1) 1131 netisr_maxthreads = 1; 1132 if (netisr_maxthreads > mp_ncpus) { 1133 printf("netisr_init: forcing maxthreads from %d to %d\n", 1134 netisr_maxthreads, mp_ncpus); 1135 netisr_maxthreads = mp_ncpus; 1136 } 1137 if (netisr_defaultqlimit > netisr_maxqlimit) { 1138 printf("netisr_init: forcing defaultqlimit from %d to %d\n", 1139 netisr_defaultqlimit, netisr_maxqlimit); 1140 netisr_defaultqlimit = netisr_maxqlimit; 1141 } 1142#ifdef DEVICE_POLLING 1143 /* 1144 * The device polling code is not yet aware of how to deal with 1145 * multiple netisr threads, so for the time being compiling in device 1146 * polling disables parallel netisr workers. 1147 */ 1148 if (netisr_maxthreads != 1 || netisr_bindthreads != 0) { 1149 printf("netisr_init: forcing maxthreads to 1 and " 1150 "bindthreads to 0 for device polling\n"); 1151 netisr_maxthreads = 1; 1152 netisr_bindthreads = 0; 1153 } 1154#endif 1155 1156 if (TUNABLE_STR_FETCH("net.isr.dispatch", tmp, sizeof(tmp))) { 1157 error = netisr_dispatch_policy_from_str(tmp, 1158 &dispatch_policy); 1159 if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT) 1160 error = EINVAL; 1161 if (error == 0) 1162 netisr_dispatch_policy = dispatch_policy; 1163 else 1164 printf( 1165 "%s: invalid dispatch policy %s, using default\n", 1166 __func__, tmp); 1167 } 1168 1169 netisr_start_swi(curcpu, pcpu_find(curcpu)); 1170} 1171SYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL); 1172 1173/* 1174 * Start worker threads for additional CPUs. No attempt to gracefully handle 1175 * work reassignment, we don't yet support dynamic reconfiguration. 1176 */ 1177static void 1178netisr_start(void *arg) 1179{ 1180 struct pcpu *pc; 1181 1182 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 1183 if (nws_count >= netisr_maxthreads) 1184 break; 1185 /* XXXRW: Is skipping absent CPUs still required here? */ 1186 if (CPU_ABSENT(pc->pc_cpuid)) 1187 continue; 1188 /* Worker will already be present for boot CPU. */ 1189 if (pc->pc_netisr != NULL) 1190 continue; 1191 netisr_start_swi(pc->pc_cpuid, pc); 1192 } 1193} 1194SYSINIT(netisr_start, SI_SUB_SMP, SI_ORDER_MIDDLE, netisr_start, NULL); 1195 1196/* 1197 * Sysctl monitoring for netisr: query a list of registered protocols. 1198 */ 1199static int 1200sysctl_netisr_proto(SYSCTL_HANDLER_ARGS) 1201{ 1202 struct rm_priotracker tracker; 1203 struct sysctl_netisr_proto *snpp, *snp_array; 1204 struct netisr_proto *npp; 1205 u_int counter, proto; 1206 int error; 1207 1208 if (req->newptr != NULL) 1209 return (EINVAL); 1210 snp_array = malloc(sizeof(*snp_array) * NETISR_MAXPROT, M_TEMP, 1211 M_ZERO | M_WAITOK); 1212 counter = 0; 1213 NETISR_RLOCK(&tracker); 1214 for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1215 npp = &netisr_proto[proto]; 1216 if (npp->np_name == NULL) 1217 continue; 1218 snpp = &snp_array[counter]; 1219 snpp->snp_version = sizeof(*snpp); 1220 strlcpy(snpp->snp_name, npp->np_name, NETISR_NAMEMAXLEN); 1221 snpp->snp_proto = proto; 1222 snpp->snp_qlimit = npp->np_qlimit; 1223 snpp->snp_policy = npp->np_policy; 1224 snpp->snp_dispatch = npp->np_dispatch; 1225 if (npp->np_m2flow != NULL) 1226 snpp->snp_flags |= NETISR_SNP_FLAGS_M2FLOW; 1227 if (npp->np_m2cpuid != NULL) 1228 snpp->snp_flags |= NETISR_SNP_FLAGS_M2CPUID; 1229 if (npp->np_drainedcpu != NULL) 1230 snpp->snp_flags |= NETISR_SNP_FLAGS_DRAINEDCPU; 1231 counter++; 1232 } 1233 NETISR_RUNLOCK(&tracker); 1234 KASSERT(counter <= NETISR_MAXPROT, 1235 ("sysctl_netisr_proto: counter too big (%d)", counter)); 1236 error = SYSCTL_OUT(req, snp_array, sizeof(*snp_array) * counter); 1237 free(snp_array, M_TEMP); 1238 return (error); 1239} 1240 1241SYSCTL_PROC(_net_isr, OID_AUTO, proto, 1242 CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_proto, 1243 "S,sysctl_netisr_proto", 1244 "Return list of protocols registered with netisr"); 1245 1246/* 1247 * Sysctl monitoring for netisr: query a list of workstreams. 1248 */ 1249static int 1250sysctl_netisr_workstream(SYSCTL_HANDLER_ARGS) 1251{ 1252 struct rm_priotracker tracker; 1253 struct sysctl_netisr_workstream *snwsp, *snws_array; 1254 struct netisr_workstream *nwsp; 1255 u_int counter, cpuid; 1256 int error; 1257 1258 if (req->newptr != NULL) 1259 return (EINVAL); 1260 snws_array = malloc(sizeof(*snws_array) * MAXCPU, M_TEMP, 1261 M_ZERO | M_WAITOK); 1262 counter = 0; 1263 NETISR_RLOCK(&tracker); 1264 CPU_FOREACH(cpuid) { 1265 nwsp = DPCPU_ID_PTR(cpuid, nws); 1266 if (nwsp->nws_intr_event == NULL) 1267 continue; 1268 NWS_LOCK(nwsp); 1269 snwsp = &snws_array[counter]; 1270 snwsp->snws_version = sizeof(*snwsp); 1271 1272 /* 1273 * For now, we equate workstream IDs and CPU IDs in the 1274 * kernel, but expose them independently to userspace in case 1275 * that assumption changes in the future. 1276 */ 1277 snwsp->snws_wsid = cpuid; 1278 snwsp->snws_cpu = cpuid; 1279 if (nwsp->nws_intr_event != NULL) 1280 snwsp->snws_flags |= NETISR_SNWS_FLAGS_INTR; 1281 NWS_UNLOCK(nwsp); 1282 counter++; 1283 } 1284 NETISR_RUNLOCK(&tracker); 1285 KASSERT(counter <= MAXCPU, 1286 ("sysctl_netisr_workstream: counter too big (%d)", counter)); 1287 error = SYSCTL_OUT(req, snws_array, sizeof(*snws_array) * counter); 1288 free(snws_array, M_TEMP); 1289 return (error); 1290} 1291 1292SYSCTL_PROC(_net_isr, OID_AUTO, workstream, 1293 CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_workstream, 1294 "S,sysctl_netisr_workstream", 1295 "Return list of workstreams implemented by netisr"); 1296 1297/* 1298 * Sysctl monitoring for netisr: query per-protocol data across all 1299 * workstreams. 1300 */ 1301static int 1302sysctl_netisr_work(SYSCTL_HANDLER_ARGS) 1303{ 1304 struct rm_priotracker tracker; 1305 struct sysctl_netisr_work *snwp, *snw_array; 1306 struct netisr_workstream *nwsp; 1307 struct netisr_proto *npp; 1308 struct netisr_work *nwp; 1309 u_int counter, cpuid, proto; 1310 int error; 1311 1312 if (req->newptr != NULL) 1313 return (EINVAL); 1314 snw_array = malloc(sizeof(*snw_array) * MAXCPU * NETISR_MAXPROT, 1315 M_TEMP, M_ZERO | M_WAITOK); 1316 counter = 0; 1317 NETISR_RLOCK(&tracker); 1318 CPU_FOREACH(cpuid) { 1319 nwsp = DPCPU_ID_PTR(cpuid, nws); 1320 if (nwsp->nws_intr_event == NULL) 1321 continue; 1322 NWS_LOCK(nwsp); 1323 for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1324 npp = &netisr_proto[proto]; 1325 if (npp->np_name == NULL) 1326 continue; 1327 nwp = &nwsp->nws_work[proto]; 1328 snwp = &snw_array[counter]; 1329 snwp->snw_version = sizeof(*snwp); 1330 snwp->snw_wsid = cpuid; /* See comment above. */ 1331 snwp->snw_proto = proto; 1332 snwp->snw_len = nwp->nw_len; 1333 snwp->snw_watermark = nwp->nw_watermark; 1334 snwp->snw_dispatched = nwp->nw_dispatched; 1335 snwp->snw_hybrid_dispatched = 1336 nwp->nw_hybrid_dispatched; 1337 snwp->snw_qdrops = nwp->nw_qdrops; 1338 snwp->snw_queued = nwp->nw_queued; 1339 snwp->snw_handled = nwp->nw_handled; 1340 counter++; 1341 } 1342 NWS_UNLOCK(nwsp); 1343 } 1344 KASSERT(counter <= MAXCPU * NETISR_MAXPROT, 1345 ("sysctl_netisr_work: counter too big (%d)", counter)); 1346 NETISR_RUNLOCK(&tracker); 1347 error = SYSCTL_OUT(req, snw_array, sizeof(*snw_array) * counter); 1348 free(snw_array, M_TEMP); 1349 return (error); 1350} 1351 1352SYSCTL_PROC(_net_isr, OID_AUTO, work, 1353 CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_work, 1354 "S,sysctl_netisr_work", 1355 "Return list of per-workstream, per-protocol work in netisr"); 1356 1357#ifdef DDB 1358DB_SHOW_COMMAND(netisr, db_show_netisr) 1359{ 1360 struct netisr_workstream *nwsp; 1361 struct netisr_work *nwp; 1362 int first, proto; 1363 u_int cpuid; 1364 1365 db_printf("%3s %6s %5s %5s %5s %8s %8s %8s %8s\n", "CPU", "Proto", 1366 "Len", "WMark", "Max", "Disp", "HDisp", "Drop", "Queue"); 1367 CPU_FOREACH(cpuid) { 1368 nwsp = DPCPU_ID_PTR(cpuid, nws); 1369 if (nwsp->nws_intr_event == NULL) 1370 continue; 1371 first = 1; 1372 for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1373 if (netisr_proto[proto].np_handler == NULL) 1374 continue; 1375 nwp = &nwsp->nws_work[proto]; 1376 if (first) { 1377 db_printf("%3d ", cpuid); 1378 first = 0; 1379 } else 1380 db_printf("%3s ", ""); 1381 db_printf( 1382 "%6s %5d %5d %5d %8ju %8ju %8ju %8ju\n", 1383 netisr_proto[proto].np_name, nwp->nw_len, 1384 nwp->nw_watermark, nwp->nw_qlimit, 1385 nwp->nw_dispatched, nwp->nw_hybrid_dispatched, 1386 nwp->nw_qdrops, nwp->nw_queued); 1387 } 1388 } 1389} 1390#endif 1391