1111888Sjlemon/*- 2193219Srwatson * Copyright (c) 2007-2009 Robert N. M. Watson 3222249Srwatson * Copyright (c) 2010-2011 Juniper Networks, Inc. 4103781Sjake * All rights reserved. 5103781Sjake * 6204199Srwatson * This software was developed by Robert N. M. Watson under contract 7204199Srwatson * to Juniper Networks, Inc. 8204199Srwatson * 9103781Sjake * Redistribution and use in source and binary forms, with or without 10103781Sjake * modification, are permitted provided that the following conditions 11103781Sjake * are met: 12103781Sjake * 1. Redistributions of source code must retain the above copyright 13111888Sjlemon * notice, this list of conditions and the following disclaimer. 14103781Sjake * 2. Redistributions in binary form must reproduce the above copyright 15103781Sjake * notice, this list of conditions and the following disclaimer in the 16103781Sjake * documentation and/or other materials provided with the distribution. 17103781Sjake * 18111888Sjlemon * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19111888Sjlemon * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20111888Sjlemon * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21111888Sjlemon * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22111888Sjlemon * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23111888Sjlemon * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24111888Sjlemon * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25111888Sjlemon * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26111888Sjlemon * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27111888Sjlemon * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28111888Sjlemon * SUCH DAMAGE. 29193219Srwatson */ 30193219Srwatson 31193219Srwatson#include <sys/cdefs.h> 32193219Srwatson__FBSDID("$FreeBSD$"); 33193219Srwatson 34193219Srwatson/* 35193219Srwatson * netisr is a packet dispatch service, allowing synchronous (directly 36193219Srwatson * dispatched) and asynchronous (deferred dispatch) processing of packets by 37193219Srwatson * registered protocol handlers. Callers pass a protocol identifier and 38193219Srwatson * packet to netisr, along with a direct dispatch hint, and work will either 39200898Srwatson * be immediately processed by the registered handler, or passed to a 40200898Srwatson * software interrupt (SWI) thread for deferred dispatch. Callers will 41200898Srwatson * generally select one or the other based on: 42103781Sjake * 43200898Srwatson * - Whether directly dispatching a netisr handler lead to code reentrance or 44193219Srwatson * lock recursion, such as entering the socket code from the socket code. 45200898Srwatson * - Whether directly dispatching a netisr handler lead to recursive 46193219Srwatson * processing, such as when decapsulating several wrapped layers of tunnel 47193219Srwatson * information (IPSEC within IPSEC within ...). 48193219Srwatson * 49193219Srwatson * Maintaining ordering for protocol streams is a critical design concern. 50193219Srwatson * Enforcing ordering limits the opportunity for concurrency, but maintains 51193219Srwatson * the strong ordering requirements found in some protocols, such as TCP. Of 52193219Srwatson * related concern is CPU affinity--it is desirable to process all data 53193219Srwatson * associated with a particular stream on the same CPU over time in order to 54193219Srwatson * avoid acquiring locks associated with the connection on different CPUs, 55193219Srwatson * keep connection data in one cache, and to generally encourage associated 56193219Srwatson * user threads to live on the same CPU as the stream. It's also desirable 57193219Srwatson * to avoid lock migration and contention where locks are associated with 58193219Srwatson * more than one flow. 59193219Srwatson * 60193219Srwatson * netisr supports several policy variations, represented by the 61200898Srwatson * NETISR_POLICY_* constants, allowing protocols to play various roles in 62193219Srwatson * identifying flows, assigning work to CPUs, etc. These are described in 63200898Srwatson * netisr.h. 64103781Sjake */ 65103781Sjake 66193219Srwatson#include "opt_ddb.h" 67150968Sglebius#include "opt_device_polling.h" 68134443Srwatson 69103781Sjake#include <sys/param.h> 70111888Sjlemon#include <sys/bus.h> 71103781Sjake#include <sys/kernel.h> 72111888Sjlemon#include <sys/kthread.h> 73193219Srwatson#include <sys/interrupt.h> 74111888Sjlemon#include <sys/lock.h> 75193219Srwatson#include <sys/mbuf.h> 76193219Srwatson#include <sys/mutex.h> 77195019Srwatson#include <sys/pcpu.h> 78111888Sjlemon#include <sys/proc.h> 79193219Srwatson#include <sys/rmlock.h> 80193219Srwatson#include <sys/sched.h> 81193219Srwatson#include <sys/smp.h> 82193219Srwatson#include <sys/socket.h> 83111888Sjlemon#include <sys/sysctl.h> 84193219Srwatson#include <sys/systm.h> 85103781Sjake 86193219Srwatson#ifdef DDB 87193219Srwatson#include <ddb/ddb.h> 88193219Srwatson#endif 89111888Sjlemon 90204497Srwatson#define _WANT_NETISR_INTERNAL /* Enable definitions from netisr_internal.h */ 91111888Sjlemon#include <net/if.h> 92111888Sjlemon#include <net/if_var.h> 93103781Sjake#include <net/netisr.h> 94204497Srwatson#include <net/netisr_internal.h> 95196019Srwatson#include <net/vnet.h> 96103781Sjake 97193219Srwatson/*- 98193219Srwatson * Synchronize use and modification of the registered netisr data structures; 99193219Srwatson * acquire a read lock while modifying the set of registered protocols to 100193219Srwatson * prevent partially registered or unregistered protocols from being run. 101193219Srwatson * 102193219Srwatson * The following data structures and fields are protected by this lock: 103193219Srwatson * 104204497Srwatson * - The netisr_proto array, including all fields of struct netisr_proto. 105193219Srwatson * - The nws array, including all fields of struct netisr_worker. 106193219Srwatson * - The nws_array array. 107193219Srwatson * 108193219Srwatson * Note: the NETISR_LOCKING define controls whether read locks are acquired 109193219Srwatson * in packet processing paths requiring netisr registration stability. This 110200898Srwatson * is disabled by default as it can lead to measurable performance 111193219Srwatson * degradation even with rmlocks (3%-6% for loopback ping-pong traffic), and 112193219Srwatson * because netisr registration and unregistration is extremely rare at 113193219Srwatson * runtime. If it becomes more common, this decision should be revisited. 114193219Srwatson * 115193219Srwatson * XXXRW: rmlocks don't support assertions. 116193219Srwatson */ 117193219Srwatsonstatic struct rmlock netisr_rmlock; 118193219Srwatson#define NETISR_LOCK_INIT() rm_init_flags(&netisr_rmlock, "netisr", \ 119193219Srwatson RM_NOWITNESS) 120193219Srwatson#define NETISR_LOCK_ASSERT() 121193219Srwatson#define NETISR_RLOCK(tracker) rm_rlock(&netisr_rmlock, (tracker)) 122193219Srwatson#define NETISR_RUNLOCK(tracker) rm_runlock(&netisr_rmlock, (tracker)) 123193219Srwatson#define NETISR_WLOCK() rm_wlock(&netisr_rmlock) 124193219Srwatson#define NETISR_WUNLOCK() rm_wunlock(&netisr_rmlock) 125193219Srwatson/* #define NETISR_LOCKING */ 126103781Sjake 127227309Sedstatic SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr"); 128103781Sjake 129193219Srwatson/*- 130222249Srwatson * Three global direct dispatch policies are supported: 131193219Srwatson * 132222249Srwatson * NETISR_DISPATCH_QUEUED: All work is deferred for a netisr, regardless of 133222249Srwatson * context (may be overriden by protocols). 134193219Srwatson * 135222249Srwatson * NETISR_DISPATCH_HYBRID: If the executing context allows direct dispatch, 136222249Srwatson * and we're running on the CPU the work would be performed on, then direct 137222249Srwatson * dispatch it if it wouldn't violate ordering constraints on the workstream. 138193219Srwatson * 139222249Srwatson * NETISR_DISPATCH_DIRECT: If the executing context allows direct dispatch, 140222249Srwatson * always direct dispatch. (The default.) 141193219Srwatson * 142193219Srwatson * Notice that changing the global policy could lead to short periods of 143193219Srwatson * misordered processing, but this is considered acceptable as compared to 144222249Srwatson * the complexity of enforcing ordering during policy changes. Protocols can 145222249Srwatson * override the global policy (when they're not doing that, they select 146222249Srwatson * NETISR_DISPATCH_DEFAULT). 147193219Srwatson */ 148222249Srwatson#define NETISR_DISPATCH_POLICY_DEFAULT NETISR_DISPATCH_DIRECT 149222249Srwatson#define NETISR_DISPATCH_POLICY_MAXSTR 20 /* Used for temporary buffers. */ 150222249Srwatsonstatic u_int netisr_dispatch_policy = NETISR_DISPATCH_POLICY_DEFAULT; 151222249Srwatsonstatic int sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS); 152222249SrwatsonSYSCTL_PROC(_net_isr, OID_AUTO, dispatch, CTLTYPE_STRING | CTLFLAG_RW | 153222249Srwatson CTLFLAG_TUN, 0, 0, sysctl_netisr_dispatch_policy, "A", 154222249Srwatson "netisr dispatch policy"); 155111888Sjlemon 156222249Srwatson/* 157193219Srwatson * Allow the administrator to limit the number of threads (CPUs) to use for 158193219Srwatson * netisr. We don't check netisr_maxthreads before creating the thread for 159193219Srwatson * CPU 0, so in practice we ignore values <= 1. This must be set at boot. 160193219Srwatson * We will create at most one thread per CPU. 161193219Srwatson */ 162195078Srwatsonstatic int netisr_maxthreads = -1; /* Max number of threads. */ 163193219SrwatsonTUNABLE_INT("net.isr.maxthreads", &netisr_maxthreads); 164203913SpjdSYSCTL_INT(_net_isr, OID_AUTO, maxthreads, CTLFLAG_RDTUN, 165193219Srwatson &netisr_maxthreads, 0, 166193219Srwatson "Use at most this many CPUs for netisr processing"); 167193219Srwatson 168193219Srwatsonstatic int netisr_bindthreads = 0; /* Bind threads to CPUs. */ 169193219SrwatsonTUNABLE_INT("net.isr.bindthreads", &netisr_bindthreads); 170203913SpjdSYSCTL_INT(_net_isr, OID_AUTO, bindthreads, CTLFLAG_RDTUN, 171193219Srwatson &netisr_bindthreads, 0, "Bind netisr threads to CPUs."); 172193219Srwatson 173193219Srwatson/* 174200898Srwatson * Limit per-workstream mbuf queue limits s to at most net.isr.maxqlimit, 175200898Srwatson * both for initial configuration and later modification using 176200898Srwatson * netisr_setqlimit(). 177193219Srwatson */ 178193219Srwatson#define NETISR_DEFAULT_MAXQLIMIT 10240 179193219Srwatsonstatic u_int netisr_maxqlimit = NETISR_DEFAULT_MAXQLIMIT; 180193219SrwatsonTUNABLE_INT("net.isr.maxqlimit", &netisr_maxqlimit); 181217322SmdfSYSCTL_UINT(_net_isr, OID_AUTO, maxqlimit, CTLFLAG_RDTUN, 182193219Srwatson &netisr_maxqlimit, 0, 183193219Srwatson "Maximum netisr per-protocol, per-CPU queue depth."); 184193219Srwatson 185193219Srwatson/* 186200898Srwatson * The default per-workstream mbuf queue limit for protocols that don't 187200898Srwatson * initialize the nh_qlimit field of their struct netisr_handler. If this is 188200898Srwatson * set above netisr_maxqlimit, we truncate it to the maximum during boot. 189193219Srwatson */ 190193219Srwatson#define NETISR_DEFAULT_DEFAULTQLIMIT 256 191193219Srwatsonstatic u_int netisr_defaultqlimit = NETISR_DEFAULT_DEFAULTQLIMIT; 192193219SrwatsonTUNABLE_INT("net.isr.defaultqlimit", &netisr_defaultqlimit); 193217322SmdfSYSCTL_UINT(_net_isr, OID_AUTO, defaultqlimit, CTLFLAG_RDTUN, 194193219Srwatson &netisr_defaultqlimit, 0, 195193219Srwatson "Default netisr per-protocol, per-CPU queue limit if not set by protocol"); 196193219Srwatson 197193219Srwatson/* 198204497Srwatson * Store and export the compile-time constant NETISR_MAXPROT limit on the 199204497Srwatson * number of protocols that can register with netisr at a time. This is 200204497Srwatson * required for crashdump analysis, as it sizes netisr_proto[]. 201193219Srwatson */ 202204497Srwatsonstatic u_int netisr_maxprot = NETISR_MAXPROT; 203217322SmdfSYSCTL_UINT(_net_isr, OID_AUTO, maxprot, CTLFLAG_RD, 204204497Srwatson &netisr_maxprot, 0, 205204497Srwatson "Compile-time limit on the number of protocols supported by netisr."); 206193219Srwatson 207193219Srwatson/* 208204497Srwatson * The netisr_proto array describes all registered protocols, indexed by 209204497Srwatson * protocol number. See netisr_internal.h for more details. 210193219Srwatson */ 211204497Srwatsonstatic struct netisr_proto netisr_proto[NETISR_MAXPROT]; 212193219Srwatson 213193219Srwatson/* 214204497Srwatson * Per-CPU workstream data. See netisr_internal.h for more details. 215193219Srwatson */ 216195019SrwatsonDPCPU_DEFINE(struct netisr_workstream, nws); 217193219Srwatson 218193219Srwatson/* 219193219Srwatson * Map contiguous values between 0 and nws_count into CPU IDs appropriate for 220195019Srwatson * accessing workstreams. This allows constructions of the form 221195019Srwatson * DPCPU_ID_GET(nws_array[arbitraryvalue % nws_count], nws). 222193219Srwatson */ 223193219Srwatsonstatic u_int nws_array[MAXCPU]; 224193219Srwatson 225193219Srwatson/* 226193219Srwatson * Number of registered workstreams. Will be at most the number of running 227193219Srwatson * CPUs once fully started. 228193219Srwatson */ 229193219Srwatsonstatic u_int nws_count; 230217322SmdfSYSCTL_UINT(_net_isr, OID_AUTO, numthreads, CTLFLAG_RD, 231193219Srwatson &nws_count, 0, "Number of extant netisr threads."); 232193219Srwatson 233193219Srwatson/* 234193219Srwatson * Synchronization for each workstream: a mutex protects all mutable fields 235193219Srwatson * in each stream, including per-protocol state (mbuf queues). The SWI is 236193219Srwatson * woken up if asynchronous dispatch is required. 237193219Srwatson */ 238193219Srwatson#define NWS_LOCK(s) mtx_lock(&(s)->nws_mtx) 239193219Srwatson#define NWS_LOCK_ASSERT(s) mtx_assert(&(s)->nws_mtx, MA_OWNED) 240193219Srwatson#define NWS_UNLOCK(s) mtx_unlock(&(s)->nws_mtx) 241193219Srwatson#define NWS_SIGNAL(s) swi_sched((s)->nws_swi_cookie, 0) 242193219Srwatson 243193219Srwatson/* 244193219Srwatson * Utility routines for protocols that implement their own mapping of flows 245193219Srwatson * to CPUs. 246193219Srwatson */ 247193219Srwatsonu_int 248193219Srwatsonnetisr_get_cpucount(void) 249193219Srwatson{ 250193219Srwatson 251193219Srwatson return (nws_count); 252193219Srwatson} 253193219Srwatson 254193219Srwatsonu_int 255193219Srwatsonnetisr_get_cpuid(u_int cpunumber) 256193219Srwatson{ 257193219Srwatson 258193219Srwatson KASSERT(cpunumber < nws_count, ("%s: %u > %u", __func__, cpunumber, 259193219Srwatson nws_count)); 260193219Srwatson 261193219Srwatson return (nws_array[cpunumber]); 262193219Srwatson} 263193219Srwatson 264193219Srwatson/* 265200898Srwatson * The default implementation of flow -> CPU ID mapping. 266193219Srwatson * 267193219Srwatson * Non-static so that protocols can use it to map their own work to specific 268193219Srwatson * CPUs in a manner consistent to netisr for affinity purposes. 269193219Srwatson */ 270193219Srwatsonu_int 271193219Srwatsonnetisr_default_flow2cpu(u_int flowid) 272193219Srwatson{ 273193219Srwatson 274193219Srwatson return (nws_array[flowid % nws_count]); 275193219Srwatson} 276193219Srwatson 277193219Srwatson/* 278222249Srwatson * Dispatch tunable and sysctl configuration. 279222249Srwatson */ 280222249Srwatsonstruct netisr_dispatch_table_entry { 281222249Srwatson u_int ndte_policy; 282222249Srwatson const char *ndte_policy_str; 283222249Srwatson}; 284222249Srwatsonstatic const struct netisr_dispatch_table_entry netisr_dispatch_table[] = { 285222249Srwatson { NETISR_DISPATCH_DEFAULT, "default" }, 286222249Srwatson { NETISR_DISPATCH_DEFERRED, "deferred" }, 287222249Srwatson { NETISR_DISPATCH_HYBRID, "hybrid" }, 288222249Srwatson { NETISR_DISPATCH_DIRECT, "direct" }, 289222249Srwatson}; 290222249Srwatsonstatic const u_int netisr_dispatch_table_len = 291222249Srwatson (sizeof(netisr_dispatch_table) / sizeof(netisr_dispatch_table[0])); 292222249Srwatson 293222249Srwatsonstatic void 294222249Srwatsonnetisr_dispatch_policy_to_str(u_int dispatch_policy, char *buffer, 295222249Srwatson u_int buflen) 296222249Srwatson{ 297222249Srwatson const struct netisr_dispatch_table_entry *ndtep; 298222249Srwatson const char *str; 299222249Srwatson u_int i; 300222249Srwatson 301222249Srwatson str = "unknown"; 302222249Srwatson for (i = 0; i < netisr_dispatch_table_len; i++) { 303222249Srwatson ndtep = &netisr_dispatch_table[i]; 304222249Srwatson if (ndtep->ndte_policy == dispatch_policy) { 305222249Srwatson str = ndtep->ndte_policy_str; 306222249Srwatson break; 307222249Srwatson } 308222249Srwatson } 309222249Srwatson snprintf(buffer, buflen, "%s", str); 310222249Srwatson} 311222249Srwatson 312222249Srwatsonstatic int 313222249Srwatsonnetisr_dispatch_policy_from_str(const char *str, u_int *dispatch_policyp) 314222249Srwatson{ 315222249Srwatson const struct netisr_dispatch_table_entry *ndtep; 316222249Srwatson u_int i; 317222249Srwatson 318222249Srwatson for (i = 0; i < netisr_dispatch_table_len; i++) { 319222249Srwatson ndtep = &netisr_dispatch_table[i]; 320222249Srwatson if (strcmp(ndtep->ndte_policy_str, str) == 0) { 321222249Srwatson *dispatch_policyp = ndtep->ndte_policy; 322222249Srwatson return (0); 323222249Srwatson } 324222249Srwatson } 325222249Srwatson return (EINVAL); 326222249Srwatson} 327222249Srwatson 328222249Srwatsonstatic int 329222249Srwatsonsysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS) 330222249Srwatson{ 331222249Srwatson char tmp[NETISR_DISPATCH_POLICY_MAXSTR]; 332222249Srwatson u_int dispatch_policy; 333222249Srwatson int error; 334222249Srwatson 335222249Srwatson netisr_dispatch_policy_to_str(netisr_dispatch_policy, tmp, 336222249Srwatson sizeof(tmp)); 337222249Srwatson error = sysctl_handle_string(oidp, tmp, sizeof(tmp), req); 338222249Srwatson if (error == 0 && req->newptr != NULL) { 339222249Srwatson error = netisr_dispatch_policy_from_str(tmp, 340222249Srwatson &dispatch_policy); 341222249Srwatson if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT) 342222249Srwatson error = EINVAL; 343255329Sdavide if (error == 0) 344222249Srwatson netisr_dispatch_policy = dispatch_policy; 345222249Srwatson } 346222249Srwatson return (error); 347222249Srwatson} 348222249Srwatson 349222249Srwatson/* 350193219Srwatson * Register a new netisr handler, which requires initializing per-protocol 351193219Srwatson * fields for each workstream. All netisr work is briefly suspended while 352193219Srwatson * the protocol is installed. 353193219Srwatson */ 354103781Sjakevoid 355193219Srwatsonnetisr_register(const struct netisr_handler *nhp) 356103781Sjake{ 357193219Srwatson struct netisr_work *npwp; 358193219Srwatson const char *name; 359193219Srwatson u_int i, proto; 360193219Srwatson 361193219Srwatson proto = nhp->nh_proto; 362193219Srwatson name = nhp->nh_name; 363193219Srwatson 364193219Srwatson /* 365193219Srwatson * Test that the requested registration is valid. 366193219Srwatson */ 367193219Srwatson KASSERT(nhp->nh_name != NULL, 368193219Srwatson ("%s: nh_name NULL for %u", __func__, proto)); 369193219Srwatson KASSERT(nhp->nh_handler != NULL, 370193219Srwatson ("%s: nh_handler NULL for %s", __func__, name)); 371193219Srwatson KASSERT(nhp->nh_policy == NETISR_POLICY_SOURCE || 372193219Srwatson nhp->nh_policy == NETISR_POLICY_FLOW || 373193219Srwatson nhp->nh_policy == NETISR_POLICY_CPU, 374193219Srwatson ("%s: unsupported nh_policy %u for %s", __func__, 375193219Srwatson nhp->nh_policy, name)); 376193219Srwatson KASSERT(nhp->nh_policy == NETISR_POLICY_FLOW || 377193219Srwatson nhp->nh_m2flow == NULL, 378193219Srwatson ("%s: nh_policy != FLOW but m2flow defined for %s", __func__, 379193219Srwatson name)); 380193219Srwatson KASSERT(nhp->nh_policy == NETISR_POLICY_CPU || nhp->nh_m2cpuid == NULL, 381193219Srwatson ("%s: nh_policy != CPU but m2cpuid defined for %s", __func__, 382193219Srwatson name)); 383193219Srwatson KASSERT(nhp->nh_policy != NETISR_POLICY_CPU || nhp->nh_m2cpuid != NULL, 384193219Srwatson ("%s: nh_policy == CPU but m2cpuid not defined for %s", __func__, 385193219Srwatson name)); 386222249Srwatson KASSERT(nhp->nh_dispatch == NETISR_DISPATCH_DEFAULT || 387222249Srwatson nhp->nh_dispatch == NETISR_DISPATCH_DEFERRED || 388222249Srwatson nhp->nh_dispatch == NETISR_DISPATCH_HYBRID || 389222249Srwatson nhp->nh_dispatch == NETISR_DISPATCH_DIRECT, 390222249Srwatson ("%s: invalid nh_dispatch (%u)", __func__, nhp->nh_dispatch)); 391222249Srwatson 392193219Srwatson KASSERT(proto < NETISR_MAXPROT, 393193219Srwatson ("%s(%u, %s): protocol too big", __func__, proto, name)); 394193219Srwatson 395193219Srwatson /* 396193219Srwatson * Test that no existing registration exists for this protocol. 397193219Srwatson */ 398193219Srwatson NETISR_WLOCK(); 399204497Srwatson KASSERT(netisr_proto[proto].np_name == NULL, 400193219Srwatson ("%s(%u, %s): name present", __func__, proto, name)); 401204497Srwatson KASSERT(netisr_proto[proto].np_handler == NULL, 402193219Srwatson ("%s(%u, %s): handler present", __func__, proto, name)); 403193219Srwatson 404204497Srwatson netisr_proto[proto].np_name = name; 405204497Srwatson netisr_proto[proto].np_handler = nhp->nh_handler; 406204497Srwatson netisr_proto[proto].np_m2flow = nhp->nh_m2flow; 407204497Srwatson netisr_proto[proto].np_m2cpuid = nhp->nh_m2cpuid; 408204497Srwatson netisr_proto[proto].np_drainedcpu = nhp->nh_drainedcpu; 409193219Srwatson if (nhp->nh_qlimit == 0) 410204497Srwatson netisr_proto[proto].np_qlimit = netisr_defaultqlimit; 411193219Srwatson else if (nhp->nh_qlimit > netisr_maxqlimit) { 412193219Srwatson printf("%s: %s requested queue limit %u capped to " 413193219Srwatson "net.isr.maxqlimit %u\n", __func__, name, nhp->nh_qlimit, 414193219Srwatson netisr_maxqlimit); 415204497Srwatson netisr_proto[proto].np_qlimit = netisr_maxqlimit; 416193219Srwatson } else 417204497Srwatson netisr_proto[proto].np_qlimit = nhp->nh_qlimit; 418204497Srwatson netisr_proto[proto].np_policy = nhp->nh_policy; 419222249Srwatson netisr_proto[proto].np_dispatch = nhp->nh_dispatch; 420209059Sjhb CPU_FOREACH(i) { 421195019Srwatson npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 422193219Srwatson bzero(npwp, sizeof(*npwp)); 423204497Srwatson npwp->nw_qlimit = netisr_proto[proto].np_qlimit; 424193219Srwatson } 425193219Srwatson NETISR_WUNLOCK(); 426103781Sjake} 427103781Sjake 428193219Srwatson/* 429193219Srwatson * Clear drop counters across all workstreams for a protocol. 430193219Srwatson */ 431111888Sjlemonvoid 432193219Srwatsonnetisr_clearqdrops(const struct netisr_handler *nhp) 433103781Sjake{ 434193219Srwatson struct netisr_work *npwp; 435193219Srwatson#ifdef INVARIANTS 436193219Srwatson const char *name; 437193219Srwatson#endif 438193219Srwatson u_int i, proto; 439193219Srwatson 440193219Srwatson proto = nhp->nh_proto; 441193219Srwatson#ifdef INVARIANTS 442193219Srwatson name = nhp->nh_name; 443193219Srwatson#endif 444193219Srwatson KASSERT(proto < NETISR_MAXPROT, 445193219Srwatson ("%s(%u): protocol too big for %s", __func__, proto, name)); 446193219Srwatson 447193219Srwatson NETISR_WLOCK(); 448204497Srwatson KASSERT(netisr_proto[proto].np_handler != NULL, 449193219Srwatson ("%s(%u): protocol not registered for %s", __func__, proto, 450193219Srwatson name)); 451193219Srwatson 452209059Sjhb CPU_FOREACH(i) { 453195019Srwatson npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 454193219Srwatson npwp->nw_qdrops = 0; 455193219Srwatson } 456193219Srwatson NETISR_WUNLOCK(); 457111888Sjlemon} 458111888Sjlemon 459193219Srwatson/* 460200898Srwatson * Query current drop counters across all workstreams for a protocol. 461193219Srwatson */ 462111888Sjlemonvoid 463193219Srwatsonnetisr_getqdrops(const struct netisr_handler *nhp, u_int64_t *qdropp) 464111888Sjlemon{ 465193219Srwatson struct netisr_work *npwp; 466193219Srwatson struct rm_priotracker tracker; 467193219Srwatson#ifdef INVARIANTS 468193219Srwatson const char *name; 469193219Srwatson#endif 470193219Srwatson u_int i, proto; 471193219Srwatson 472193219Srwatson *qdropp = 0; 473193219Srwatson proto = nhp->nh_proto; 474193219Srwatson#ifdef INVARIANTS 475193219Srwatson name = nhp->nh_name; 476193219Srwatson#endif 477193219Srwatson KASSERT(proto < NETISR_MAXPROT, 478193219Srwatson ("%s(%u): protocol too big for %s", __func__, proto, name)); 479193219Srwatson 480193219Srwatson NETISR_RLOCK(&tracker); 481204497Srwatson KASSERT(netisr_proto[proto].np_handler != NULL, 482193219Srwatson ("%s(%u): protocol not registered for %s", __func__, proto, 483193219Srwatson name)); 484193219Srwatson 485209059Sjhb CPU_FOREACH(i) { 486195019Srwatson npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 487193219Srwatson *qdropp += npwp->nw_qdrops; 488193219Srwatson } 489193219Srwatson NETISR_RUNLOCK(&tracker); 490103781Sjake} 491103781Sjake 492193219Srwatson/* 493200898Srwatson * Query current per-workstream queue limit for a protocol. 494193219Srwatson */ 495193219Srwatsonvoid 496193219Srwatsonnetisr_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp) 497193219Srwatson{ 498193219Srwatson struct rm_priotracker tracker; 499193219Srwatson#ifdef INVARIANTS 500193219Srwatson const char *name; 501193219Srwatson#endif 502193219Srwatson u_int proto; 503111888Sjlemon 504193219Srwatson proto = nhp->nh_proto; 505193219Srwatson#ifdef INVARIANTS 506193219Srwatson name = nhp->nh_name; 507193219Srwatson#endif 508193219Srwatson KASSERT(proto < NETISR_MAXPROT, 509193219Srwatson ("%s(%u): protocol too big for %s", __func__, proto, name)); 510111888Sjlemon 511193219Srwatson NETISR_RLOCK(&tracker); 512204497Srwatson KASSERT(netisr_proto[proto].np_handler != NULL, 513193219Srwatson ("%s(%u): protocol not registered for %s", __func__, proto, 514193219Srwatson name)); 515204497Srwatson *qlimitp = netisr_proto[proto].np_qlimit; 516193219Srwatson NETISR_RUNLOCK(&tracker); 517193219Srwatson} 518111888Sjlemon 519193219Srwatson/* 520193219Srwatson * Update the queue limit across per-workstream queues for a protocol. We 521193219Srwatson * simply change the limits, and don't drain overflowed packets as they will 522193219Srwatson * (hopefully) take care of themselves shortly. 523193219Srwatson */ 524193219Srwatsonint 525193219Srwatsonnetisr_setqlimit(const struct netisr_handler *nhp, u_int qlimit) 526193219Srwatson{ 527193219Srwatson struct netisr_work *npwp; 528193219Srwatson#ifdef INVARIANTS 529193219Srwatson const char *name; 530193219Srwatson#endif 531193219Srwatson u_int i, proto; 532111888Sjlemon 533193219Srwatson if (qlimit > netisr_maxqlimit) 534193219Srwatson return (EINVAL); 535193219Srwatson 536193219Srwatson proto = nhp->nh_proto; 537193219Srwatson#ifdef INVARIANTS 538193219Srwatson name = nhp->nh_name; 539193219Srwatson#endif 540193219Srwatson KASSERT(proto < NETISR_MAXPROT, 541193219Srwatson ("%s(%u): protocol too big for %s", __func__, proto, name)); 542193219Srwatson 543193219Srwatson NETISR_WLOCK(); 544204497Srwatson KASSERT(netisr_proto[proto].np_handler != NULL, 545193219Srwatson ("%s(%u): protocol not registered for %s", __func__, proto, 546193219Srwatson name)); 547193219Srwatson 548204497Srwatson netisr_proto[proto].np_qlimit = qlimit; 549209059Sjhb CPU_FOREACH(i) { 550195019Srwatson npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 551193219Srwatson npwp->nw_qlimit = qlimit; 552193219Srwatson } 553193219Srwatson NETISR_WUNLOCK(); 554193219Srwatson return (0); 555193219Srwatson} 556193219Srwatson 557111888Sjlemon/* 558193219Srwatson * Drain all packets currently held in a particular protocol work queue. 559120704Srwatson */ 560120704Srwatsonstatic void 561193219Srwatsonnetisr_drain_proto(struct netisr_work *npwp) 562120704Srwatson{ 563120704Srwatson struct mbuf *m; 564120704Srwatson 565193219Srwatson /* 566193219Srwatson * We would assert the lock on the workstream but it's not passed in. 567193219Srwatson */ 568193219Srwatson while ((m = npwp->nw_head) != NULL) { 569193219Srwatson npwp->nw_head = m->m_nextpkt; 570193219Srwatson m->m_nextpkt = NULL; 571193219Srwatson if (npwp->nw_head == NULL) 572193219Srwatson npwp->nw_tail = NULL; 573193219Srwatson npwp->nw_len--; 574193219Srwatson m_freem(m); 575193219Srwatson } 576193219Srwatson KASSERT(npwp->nw_tail == NULL, ("%s: tail", __func__)); 577193219Srwatson KASSERT(npwp->nw_len == 0, ("%s: len", __func__)); 578193219Srwatson} 579193219Srwatson 580193219Srwatson/* 581193219Srwatson * Remove the registration of a network protocol, which requires clearing 582193219Srwatson * per-protocol fields across all workstreams, including freeing all mbufs in 583193219Srwatson * the queues at time of unregister. All work in netisr is briefly suspended 584193219Srwatson * while this takes place. 585193219Srwatson */ 586193219Srwatsonvoid 587193219Srwatsonnetisr_unregister(const struct netisr_handler *nhp) 588193219Srwatson{ 589193219Srwatson struct netisr_work *npwp; 590193219Srwatson#ifdef INVARIANTS 591193219Srwatson const char *name; 592193219Srwatson#endif 593193219Srwatson u_int i, proto; 594193219Srwatson 595193219Srwatson proto = nhp->nh_proto; 596193219Srwatson#ifdef INVARIANTS 597193219Srwatson name = nhp->nh_name; 598193219Srwatson#endif 599193219Srwatson KASSERT(proto < NETISR_MAXPROT, 600193219Srwatson ("%s(%u): protocol too big for %s", __func__, proto, name)); 601193219Srwatson 602193219Srwatson NETISR_WLOCK(); 603204497Srwatson KASSERT(netisr_proto[proto].np_handler != NULL, 604193219Srwatson ("%s(%u): protocol not registered for %s", __func__, proto, 605193219Srwatson name)); 606193219Srwatson 607204497Srwatson netisr_proto[proto].np_name = NULL; 608204497Srwatson netisr_proto[proto].np_handler = NULL; 609204497Srwatson netisr_proto[proto].np_m2flow = NULL; 610204497Srwatson netisr_proto[proto].np_m2cpuid = NULL; 611204497Srwatson netisr_proto[proto].np_qlimit = 0; 612204497Srwatson netisr_proto[proto].np_policy = 0; 613209059Sjhb CPU_FOREACH(i) { 614195019Srwatson npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; 615193219Srwatson netisr_drain_proto(npwp); 616193219Srwatson bzero(npwp, sizeof(*npwp)); 617193219Srwatson } 618193219Srwatson NETISR_WUNLOCK(); 619193219Srwatson} 620193219Srwatson 621193219Srwatson/* 622222249Srwatson * Compose the global and per-protocol policies on dispatch, and return the 623222249Srwatson * dispatch policy to use. 624222249Srwatson */ 625222249Srwatsonstatic u_int 626222249Srwatsonnetisr_get_dispatch(struct netisr_proto *npp) 627222249Srwatson{ 628222249Srwatson 629222249Srwatson /* 630222249Srwatson * Protocol-specific configuration overrides the global default. 631222249Srwatson */ 632222249Srwatson if (npp->np_dispatch != NETISR_DISPATCH_DEFAULT) 633222249Srwatson return (npp->np_dispatch); 634222249Srwatson return (netisr_dispatch_policy); 635222249Srwatson} 636222249Srwatson 637222249Srwatson/* 638193219Srwatson * Look up the workstream given a packet and source identifier. Do this by 639193219Srwatson * checking the protocol's policy, and optionally call out to the protocol 640193219Srwatson * for assistance if required. 641193219Srwatson */ 642193219Srwatsonstatic struct mbuf * 643222249Srwatsonnetisr_select_cpuid(struct netisr_proto *npp, u_int dispatch_policy, 644222249Srwatson uintptr_t source, struct mbuf *m, u_int *cpuidp) 645193219Srwatson{ 646193219Srwatson struct ifnet *ifp; 647222249Srwatson u_int policy; 648193219Srwatson 649193219Srwatson NETISR_LOCK_ASSERT(); 650193219Srwatson 651193219Srwatson /* 652193219Srwatson * In the event we have only one worker, shortcut and deliver to it 653193219Srwatson * without further ado. 654193219Srwatson */ 655193219Srwatson if (nws_count == 1) { 656193219Srwatson *cpuidp = nws_array[0]; 657193219Srwatson return (m); 658193219Srwatson } 659193219Srwatson 660193219Srwatson /* 661193219Srwatson * What happens next depends on the policy selected by the protocol. 662193219Srwatson * If we want to support per-interface policies, we should do that 663193219Srwatson * here first. 664193219Srwatson */ 665222249Srwatson policy = npp->np_policy; 666222249Srwatson if (policy == NETISR_POLICY_CPU) { 667222249Srwatson m = npp->np_m2cpuid(m, source, cpuidp); 668222249Srwatson if (m == NULL) 669222249Srwatson return (NULL); 670193219Srwatson 671222249Srwatson /* 672222249Srwatson * It's possible for a protocol not to have a good idea about 673222249Srwatson * where to process a packet, in which case we fall back on 674222249Srwatson * the netisr code to decide. In the hybrid case, return the 675222249Srwatson * current CPU ID, which will force an immediate direct 676222249Srwatson * dispatch. In the queued case, fall back on the SOURCE 677222249Srwatson * policy. 678222249Srwatson */ 679222249Srwatson if (*cpuidp != NETISR_CPUID_NONE) 680222249Srwatson return (m); 681222249Srwatson if (dispatch_policy == NETISR_DISPATCH_HYBRID) { 682222249Srwatson *cpuidp = curcpu; 683222249Srwatson return (m); 684222249Srwatson } 685222249Srwatson policy = NETISR_POLICY_SOURCE; 686222249Srwatson } 687222249Srwatson 688222249Srwatson if (policy == NETISR_POLICY_FLOW) { 689193219Srwatson if (!(m->m_flags & M_FLOWID) && npp->np_m2flow != NULL) { 690193219Srwatson m = npp->np_m2flow(m, source); 691193219Srwatson if (m == NULL) 692193219Srwatson return (NULL); 693193219Srwatson } 694193219Srwatson if (m->m_flags & M_FLOWID) { 695193219Srwatson *cpuidp = 696193219Srwatson netisr_default_flow2cpu(m->m_pkthdr.flowid); 697193219Srwatson return (m); 698193219Srwatson } 699222249Srwatson policy = NETISR_POLICY_SOURCE; 700222249Srwatson } 701193219Srwatson 702222249Srwatson KASSERT(policy == NETISR_POLICY_SOURCE, 703222249Srwatson ("%s: invalid policy %u for %s", __func__, npp->np_policy, 704222249Srwatson npp->np_name)); 705193219Srwatson 706222249Srwatson ifp = m->m_pkthdr.rcvif; 707222249Srwatson if (ifp != NULL) 708222249Srwatson *cpuidp = nws_array[(ifp->if_index + source) % nws_count]; 709222249Srwatson else 710222249Srwatson *cpuidp = nws_array[source % nws_count]; 711222249Srwatson return (m); 712193219Srwatson} 713193219Srwatson 714193219Srwatson/* 715193219Srwatson * Process packets associated with a workstream and protocol. For reasons of 716193219Srwatson * fairness, we process up to one complete netisr queue at a time, moving the 717193219Srwatson * queue to a stack-local queue for processing, but do not loop refreshing 718193219Srwatson * from the global queue. The caller is responsible for deciding whether to 719193219Srwatson * loop, and for setting the NWS_RUNNING flag. The passed workstream will be 720193219Srwatson * locked on entry and relocked before return, but will be released while 721193219Srwatson * processing. The number of packets processed is returned. 722193219Srwatson */ 723193219Srwatsonstatic u_int 724193219Srwatsonnetisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) 725193219Srwatson{ 726193219Srwatson struct netisr_work local_npw, *npwp; 727193219Srwatson u_int handled; 728193219Srwatson struct mbuf *m; 729193219Srwatson 730193219Srwatson NETISR_LOCK_ASSERT(); 731193219Srwatson NWS_LOCK_ASSERT(nwsp); 732193219Srwatson 733193219Srwatson KASSERT(nwsp->nws_flags & NWS_RUNNING, 734193219Srwatson ("%s(%u): not running", __func__, proto)); 735193219Srwatson KASSERT(proto >= 0 && proto < NETISR_MAXPROT, 736193219Srwatson ("%s(%u): invalid proto\n", __func__, proto)); 737193219Srwatson 738193219Srwatson npwp = &nwsp->nws_work[proto]; 739193219Srwatson if (npwp->nw_len == 0) 740193219Srwatson return (0); 741193219Srwatson 742193219Srwatson /* 743193219Srwatson * Move the global work queue to a thread-local work queue. 744193219Srwatson * 745193219Srwatson * Notice that this means the effective maximum length of the queue 746193219Srwatson * is actually twice that of the maximum queue length specified in 747193219Srwatson * the protocol registration call. 748193219Srwatson */ 749193219Srwatson handled = npwp->nw_len; 750193219Srwatson local_npw = *npwp; 751193219Srwatson npwp->nw_head = NULL; 752193219Srwatson npwp->nw_tail = NULL; 753193219Srwatson npwp->nw_len = 0; 754193219Srwatson nwsp->nws_pendingbits &= ~(1 << proto); 755193219Srwatson NWS_UNLOCK(nwsp); 756193219Srwatson while ((m = local_npw.nw_head) != NULL) { 757193219Srwatson local_npw.nw_head = m->m_nextpkt; 758193219Srwatson m->m_nextpkt = NULL; 759193219Srwatson if (local_npw.nw_head == NULL) 760193219Srwatson local_npw.nw_tail = NULL; 761193219Srwatson local_npw.nw_len--; 762218559Sbz VNET_ASSERT(m->m_pkthdr.rcvif != NULL, 763218559Sbz ("%s:%d rcvif == NULL: m=%p", __func__, __LINE__, m)); 764191816Szec CURVNET_SET(m->m_pkthdr.rcvif->if_vnet); 765204497Srwatson netisr_proto[proto].np_handler(m); 766191816Szec CURVNET_RESTORE(); 767120704Srwatson } 768193219Srwatson KASSERT(local_npw.nw_len == 0, 769193219Srwatson ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); 770204497Srwatson if (netisr_proto[proto].np_drainedcpu) 771204497Srwatson netisr_proto[proto].np_drainedcpu(nwsp->nws_cpu); 772193219Srwatson NWS_LOCK(nwsp); 773193219Srwatson npwp->nw_handled += handled; 774193219Srwatson return (handled); 775120704Srwatson} 776120704Srwatson 777120704Srwatson/* 778200898Srwatson * SWI handler for netisr -- processes packets in a set of workstreams that 779193219Srwatson * it owns, woken up by calls to NWS_SIGNAL(). If this workstream is already 780193219Srwatson * being direct dispatched, go back to sleep and wait for the dispatching 781193219Srwatson * thread to wake us up again. 782111888Sjlemon */ 783193219Srwatsonstatic void 784193219Srwatsonswi_net(void *arg) 785103781Sjake{ 786193219Srwatson#ifdef NETISR_LOCKING 787193219Srwatson struct rm_priotracker tracker; 788193219Srwatson#endif 789193219Srwatson struct netisr_workstream *nwsp; 790193219Srwatson u_int bits, prot; 791193219Srwatson 792193219Srwatson nwsp = arg; 793193219Srwatson 794193219Srwatson#ifdef DEVICE_POLLING 795193219Srwatson KASSERT(nws_count == 1, 796193219Srwatson ("%s: device_polling but nws_count != 1", __func__)); 797193219Srwatson netisr_poll(); 798193219Srwatson#endif 799193219Srwatson#ifdef NETISR_LOCKING 800193219Srwatson NETISR_RLOCK(&tracker); 801193219Srwatson#endif 802193219Srwatson NWS_LOCK(nwsp); 803193219Srwatson KASSERT(!(nwsp->nws_flags & NWS_RUNNING), ("swi_net: running")); 804193219Srwatson if (nwsp->nws_flags & NWS_DISPATCHING) 805193219Srwatson goto out; 806193219Srwatson nwsp->nws_flags |= NWS_RUNNING; 807193219Srwatson nwsp->nws_flags &= ~NWS_SCHEDULED; 808193219Srwatson while ((bits = nwsp->nws_pendingbits) != 0) { 809193219Srwatson while ((prot = ffs(bits)) != 0) { 810193219Srwatson prot--; 811193219Srwatson bits &= ~(1 << prot); 812193219Srwatson (void)netisr_process_workstream_proto(nwsp, prot); 813193219Srwatson } 814112011Sjlemon } 815193219Srwatson nwsp->nws_flags &= ~NWS_RUNNING; 816193219Srwatsonout: 817193219Srwatson NWS_UNLOCK(nwsp); 818193219Srwatson#ifdef NETISR_LOCKING 819193219Srwatson NETISR_RUNLOCK(&tracker); 820193219Srwatson#endif 821193219Srwatson#ifdef DEVICE_POLLING 822193219Srwatson netisr_pollmore(); 823193219Srwatson#endif 824193219Srwatson} 825180239Srwatson 826193219Srwatsonstatic int 827193219Srwatsonnetisr_queue_workstream(struct netisr_workstream *nwsp, u_int proto, 828193219Srwatson struct netisr_work *npwp, struct mbuf *m, int *dosignalp) 829193219Srwatson{ 830193219Srwatson 831193219Srwatson NWS_LOCK_ASSERT(nwsp); 832193219Srwatson 833193219Srwatson *dosignalp = 0; 834193219Srwatson if (npwp->nw_len < npwp->nw_qlimit) { 835193219Srwatson m->m_nextpkt = NULL; 836193219Srwatson if (npwp->nw_head == NULL) { 837193219Srwatson npwp->nw_head = m; 838193219Srwatson npwp->nw_tail = m; 839193219Srwatson } else { 840193219Srwatson npwp->nw_tail->m_nextpkt = m; 841193219Srwatson npwp->nw_tail = m; 842193219Srwatson } 843193219Srwatson npwp->nw_len++; 844193219Srwatson if (npwp->nw_len > npwp->nw_watermark) 845193219Srwatson npwp->nw_watermark = npwp->nw_len; 846200898Srwatson 847200898Srwatson /* 848200898Srwatson * We must set the bit regardless of NWS_RUNNING, so that 849200898Srwatson * swi_net() keeps calling netisr_process_workstream_proto(). 850200898Srwatson */ 851193219Srwatson nwsp->nws_pendingbits |= (1 << proto); 852193219Srwatson if (!(nwsp->nws_flags & 853193219Srwatson (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED))) { 854193219Srwatson nwsp->nws_flags |= NWS_SCHEDULED; 855193219Srwatson *dosignalp = 1; /* Defer until unlocked. */ 856193219Srwatson } 857193219Srwatson npwp->nw_queued++; 858193219Srwatson return (0); 859111888Sjlemon } else { 860195182Sbz m_freem(m); 861193219Srwatson npwp->nw_qdrops++; 862193219Srwatson return (ENOBUFS); 863103781Sjake } 864103781Sjake} 865103781Sjake 866193219Srwatsonstatic int 867193219Srwatsonnetisr_queue_internal(u_int proto, struct mbuf *m, u_int cpuid) 868193219Srwatson{ 869193219Srwatson struct netisr_workstream *nwsp; 870193219Srwatson struct netisr_work *npwp; 871193219Srwatson int dosignal, error; 872193219Srwatson 873193219Srwatson#ifdef NETISR_LOCKING 874193219Srwatson NETISR_LOCK_ASSERT(); 875193219Srwatson#endif 876195078Srwatson KASSERT(cpuid <= mp_maxid, ("%s: cpuid too big (%u, %u)", __func__, 877195078Srwatson cpuid, mp_maxid)); 878195019Srwatson KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 879193219Srwatson 880193219Srwatson dosignal = 0; 881193219Srwatson error = 0; 882195019Srwatson nwsp = DPCPU_ID_PTR(cpuid, nws); 883193219Srwatson npwp = &nwsp->nws_work[proto]; 884193219Srwatson NWS_LOCK(nwsp); 885193219Srwatson error = netisr_queue_workstream(nwsp, proto, npwp, m, &dosignal); 886193219Srwatson NWS_UNLOCK(nwsp); 887193219Srwatson if (dosignal) 888193219Srwatson NWS_SIGNAL(nwsp); 889193219Srwatson return (error); 890193219Srwatson} 891193219Srwatson 892193219Srwatsonint 893193219Srwatsonnetisr_queue_src(u_int proto, uintptr_t source, struct mbuf *m) 894193219Srwatson{ 895193219Srwatson#ifdef NETISR_LOCKING 896193219Srwatson struct rm_priotracker tracker; 897193219Srwatson#endif 898193219Srwatson u_int cpuid; 899193219Srwatson int error; 900193219Srwatson 901193219Srwatson KASSERT(proto < NETISR_MAXPROT, 902193219Srwatson ("%s: invalid proto %u", __func__, proto)); 903193219Srwatson 904193219Srwatson#ifdef NETISR_LOCKING 905193219Srwatson NETISR_RLOCK(&tracker); 906193219Srwatson#endif 907204497Srwatson KASSERT(netisr_proto[proto].np_handler != NULL, 908193219Srwatson ("%s: invalid proto %u", __func__, proto)); 909193219Srwatson 910222249Srwatson m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_DEFERRED, 911222249Srwatson source, m, &cpuid); 912195019Srwatson if (m != NULL) { 913195019Srwatson KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, 914195019Srwatson cpuid)); 915193219Srwatson error = netisr_queue_internal(proto, m, cpuid); 916195019Srwatson } else 917193219Srwatson error = ENOBUFS; 918193219Srwatson#ifdef NETISR_LOCKING 919193219Srwatson NETISR_RUNLOCK(&tracker); 920193219Srwatson#endif 921193219Srwatson return (error); 922193219Srwatson} 923193219Srwatson 924193219Srwatsonint 925193219Srwatsonnetisr_queue(u_int proto, struct mbuf *m) 926193219Srwatson{ 927193219Srwatson 928193219Srwatson return (netisr_queue_src(proto, 0, m)); 929193219Srwatson} 930193219Srwatson 931111888Sjlemon/* 932200898Srwatson * Dispatch a packet for netisr processing; direct dispatch is permitted by 933193219Srwatson * calling context. 934111888Sjlemon */ 935111888Sjlemonint 936193219Srwatsonnetisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) 937111888Sjlemon{ 938193219Srwatson#ifdef NETISR_LOCKING 939193219Srwatson struct rm_priotracker tracker; 940193219Srwatson#endif 941193219Srwatson struct netisr_workstream *nwsp; 942222249Srwatson struct netisr_proto *npp; 943193219Srwatson struct netisr_work *npwp; 944193219Srwatson int dosignal, error; 945222249Srwatson u_int cpuid, dispatch_policy; 946193219Srwatson 947193219Srwatson KASSERT(proto < NETISR_MAXPROT, 948193219Srwatson ("%s: invalid proto %u", __func__, proto)); 949193219Srwatson#ifdef NETISR_LOCKING 950193219Srwatson NETISR_RLOCK(&tracker); 951193219Srwatson#endif 952222249Srwatson npp = &netisr_proto[proto]; 953222249Srwatson KASSERT(npp->np_handler != NULL, ("%s: invalid proto %u", __func__, 954222249Srwatson proto)); 955193219Srwatson 956222249Srwatson dispatch_policy = netisr_get_dispatch(npp); 957222249Srwatson if (dispatch_policy == NETISR_DISPATCH_DEFERRED) 958222249Srwatson return (netisr_queue_src(proto, source, m)); 959222249Srwatson 960193219Srwatson /* 961193219Srwatson * If direct dispatch is forced, then unconditionally dispatch 962193219Srwatson * without a formal CPU selection. Borrow the current CPU's stats, 963193219Srwatson * even if there's no worker on it. In this case we don't update 964193219Srwatson * nws_flags because all netisr processing will be source ordered due 965193219Srwatson * to always being forced to directly dispatch. 966193219Srwatson */ 967222249Srwatson if (dispatch_policy == NETISR_DISPATCH_DIRECT) { 968195019Srwatson nwsp = DPCPU_PTR(nws); 969193219Srwatson npwp = &nwsp->nws_work[proto]; 970193219Srwatson npwp->nw_dispatched++; 971193219Srwatson npwp->nw_handled++; 972204497Srwatson netisr_proto[proto].np_handler(m); 973193219Srwatson error = 0; 974193219Srwatson goto out_unlock; 975112011Sjlemon } 976193219Srwatson 977222249Srwatson KASSERT(dispatch_policy == NETISR_DISPATCH_HYBRID, 978222249Srwatson ("%s: unknown dispatch policy (%u)", __func__, dispatch_policy)); 979222249Srwatson 980193219Srwatson /* 981193219Srwatson * Otherwise, we execute in a hybrid mode where we will try to direct 982193219Srwatson * dispatch if we're on the right CPU and the netisr worker isn't 983193219Srwatson * already running. 984193219Srwatson */ 985222249Srwatson sched_pin(); 986222249Srwatson m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_HYBRID, 987222249Srwatson source, m, &cpuid); 988193219Srwatson if (m == NULL) { 989193219Srwatson error = ENOBUFS; 990222249Srwatson goto out_unpin; 991193219Srwatson } 992195019Srwatson KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 993193219Srwatson if (cpuid != curcpu) 994193219Srwatson goto queue_fallback; 995195019Srwatson nwsp = DPCPU_PTR(nws); 996193219Srwatson npwp = &nwsp->nws_work[proto]; 997193219Srwatson 998193219Srwatson /*- 999193219Srwatson * We are willing to direct dispatch only if three conditions hold: 1000193219Srwatson * 1001193219Srwatson * (1) The netisr worker isn't already running, 1002193219Srwatson * (2) Another thread isn't already directly dispatching, and 1003193219Srwatson * (3) The netisr hasn't already been woken up. 1004193219Srwatson */ 1005193219Srwatson NWS_LOCK(nwsp); 1006193219Srwatson if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) { 1007193219Srwatson error = netisr_queue_workstream(nwsp, proto, npwp, m, 1008193219Srwatson &dosignal); 1009195019Srwatson NWS_UNLOCK(nwsp); 1010193219Srwatson if (dosignal) 1011193219Srwatson NWS_SIGNAL(nwsp); 1012193219Srwatson goto out_unpin; 1013193219Srwatson } 1014193219Srwatson 1015193219Srwatson /* 1016193219Srwatson * The current thread is now effectively the netisr worker, so set 1017193219Srwatson * the dispatching flag to prevent concurrent processing of the 1018193219Srwatson * stream from another thread (even the netisr worker), which could 1019193219Srwatson * otherwise lead to effective misordering of the stream. 1020193219Srwatson */ 1021193219Srwatson nwsp->nws_flags |= NWS_DISPATCHING; 1022193219Srwatson NWS_UNLOCK(nwsp); 1023204497Srwatson netisr_proto[proto].np_handler(m); 1024193219Srwatson NWS_LOCK(nwsp); 1025193219Srwatson nwsp->nws_flags &= ~NWS_DISPATCHING; 1026193219Srwatson npwp->nw_handled++; 1027193219Srwatson npwp->nw_hybrid_dispatched++; 1028193219Srwatson 1029193219Srwatson /* 1030193219Srwatson * If other work was enqueued by another thread while we were direct 1031193219Srwatson * dispatching, we need to signal the netisr worker to do that work. 1032193219Srwatson * In the future, we might want to do some of that work in the 1033193219Srwatson * current thread, rather than trigger further context switches. If 1034193219Srwatson * so, we'll want to establish a reasonable bound on the work done in 1035193219Srwatson * the "borrowed" context. 1036193219Srwatson */ 1037193219Srwatson if (nwsp->nws_pendingbits != 0) { 1038193219Srwatson nwsp->nws_flags |= NWS_SCHEDULED; 1039193219Srwatson dosignal = 1; 1040193219Srwatson } else 1041193219Srwatson dosignal = 0; 1042193219Srwatson NWS_UNLOCK(nwsp); 1043193219Srwatson if (dosignal) 1044193219Srwatson NWS_SIGNAL(nwsp); 1045193219Srwatson error = 0; 1046193219Srwatson goto out_unpin; 1047193219Srwatson 1048193219Srwatsonqueue_fallback: 1049193219Srwatson error = netisr_queue_internal(proto, m, cpuid); 1050193219Srwatsonout_unpin: 1051193219Srwatson sched_unpin(); 1052193219Srwatsonout_unlock: 1053193219Srwatson#ifdef NETISR_LOCKING 1054193219Srwatson NETISR_RUNLOCK(&tracker); 1055193219Srwatson#endif 1056193219Srwatson return (error); 1057111888Sjlemon} 1058103781Sjake 1059193219Srwatsonint 1060193219Srwatsonnetisr_dispatch(u_int proto, struct mbuf *m) 1061193219Srwatson{ 1062193219Srwatson 1063193219Srwatson return (netisr_dispatch_src(proto, 0, m)); 1064193219Srwatson} 1065193219Srwatson 1066193219Srwatson#ifdef DEVICE_POLLING 1067193219Srwatson/* 1068193219Srwatson * Kernel polling borrows a netisr thread to run interface polling in; this 1069193219Srwatson * function allows kernel polling to request that the netisr thread be 1070193219Srwatson * scheduled even if no packets are pending for protocols. 1071193219Srwatson */ 1072193219Srwatsonvoid 1073193219Srwatsonnetisr_sched_poll(void) 1074193219Srwatson{ 1075193219Srwatson struct netisr_workstream *nwsp; 1076193219Srwatson 1077195019Srwatson nwsp = DPCPU_ID_PTR(nws_array[0], nws); 1078193219Srwatson NWS_SIGNAL(nwsp); 1079193219Srwatson} 1080193219Srwatson#endif 1081193219Srwatson 1082103781Sjakestatic void 1083193219Srwatsonnetisr_start_swi(u_int cpuid, struct pcpu *pc) 1084103781Sjake{ 1085193219Srwatson char swiname[12]; 1086193219Srwatson struct netisr_workstream *nwsp; 1087193219Srwatson int error; 1088193219Srwatson 1089195019Srwatson KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); 1090195019Srwatson 1091195019Srwatson nwsp = DPCPU_ID_PTR(cpuid, nws); 1092193219Srwatson mtx_init(&nwsp->nws_mtx, "netisr_mtx", NULL, MTX_DEF); 1093193219Srwatson nwsp->nws_cpu = cpuid; 1094193219Srwatson snprintf(swiname, sizeof(swiname), "netisr %u", cpuid); 1095193219Srwatson error = swi_add(&nwsp->nws_intr_event, swiname, swi_net, nwsp, 1096193219Srwatson SWI_NET, INTR_MPSAFE, &nwsp->nws_swi_cookie); 1097193219Srwatson if (error) 1098193219Srwatson panic("%s: swi_add %d", __func__, error); 1099193219Srwatson pc->pc_netisr = nwsp->nws_intr_event; 1100193219Srwatson if (netisr_bindthreads) { 1101193219Srwatson error = intr_event_bind(nwsp->nws_intr_event, cpuid); 1102193219Srwatson if (error != 0) 1103193219Srwatson printf("%s: cpu %u: intr_event_bind: %d", __func__, 1104193219Srwatson cpuid, error); 1105193219Srwatson } 1106193219Srwatson NETISR_WLOCK(); 1107193219Srwatson nws_array[nws_count] = nwsp->nws_cpu; 1108193219Srwatson nws_count++; 1109193219Srwatson NETISR_WUNLOCK(); 1110193219Srwatson} 1111193219Srwatson 1112193219Srwatson/* 1113193219Srwatson * Initialize the netisr subsystem. We rely on BSS and static initialization 1114193219Srwatson * of most fields in global data structures. 1115193219Srwatson * 1116193219Srwatson * Start a worker thread for the boot CPU so that we can support network 1117193219Srwatson * traffic immediately in case the network stack is used before additional 1118193219Srwatson * CPUs are started (for example, diskless boot). 1119193219Srwatson */ 1120193219Srwatsonstatic void 1121193219Srwatsonnetisr_init(void *arg) 1122193219Srwatson{ 1123222249Srwatson char tmp[NETISR_DISPATCH_POLICY_MAXSTR]; 1124222249Srwatson u_int dispatch_policy; 1125222249Srwatson int error; 1126193219Srwatson 1127193219Srwatson KASSERT(curcpu == 0, ("%s: not on CPU 0", __func__)); 1128193219Srwatson 1129193219Srwatson NETISR_LOCK_INIT(); 1130195078Srwatson if (netisr_maxthreads < 1) 1131193219Srwatson netisr_maxthreads = 1; 1132195078Srwatson if (netisr_maxthreads > mp_ncpus) { 1133200899Srwatson printf("netisr_init: forcing maxthreads from %d to %d\n", 1134195078Srwatson netisr_maxthreads, mp_ncpus); 1135195078Srwatson netisr_maxthreads = mp_ncpus; 1136193230Srwatson } 1137193230Srwatson if (netisr_defaultqlimit > netisr_maxqlimit) { 1138200899Srwatson printf("netisr_init: forcing defaultqlimit from %d to %d\n", 1139195078Srwatson netisr_defaultqlimit, netisr_maxqlimit); 1140193219Srwatson netisr_defaultqlimit = netisr_maxqlimit; 1141193230Srwatson } 1142103781Sjake#ifdef DEVICE_POLLING 1143193219Srwatson /* 1144193219Srwatson * The device polling code is not yet aware of how to deal with 1145193219Srwatson * multiple netisr threads, so for the time being compiling in device 1146193219Srwatson * polling disables parallel netisr workers. 1147193219Srwatson */ 1148193230Srwatson if (netisr_maxthreads != 1 || netisr_bindthreads != 0) { 1149200899Srwatson printf("netisr_init: forcing maxthreads to 1 and " 1150200899Srwatson "bindthreads to 0 for device polling\n"); 1151193230Srwatson netisr_maxthreads = 1; 1152193230Srwatson netisr_bindthreads = 0; 1153193230Srwatson } 1154103781Sjake#endif 1155111888Sjlemon 1156222249Srwatson if (TUNABLE_STR_FETCH("net.isr.dispatch", tmp, sizeof(tmp))) { 1157222249Srwatson error = netisr_dispatch_policy_from_str(tmp, 1158222249Srwatson &dispatch_policy); 1159222249Srwatson if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT) 1160222249Srwatson error = EINVAL; 1161255329Sdavide if (error == 0) 1162222249Srwatson netisr_dispatch_policy = dispatch_policy; 1163255329Sdavide else 1164222249Srwatson printf( 1165222249Srwatson "%s: invalid dispatch policy %s, using default\n", 1166222249Srwatson __func__, tmp); 1167222249Srwatson } 1168222249Srwatson 1169193219Srwatson netisr_start_swi(curcpu, pcpu_find(curcpu)); 1170103781Sjake} 1171193219SrwatsonSYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL); 1172103781Sjake 1173193219Srwatson/* 1174193219Srwatson * Start worker threads for additional CPUs. No attempt to gracefully handle 1175193219Srwatson * work reassignment, we don't yet support dynamic reconfiguration. 1176193219Srwatson */ 1177103781Sjakestatic void 1178193219Srwatsonnetisr_start(void *arg) 1179103781Sjake{ 1180193219Srwatson struct pcpu *pc; 1181103781Sjake 1182222531Snwhitehorn STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 1183193219Srwatson if (nws_count >= netisr_maxthreads) 1184193219Srwatson break; 1185193219Srwatson /* XXXRW: Is skipping absent CPUs still required here? */ 1186193219Srwatson if (CPU_ABSENT(pc->pc_cpuid)) 1187193219Srwatson continue; 1188193219Srwatson /* Worker will already be present for boot CPU. */ 1189193219Srwatson if (pc->pc_netisr != NULL) 1190193219Srwatson continue; 1191193219Srwatson netisr_start_swi(pc->pc_cpuid, pc); 1192193219Srwatson } 1193103781Sjake} 1194193219SrwatsonSYSINIT(netisr_start, SI_SUB_SMP, SI_ORDER_MIDDLE, netisr_start, NULL); 1195193219Srwatson 1196204199Srwatson/* 1197204199Srwatson * Sysctl monitoring for netisr: query a list of registered protocols. 1198204199Srwatson */ 1199204199Srwatsonstatic int 1200204199Srwatsonsysctl_netisr_proto(SYSCTL_HANDLER_ARGS) 1201204199Srwatson{ 1202204199Srwatson struct rm_priotracker tracker; 1203204199Srwatson struct sysctl_netisr_proto *snpp, *snp_array; 1204204199Srwatson struct netisr_proto *npp; 1205204199Srwatson u_int counter, proto; 1206204199Srwatson int error; 1207204199Srwatson 1208204199Srwatson if (req->newptr != NULL) 1209204199Srwatson return (EINVAL); 1210204199Srwatson snp_array = malloc(sizeof(*snp_array) * NETISR_MAXPROT, M_TEMP, 1211204199Srwatson M_ZERO | M_WAITOK); 1212204199Srwatson counter = 0; 1213204199Srwatson NETISR_RLOCK(&tracker); 1214204199Srwatson for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1215204497Srwatson npp = &netisr_proto[proto]; 1216204199Srwatson if (npp->np_name == NULL) 1217204199Srwatson continue; 1218204199Srwatson snpp = &snp_array[counter]; 1219204199Srwatson snpp->snp_version = sizeof(*snpp); 1220204199Srwatson strlcpy(snpp->snp_name, npp->np_name, NETISR_NAMEMAXLEN); 1221204199Srwatson snpp->snp_proto = proto; 1222204199Srwatson snpp->snp_qlimit = npp->np_qlimit; 1223204199Srwatson snpp->snp_policy = npp->np_policy; 1224222249Srwatson snpp->snp_dispatch = npp->np_dispatch; 1225204199Srwatson if (npp->np_m2flow != NULL) 1226204199Srwatson snpp->snp_flags |= NETISR_SNP_FLAGS_M2FLOW; 1227204199Srwatson if (npp->np_m2cpuid != NULL) 1228204199Srwatson snpp->snp_flags |= NETISR_SNP_FLAGS_M2CPUID; 1229204199Srwatson if (npp->np_drainedcpu != NULL) 1230204199Srwatson snpp->snp_flags |= NETISR_SNP_FLAGS_DRAINEDCPU; 1231204199Srwatson counter++; 1232204199Srwatson } 1233204199Srwatson NETISR_RUNLOCK(&tracker); 1234204303Srwatson KASSERT(counter <= NETISR_MAXPROT, 1235204199Srwatson ("sysctl_netisr_proto: counter too big (%d)", counter)); 1236204199Srwatson error = SYSCTL_OUT(req, snp_array, sizeof(*snp_array) * counter); 1237204199Srwatson free(snp_array, M_TEMP); 1238204199Srwatson return (error); 1239204199Srwatson} 1240204199Srwatson 1241204199SrwatsonSYSCTL_PROC(_net_isr, OID_AUTO, proto, 1242204199Srwatson CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_proto, 1243204199Srwatson "S,sysctl_netisr_proto", 1244204199Srwatson "Return list of protocols registered with netisr"); 1245204199Srwatson 1246204199Srwatson/* 1247204199Srwatson * Sysctl monitoring for netisr: query a list of workstreams. 1248204199Srwatson */ 1249204199Srwatsonstatic int 1250204199Srwatsonsysctl_netisr_workstream(SYSCTL_HANDLER_ARGS) 1251204199Srwatson{ 1252204199Srwatson struct rm_priotracker tracker; 1253204199Srwatson struct sysctl_netisr_workstream *snwsp, *snws_array; 1254204199Srwatson struct netisr_workstream *nwsp; 1255204199Srwatson u_int counter, cpuid; 1256204199Srwatson int error; 1257204199Srwatson 1258204199Srwatson if (req->newptr != NULL) 1259204199Srwatson return (EINVAL); 1260204199Srwatson snws_array = malloc(sizeof(*snws_array) * MAXCPU, M_TEMP, 1261204199Srwatson M_ZERO | M_WAITOK); 1262204199Srwatson counter = 0; 1263204199Srwatson NETISR_RLOCK(&tracker); 1264209059Sjhb CPU_FOREACH(cpuid) { 1265204199Srwatson nwsp = DPCPU_ID_PTR(cpuid, nws); 1266204199Srwatson if (nwsp->nws_intr_event == NULL) 1267204199Srwatson continue; 1268204199Srwatson NWS_LOCK(nwsp); 1269204199Srwatson snwsp = &snws_array[counter]; 1270204199Srwatson snwsp->snws_version = sizeof(*snwsp); 1271204199Srwatson 1272204199Srwatson /* 1273204199Srwatson * For now, we equate workstream IDs and CPU IDs in the 1274204199Srwatson * kernel, but expose them independently to userspace in case 1275204199Srwatson * that assumption changes in the future. 1276204199Srwatson */ 1277204199Srwatson snwsp->snws_wsid = cpuid; 1278204199Srwatson snwsp->snws_cpu = cpuid; 1279204199Srwatson if (nwsp->nws_intr_event != NULL) 1280204199Srwatson snwsp->snws_flags |= NETISR_SNWS_FLAGS_INTR; 1281204199Srwatson NWS_UNLOCK(nwsp); 1282204199Srwatson counter++; 1283204199Srwatson } 1284204199Srwatson NETISR_RUNLOCK(&tracker); 1285204303Srwatson KASSERT(counter <= MAXCPU, 1286204199Srwatson ("sysctl_netisr_workstream: counter too big (%d)", counter)); 1287204199Srwatson error = SYSCTL_OUT(req, snws_array, sizeof(*snws_array) * counter); 1288204199Srwatson free(snws_array, M_TEMP); 1289204199Srwatson return (error); 1290204199Srwatson} 1291204199Srwatson 1292204199SrwatsonSYSCTL_PROC(_net_isr, OID_AUTO, workstream, 1293204199Srwatson CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_workstream, 1294204199Srwatson "S,sysctl_netisr_workstream", 1295204199Srwatson "Return list of workstreams implemented by netisr"); 1296204199Srwatson 1297204199Srwatson/* 1298204199Srwatson * Sysctl monitoring for netisr: query per-protocol data across all 1299204199Srwatson * workstreams. 1300204199Srwatson */ 1301204199Srwatsonstatic int 1302204199Srwatsonsysctl_netisr_work(SYSCTL_HANDLER_ARGS) 1303204199Srwatson{ 1304204199Srwatson struct rm_priotracker tracker; 1305204199Srwatson struct sysctl_netisr_work *snwp, *snw_array; 1306204199Srwatson struct netisr_workstream *nwsp; 1307204199Srwatson struct netisr_proto *npp; 1308204199Srwatson struct netisr_work *nwp; 1309204199Srwatson u_int counter, cpuid, proto; 1310204199Srwatson int error; 1311204199Srwatson 1312204199Srwatson if (req->newptr != NULL) 1313204199Srwatson return (EINVAL); 1314204199Srwatson snw_array = malloc(sizeof(*snw_array) * MAXCPU * NETISR_MAXPROT, 1315204199Srwatson M_TEMP, M_ZERO | M_WAITOK); 1316204199Srwatson counter = 0; 1317204199Srwatson NETISR_RLOCK(&tracker); 1318209059Sjhb CPU_FOREACH(cpuid) { 1319204199Srwatson nwsp = DPCPU_ID_PTR(cpuid, nws); 1320204199Srwatson if (nwsp->nws_intr_event == NULL) 1321204199Srwatson continue; 1322204199Srwatson NWS_LOCK(nwsp); 1323204199Srwatson for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1324204497Srwatson npp = &netisr_proto[proto]; 1325204199Srwatson if (npp->np_name == NULL) 1326204199Srwatson continue; 1327204199Srwatson nwp = &nwsp->nws_work[proto]; 1328204199Srwatson snwp = &snw_array[counter]; 1329204199Srwatson snwp->snw_version = sizeof(*snwp); 1330204199Srwatson snwp->snw_wsid = cpuid; /* See comment above. */ 1331204199Srwatson snwp->snw_proto = proto; 1332204199Srwatson snwp->snw_len = nwp->nw_len; 1333204199Srwatson snwp->snw_watermark = nwp->nw_watermark; 1334204199Srwatson snwp->snw_dispatched = nwp->nw_dispatched; 1335204199Srwatson snwp->snw_hybrid_dispatched = 1336204199Srwatson nwp->nw_hybrid_dispatched; 1337204199Srwatson snwp->snw_qdrops = nwp->nw_qdrops; 1338204199Srwatson snwp->snw_queued = nwp->nw_queued; 1339204199Srwatson snwp->snw_handled = nwp->nw_handled; 1340204199Srwatson counter++; 1341204199Srwatson } 1342204199Srwatson NWS_UNLOCK(nwsp); 1343204199Srwatson } 1344204303Srwatson KASSERT(counter <= MAXCPU * NETISR_MAXPROT, 1345204199Srwatson ("sysctl_netisr_work: counter too big (%d)", counter)); 1346204199Srwatson NETISR_RUNLOCK(&tracker); 1347204199Srwatson error = SYSCTL_OUT(req, snw_array, sizeof(*snw_array) * counter); 1348204199Srwatson free(snw_array, M_TEMP); 1349204199Srwatson return (error); 1350204199Srwatson} 1351204199Srwatson 1352204199SrwatsonSYSCTL_PROC(_net_isr, OID_AUTO, work, 1353204199Srwatson CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_work, 1354204199Srwatson "S,sysctl_netisr_work", 1355204199Srwatson "Return list of per-workstream, per-protocol work in netisr"); 1356204199Srwatson 1357193219Srwatson#ifdef DDB 1358193219SrwatsonDB_SHOW_COMMAND(netisr, db_show_netisr) 1359193219Srwatson{ 1360193219Srwatson struct netisr_workstream *nwsp; 1361193219Srwatson struct netisr_work *nwp; 1362193219Srwatson int first, proto; 1363195019Srwatson u_int cpuid; 1364193219Srwatson 1365193219Srwatson db_printf("%3s %6s %5s %5s %5s %8s %8s %8s %8s\n", "CPU", "Proto", 1366193219Srwatson "Len", "WMark", "Max", "Disp", "HDisp", "Drop", "Queue"); 1367209059Sjhb CPU_FOREACH(cpuid) { 1368195019Srwatson nwsp = DPCPU_ID_PTR(cpuid, nws); 1369193219Srwatson if (nwsp->nws_intr_event == NULL) 1370193219Srwatson continue; 1371193219Srwatson first = 1; 1372193219Srwatson for (proto = 0; proto < NETISR_MAXPROT; proto++) { 1373204497Srwatson if (netisr_proto[proto].np_handler == NULL) 1374193219Srwatson continue; 1375193219Srwatson nwp = &nwsp->nws_work[proto]; 1376193219Srwatson if (first) { 1377195019Srwatson db_printf("%3d ", cpuid); 1378193219Srwatson first = 0; 1379193219Srwatson } else 1380193219Srwatson db_printf("%3s ", ""); 1381193219Srwatson db_printf( 1382193219Srwatson "%6s %5d %5d %5d %8ju %8ju %8ju %8ju\n", 1383204497Srwatson netisr_proto[proto].np_name, nwp->nw_len, 1384193219Srwatson nwp->nw_watermark, nwp->nw_qlimit, 1385193219Srwatson nwp->nw_dispatched, nwp->nw_hybrid_dispatched, 1386193219Srwatson nwp->nw_qdrops, nwp->nw_queued); 1387193219Srwatson } 1388193219Srwatson } 1389193219Srwatson} 1390193219Srwatson#endif 1391