1222748Srwatson/*- 2222748Srwatson * Copyright (c) 2010-2011 Juniper Networks, Inc. 3222748Srwatson * All rights reserved. 4222748Srwatson * 5222748Srwatson * This software was developed by Robert N. M. Watson under contract 6222748Srwatson * to Juniper Networks, Inc. 7222748Srwatson * 8222748Srwatson * Redistribution and use in source and binary forms, with or without 9222748Srwatson * modification, are permitted provided that the following conditions 10222748Srwatson * are met: 11222748Srwatson * 1. Redistributions of source code must retain the above copyright 12222748Srwatson * notice, this list of conditions and the following disclaimer. 13222748Srwatson * 2. Redistributions in binary form must reproduce the above copyright 14222748Srwatson * notice, this list of conditions and the following disclaimer in the 15222748Srwatson * documentation and/or other materials provided with the distribution. 16222748Srwatson * 17222748Srwatson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18222748Srwatson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19222748Srwatson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20222748Srwatson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21222748Srwatson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22222748Srwatson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23222748Srwatson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24222748Srwatson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25222748Srwatson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26222748Srwatson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27222748Srwatson * SUCH DAMAGE. 28222748Srwatson */ 29222748Srwatson 30222748Srwatson#include <sys/cdefs.h> 31222748Srwatson 32222748Srwatson__FBSDID("$FreeBSD$"); 33222748Srwatson 34222748Srwatson#include "opt_inet6.h" 35222748Srwatson 36222748Srwatson#include <sys/param.h> 37222748Srwatson#include <sys/lock.h> 38222748Srwatson#include <sys/malloc.h> 39222748Srwatson#include <sys/mbuf.h> 40222748Srwatson#include <sys/mutex.h> 41222748Srwatson#include <sys/smp.h> 42222748Srwatson#include <sys/socketvar.h> 43222748Srwatson 44222748Srwatson#include <netinet/in.h> 45222748Srwatson#include <netinet/in_pcb.h> 46222748Srwatson#ifdef INET6 47222748Srwatson#include <netinet6/in6_pcb.h> 48222748Srwatson#endif /* INET6 */ 49222748Srwatson 50222748Srwatson/* 51222748Srwatson * pcbgroups, or "connection groups" are based on Willman, Rixner, and Cox's 52222748Srwatson * 2006 USENIX paper, "An Evaluation of Network Stack Parallelization 53222748Srwatson * Strategies in Modern Operating Systems". This implementation differs 54222748Srwatson * significantly from that described in the paper, in that it attempts to 55222748Srwatson * introduce not just notions of affinity for connections and distribute work 56222748Srwatson * so as to reduce lock contention, but also align those notions with 57222748Srwatson * hardware work distribution strategies such as RSS. In this construction, 58222748Srwatson * connection groups supplement, rather than replace, existing reservation 59222748Srwatson * tables for protocol 4-tuples, offering CPU-affine lookup tables with 60222748Srwatson * minimal cache line migration and lock contention during steady state 61222748Srwatson * operation. 62222748Srwatson * 63222748Srwatson * Internet protocols, such as UDP and TCP, register to use connection groups 64222748Srwatson * by providing an ipi_hashfields value other than IPI_HASHFIELDS_NONE; this 65222748Srwatson * indicates to the connection group code whether a 2-tuple or 4-tuple is 66222748Srwatson * used as an argument to hashes that assign a connection to a particular 67222748Srwatson * group. This must be aligned with any hardware offloaded distribution 68222748Srwatson * model, such as RSS or similar approaches taken in embedded network boards. 69222748Srwatson * Wildcard sockets require special handling, as in Willman 2006, and are 70222748Srwatson * shared between connection groups -- while being protected by group-local 71222748Srwatson * locks. This means that connection establishment and teardown can be 72222748Srwatson * signficantly more expensive than without connection groups, but that 73222748Srwatson * steady-state processing can be significantly faster. 74222748Srwatson * 75222748Srwatson * Most of the implementation of connection groups is in this file; however, 76222748Srwatson * connection group lookup is implemented in in_pcb.c alongside reservation 77222748Srwatson * table lookups -- see in_pcblookup_group(). 78222748Srwatson * 79222748Srwatson * TODO: 80222748Srwatson * 81222748Srwatson * Implement dynamic rebalancing of buckets with connection groups; when 82222748Srwatson * load is unevenly distributed, search for more optimal balancing on 83222748Srwatson * demand. This might require scaling up the number of connection groups 84222748Srwatson * by <<1. 85222748Srwatson * 86222748Srwatson * Provide an IP 2-tuple or 4-tuple netisr m2cpu handler based on connection 87222748Srwatson * groups for ip_input and ip6_input, allowing non-offloaded work 88222748Srwatson * distribution. 89222748Srwatson * 90222748Srwatson * Expose effective CPU affinity of connections to userspace using socket 91222748Srwatson * options. 92222748Srwatson * 93222748Srwatson * Investigate per-connection affinity overrides based on socket options; an 94222748Srwatson * option could be set, certainly resulting in work being distributed 95222748Srwatson * differently in software, and possibly propagated to supporting hardware 96222748Srwatson * with TCAMs or hardware hash tables. This might require connections to 97222748Srwatson * exist in more than one connection group at a time. 98222748Srwatson * 99222748Srwatson * Hook netisr thread reconfiguration events, and propagate those to RSS so 100222748Srwatson * that rebalancing can occur when the thread pool grows or shrinks. 101222748Srwatson * 102222748Srwatson * Expose per-pcbgroup statistics to userspace monitoring tools such as 103222748Srwatson * netstat, in order to allow better debugging and profiling. 104222748Srwatson */ 105222748Srwatson 106222748Srwatsonvoid 107222748Srwatsonin_pcbgroup_init(struct inpcbinfo *pcbinfo, u_int hashfields, 108222748Srwatson int hash_nelements) 109222748Srwatson{ 110222748Srwatson struct inpcbgroup *pcbgroup; 111222748Srwatson u_int numpcbgroups, pgn; 112222748Srwatson 113222748Srwatson /* 114222748Srwatson * Only enable connection groups for a protocol if it has been 115222748Srwatson * specifically requested. 116222748Srwatson */ 117222748Srwatson if (hashfields == IPI_HASHFIELDS_NONE) 118222748Srwatson return; 119222748Srwatson 120222748Srwatson /* 121222748Srwatson * Connection groups are about multi-processor load distribution, 122222748Srwatson * lock contention, and connection CPU affinity. As such, no point 123222748Srwatson * in turning them on for a uniprocessor machine, it only wastes 124222748Srwatson * memory. 125222748Srwatson */ 126222748Srwatson if (mp_ncpus == 1) 127222748Srwatson return; 128222748Srwatson 129222748Srwatson /* 130222748Srwatson * Use one group per CPU for now. If we decide to do dynamic 131222748Srwatson * rebalancing a la RSS, we'll need to shift left by at least 1. 132222748Srwatson */ 133222748Srwatson numpcbgroups = mp_ncpus; 134222748Srwatson 135222748Srwatson pcbinfo->ipi_hashfields = hashfields; 136222748Srwatson pcbinfo->ipi_pcbgroups = malloc(numpcbgroups * 137222748Srwatson sizeof(*pcbinfo->ipi_pcbgroups), M_PCB, M_WAITOK | M_ZERO); 138222748Srwatson pcbinfo->ipi_npcbgroups = numpcbgroups; 139222748Srwatson pcbinfo->ipi_wildbase = hashinit(hash_nelements, M_PCB, 140222748Srwatson &pcbinfo->ipi_wildmask); 141222748Srwatson for (pgn = 0; pgn < pcbinfo->ipi_npcbgroups; pgn++) { 142222748Srwatson pcbgroup = &pcbinfo->ipi_pcbgroups[pgn]; 143222748Srwatson pcbgroup->ipg_hashbase = hashinit(hash_nelements, M_PCB, 144222748Srwatson &pcbgroup->ipg_hashmask); 145222748Srwatson INP_GROUP_LOCK_INIT(pcbgroup, "pcbgroup"); 146222748Srwatson 147222748Srwatson /* 148222748Srwatson * Initialise notional affinity of the pcbgroup -- for RSS, 149222748Srwatson * we want the same notion of affinity as NICs to be used. 150222748Srwatson * Just round robin for the time being. 151222748Srwatson */ 152222748Srwatson pcbgroup->ipg_cpu = (pgn % mp_ncpus); 153222748Srwatson } 154222748Srwatson} 155222748Srwatson 156222748Srwatsonvoid 157222748Srwatsonin_pcbgroup_destroy(struct inpcbinfo *pcbinfo) 158222748Srwatson{ 159222748Srwatson struct inpcbgroup *pcbgroup; 160222748Srwatson u_int pgn; 161222748Srwatson 162222748Srwatson if (pcbinfo->ipi_npcbgroups == 0) 163222748Srwatson return; 164222748Srwatson 165222748Srwatson for (pgn = 0; pgn < pcbinfo->ipi_npcbgroups; pgn++) { 166222748Srwatson pcbgroup = &pcbinfo->ipi_pcbgroups[pgn]; 167222748Srwatson KASSERT(LIST_EMPTY(pcbinfo->ipi_listhead), 168222748Srwatson ("in_pcbinfo_destroy: listhead not empty")); 169222748Srwatson INP_GROUP_LOCK_DESTROY(pcbgroup); 170222748Srwatson hashdestroy(pcbgroup->ipg_hashbase, M_PCB, 171222748Srwatson pcbgroup->ipg_hashmask); 172222748Srwatson } 173222748Srwatson hashdestroy(pcbinfo->ipi_wildbase, M_PCB, pcbinfo->ipi_wildmask); 174222748Srwatson free(pcbinfo->ipi_pcbgroups, M_PCB); 175222748Srwatson pcbinfo->ipi_pcbgroups = NULL; 176222748Srwatson pcbinfo->ipi_npcbgroups = 0; 177222748Srwatson pcbinfo->ipi_hashfields = 0; 178222748Srwatson} 179222748Srwatson 180222748Srwatson/* 181222748Srwatson * Given a hash of whatever the covered tuple might be, return a pcbgroup 182222748Srwatson * index. 183222748Srwatson */ 184222748Srwatsonstatic __inline u_int 185222748Srwatsonin_pcbgroup_getbucket(struct inpcbinfo *pcbinfo, uint32_t hash) 186222748Srwatson{ 187222748Srwatson 188222748Srwatson return (hash % pcbinfo->ipi_npcbgroups); 189222748Srwatson} 190222748Srwatson 191222748Srwatson/* 192222748Srwatson * Map a (hashtype, hash) tuple into a connection group, or NULL if the hash 193222748Srwatson * information is insufficient to identify the pcbgroup. 194222748Srwatson */ 195222748Srwatsonstruct inpcbgroup * 196222748Srwatsonin_pcbgroup_byhash(struct inpcbinfo *pcbinfo, u_int hashtype, uint32_t hash) 197222748Srwatson{ 198222748Srwatson 199222748Srwatson return (NULL); 200222748Srwatson} 201222748Srwatson 202222748Srwatsonstatic struct inpcbgroup * 203222748Srwatsonin_pcbgroup_bymbuf(struct inpcbinfo *pcbinfo, struct mbuf *m) 204222748Srwatson{ 205222748Srwatson 206222748Srwatson return (in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m), 207222748Srwatson m->m_pkthdr.flowid)); 208222748Srwatson} 209222748Srwatson 210222748Srwatsonstruct inpcbgroup * 211222748Srwatsonin_pcbgroup_bytuple(struct inpcbinfo *pcbinfo, struct in_addr laddr, 212222748Srwatson u_short lport, struct in_addr faddr, u_short fport) 213222748Srwatson{ 214222748Srwatson uint32_t hash; 215222748Srwatson 216222748Srwatson switch (pcbinfo->ipi_hashfields) { 217222748Srwatson case IPI_HASHFIELDS_4TUPLE: 218222748Srwatson hash = faddr.s_addr ^ fport; 219222748Srwatson break; 220222748Srwatson 221222748Srwatson case IPI_HASHFIELDS_2TUPLE: 222222748Srwatson hash = faddr.s_addr ^ laddr.s_addr; 223222748Srwatson break; 224222748Srwatson 225222748Srwatson default: 226222748Srwatson hash = 0; 227222748Srwatson } 228222748Srwatson return (&pcbinfo->ipi_pcbgroups[in_pcbgroup_getbucket(pcbinfo, 229222748Srwatson hash)]); 230222748Srwatson} 231222748Srwatson 232222748Srwatsonstruct inpcbgroup * 233222748Srwatsonin_pcbgroup_byinpcb(struct inpcb *inp) 234222748Srwatson{ 235222748Srwatson 236222748Srwatson return (in_pcbgroup_bytuple(inp->inp_pcbinfo, inp->inp_laddr, 237222748Srwatson inp->inp_lport, inp->inp_faddr, inp->inp_fport)); 238222748Srwatson} 239222748Srwatson 240222748Srwatsonstatic void 241222748Srwatsonin_pcbwild_add(struct inpcb *inp) 242222748Srwatson{ 243222748Srwatson struct inpcbinfo *pcbinfo; 244222748Srwatson struct inpcbhead *head; 245222748Srwatson u_int pgn; 246222748Srwatson 247222748Srwatson INP_WLOCK_ASSERT(inp); 248222748Srwatson KASSERT(!(inp->inp_flags2 & INP_PCBGROUPWILD), 249222748Srwatson ("%s: is wild",__func__)); 250222748Srwatson 251222748Srwatson pcbinfo = inp->inp_pcbinfo; 252222748Srwatson for (pgn = 0; pgn < pcbinfo->ipi_npcbgroups; pgn++) 253222748Srwatson INP_GROUP_LOCK(&pcbinfo->ipi_pcbgroups[pgn]); 254222748Srwatson head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, inp->inp_lport, 255222748Srwatson 0, pcbinfo->ipi_wildmask)]; 256222748Srwatson LIST_INSERT_HEAD(head, inp, inp_pcbgroup_wild); 257222748Srwatson inp->inp_flags2 |= INP_PCBGROUPWILD; 258222748Srwatson for (pgn = 0; pgn < pcbinfo->ipi_npcbgroups; pgn++) 259222748Srwatson INP_GROUP_UNLOCK(&pcbinfo->ipi_pcbgroups[pgn]); 260222748Srwatson} 261222748Srwatson 262222748Srwatsonstatic void 263222748Srwatsonin_pcbwild_remove(struct inpcb *inp) 264222748Srwatson{ 265222748Srwatson struct inpcbinfo *pcbinfo; 266222748Srwatson u_int pgn; 267222748Srwatson 268222748Srwatson INP_WLOCK_ASSERT(inp); 269222748Srwatson KASSERT((inp->inp_flags2 & INP_PCBGROUPWILD), 270222748Srwatson ("%s: not wild", __func__)); 271222748Srwatson 272222748Srwatson pcbinfo = inp->inp_pcbinfo; 273222748Srwatson for (pgn = 0; pgn < pcbinfo->ipi_npcbgroups; pgn++) 274222748Srwatson INP_GROUP_LOCK(&pcbinfo->ipi_pcbgroups[pgn]); 275222748Srwatson LIST_REMOVE(inp, inp_pcbgroup_wild); 276222748Srwatson for (pgn = 0; pgn < pcbinfo->ipi_npcbgroups; pgn++) 277222748Srwatson INP_GROUP_UNLOCK(&pcbinfo->ipi_pcbgroups[pgn]); 278222748Srwatson inp->inp_flags2 &= ~INP_PCBGROUPWILD; 279222748Srwatson} 280222748Srwatson 281222748Srwatsonstatic __inline int 282222748Srwatsonin_pcbwild_needed(struct inpcb *inp) 283222748Srwatson{ 284222748Srwatson 285222748Srwatson#ifdef INET6 286222748Srwatson if (inp->inp_vflag & INP_IPV6) 287222748Srwatson return (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)); 288222748Srwatson else 289222748Srwatson#endif 290222748Srwatson return (inp->inp_faddr.s_addr == htonl(INADDR_ANY)); 291222748Srwatson} 292222748Srwatson 293222748Srwatsonstatic void 294222748Srwatsonin_pcbwild_update_internal(struct inpcb *inp) 295222748Srwatson{ 296222748Srwatson int wildcard_needed; 297222748Srwatson 298222748Srwatson wildcard_needed = in_pcbwild_needed(inp); 299222748Srwatson if (wildcard_needed && !(inp->inp_flags2 & INP_PCBGROUPWILD)) 300222748Srwatson in_pcbwild_add(inp); 301222748Srwatson else if (!wildcard_needed && (inp->inp_flags2 & INP_PCBGROUPWILD)) 302222748Srwatson in_pcbwild_remove(inp); 303222748Srwatson} 304222748Srwatson 305222748Srwatson/* 306222748Srwatson * Update the pcbgroup of an inpcb, which might include removing an old 307222748Srwatson * pcbgroup reference and/or adding a new one. Wildcard processing is not 308222748Srwatson * performed here, although ideally we'll never install a pcbgroup for a 309222748Srwatson * wildcard inpcb (asserted below). 310222748Srwatson */ 311222748Srwatsonstatic void 312222748Srwatsonin_pcbgroup_update_internal(struct inpcbinfo *pcbinfo, 313222748Srwatson struct inpcbgroup *newpcbgroup, struct inpcb *inp) 314222748Srwatson{ 315222748Srwatson struct inpcbgroup *oldpcbgroup; 316222748Srwatson struct inpcbhead *pcbhash; 317222748Srwatson uint32_t hashkey_faddr; 318222748Srwatson 319222748Srwatson INP_WLOCK_ASSERT(inp); 320222748Srwatson 321222748Srwatson oldpcbgroup = inp->inp_pcbgroup; 322222748Srwatson if (oldpcbgroup != NULL && oldpcbgroup != newpcbgroup) { 323222748Srwatson INP_GROUP_LOCK(oldpcbgroup); 324222748Srwatson LIST_REMOVE(inp, inp_pcbgrouphash); 325222748Srwatson inp->inp_pcbgroup = NULL; 326222748Srwatson INP_GROUP_UNLOCK(oldpcbgroup); 327222748Srwatson } 328222748Srwatson if (newpcbgroup != NULL && oldpcbgroup != newpcbgroup) { 329222748Srwatson#ifdef INET6 330222748Srwatson if (inp->inp_vflag & INP_IPV6) 331222748Srwatson hashkey_faddr = inp->in6p_faddr.s6_addr32[3]; /* XXX */ 332222748Srwatson else 333222748Srwatson#endif 334222748Srwatson hashkey_faddr = inp->inp_faddr.s_addr; 335222748Srwatson INP_GROUP_LOCK(newpcbgroup); 336222748Srwatson pcbhash = &newpcbgroup->ipg_hashbase[ 337222748Srwatson INP_PCBHASH(hashkey_faddr, inp->inp_lport, inp->inp_fport, 338222748Srwatson newpcbgroup->ipg_hashmask)]; 339222748Srwatson LIST_INSERT_HEAD(pcbhash, inp, inp_pcbgrouphash); 340222748Srwatson inp->inp_pcbgroup = newpcbgroup; 341222748Srwatson INP_GROUP_UNLOCK(newpcbgroup); 342222748Srwatson } 343222748Srwatson 344222748Srwatson KASSERT(!(newpcbgroup != NULL && in_pcbwild_needed(inp)), 345222748Srwatson ("%s: pcbgroup and wildcard!", __func__)); 346222748Srwatson} 347222748Srwatson 348222748Srwatson/* 349222748Srwatson * Two update paths: one in which the 4-tuple on an inpcb has been updated 350222748Srwatson * and therefore connection groups may need to change (or a wildcard entry 351222748Srwatson * may needed to be installed), and another in which the 4-tuple has been 352222748Srwatson * set as a result of a packet received, in which case we may be able to use 353222748Srwatson * the hash on the mbuf to avoid doing a software hash calculation for RSS. 354222748Srwatson * 355222748Srwatson * In each case: first, let the wildcard code have a go at placing it as a 356222748Srwatson * wildcard socket. If it was a wildcard, or if the connection has been 357222748Srwatson * dropped, then no pcbgroup is required (so potentially clear it); 358222748Srwatson * otherwise, calculate and update the pcbgroup for the inpcb. 359222748Srwatson */ 360222748Srwatsonvoid 361222748Srwatsonin_pcbgroup_update(struct inpcb *inp) 362222748Srwatson{ 363222748Srwatson struct inpcbinfo *pcbinfo; 364222748Srwatson struct inpcbgroup *newpcbgroup; 365222748Srwatson 366222748Srwatson INP_WLOCK_ASSERT(inp); 367222748Srwatson 368222748Srwatson pcbinfo = inp->inp_pcbinfo; 369222748Srwatson if (!in_pcbgroup_enabled(pcbinfo)) 370222748Srwatson return; 371222748Srwatson 372222748Srwatson in_pcbwild_update_internal(inp); 373222748Srwatson if (!(inp->inp_flags2 & INP_PCBGROUPWILD) && 374222748Srwatson !(inp->inp_flags & INP_DROPPED)) { 375222748Srwatson#ifdef INET6 376222748Srwatson if (inp->inp_vflag & INP_IPV6) 377222748Srwatson newpcbgroup = in6_pcbgroup_byinpcb(inp); 378222748Srwatson else 379222748Srwatson#endif 380222748Srwatson newpcbgroup = in_pcbgroup_byinpcb(inp); 381222748Srwatson } else 382222748Srwatson newpcbgroup = NULL; 383222748Srwatson in_pcbgroup_update_internal(pcbinfo, newpcbgroup, inp); 384222748Srwatson} 385222748Srwatson 386222748Srwatsonvoid 387222748Srwatsonin_pcbgroup_update_mbuf(struct inpcb *inp, struct mbuf *m) 388222748Srwatson{ 389222748Srwatson struct inpcbinfo *pcbinfo; 390222748Srwatson struct inpcbgroup *newpcbgroup; 391222748Srwatson 392222748Srwatson INP_WLOCK_ASSERT(inp); 393222748Srwatson 394222748Srwatson pcbinfo = inp->inp_pcbinfo; 395222748Srwatson if (!in_pcbgroup_enabled(pcbinfo)) 396222748Srwatson return; 397222748Srwatson 398222748Srwatson /* 399222748Srwatson * Possibly should assert !INP_PCBGROUPWILD rather than testing for 400222748Srwatson * it; presumably this function should never be called for anything 401222748Srwatson * other than non-wildcard socket? 402222748Srwatson */ 403222748Srwatson in_pcbwild_update_internal(inp); 404222748Srwatson if (!(inp->inp_flags2 & INP_PCBGROUPWILD) && 405222748Srwatson !(inp->inp_flags & INP_DROPPED)) { 406222748Srwatson newpcbgroup = in_pcbgroup_bymbuf(pcbinfo, m); 407222748Srwatson#ifdef INET6 408222748Srwatson if (inp->inp_vflag & INP_IPV6) { 409222748Srwatson if (newpcbgroup == NULL) 410222748Srwatson newpcbgroup = in6_pcbgroup_byinpcb(inp); 411222748Srwatson } else { 412222748Srwatson#endif 413222748Srwatson if (newpcbgroup == NULL) 414222748Srwatson newpcbgroup = in_pcbgroup_byinpcb(inp); 415222748Srwatson#ifdef INET6 416222748Srwatson } 417222748Srwatson#endif 418222748Srwatson } else 419222748Srwatson newpcbgroup = NULL; 420222748Srwatson in_pcbgroup_update_internal(pcbinfo, newpcbgroup, inp); 421222748Srwatson} 422222748Srwatson 423222748Srwatson/* 424222748Srwatson * Remove pcbgroup entry and optional pcbgroup wildcard entry for this inpcb. 425222748Srwatson */ 426222748Srwatsonvoid 427222748Srwatsonin_pcbgroup_remove(struct inpcb *inp) 428222748Srwatson{ 429222748Srwatson struct inpcbgroup *pcbgroup; 430222748Srwatson 431222748Srwatson INP_WLOCK_ASSERT(inp); 432222748Srwatson 433222748Srwatson if (!in_pcbgroup_enabled(inp->inp_pcbinfo)) 434222748Srwatson return; 435222748Srwatson 436222748Srwatson if (inp->inp_flags2 & INP_PCBGROUPWILD) 437222748Srwatson in_pcbwild_remove(inp); 438222748Srwatson 439222748Srwatson pcbgroup = inp->inp_pcbgroup; 440222748Srwatson if (pcbgroup != NULL) { 441222748Srwatson INP_GROUP_LOCK(pcbgroup); 442222748Srwatson LIST_REMOVE(inp, inp_pcbgrouphash); 443222748Srwatson inp->inp_pcbgroup = NULL; 444222748Srwatson INP_GROUP_UNLOCK(pcbgroup); 445222748Srwatson } 446222748Srwatson} 447222748Srwatson 448222748Srwatson/* 449222748Srwatson * Query whether or not it is appropriate to use pcbgroups to look up inpcbs 450222748Srwatson * for a protocol. 451222748Srwatson */ 452222748Srwatsonint 453222748Srwatsonin_pcbgroup_enabled(struct inpcbinfo *pcbinfo) 454222748Srwatson{ 455222748Srwatson 456222748Srwatson return (pcbinfo->ipi_npcbgroups > 0); 457222748Srwatson} 458