1263198Srwatson/*- 2263198Srwatson * Copyright (c) 2010-2011 Juniper Networks, Inc. 3263198Srwatson * All rights reserved. 4263198Srwatson * 5263198Srwatson * This software was developed by Robert N. M. Watson under contract 6263198Srwatson * to Juniper Networks, Inc. 7263198Srwatson * 8263198Srwatson * Redistribution and use in source and binary forms, with or without 9263198Srwatson * modification, are permitted provided that the following conditions 10263198Srwatson * are met: 11263198Srwatson * 1. Redistributions of source code must retain the above copyright 12263198Srwatson * notice, this list of conditions and the following disclaimer. 13263198Srwatson * 2. Redistributions in binary form must reproduce the above copyright 14263198Srwatson * notice, this list of conditions and the following disclaimer in the 15263198Srwatson * documentation and/or other materials provided with the distribution. 16263198Srwatson * 17263198Srwatson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18263198Srwatson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19263198Srwatson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20263198Srwatson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21263198Srwatson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22263198Srwatson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23263198Srwatson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24263198Srwatson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25263198Srwatson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26263198Srwatson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27263198Srwatson * SUCH DAMAGE. 28263198Srwatson */ 29263198Srwatson 30263198Srwatson#include <sys/cdefs.h> 31263198Srwatson 32263198Srwatson__FBSDID("$FreeBSD$"); 33263198Srwatson 34263198Srwatson#include "opt_inet6.h" 35263198Srwatson#include "opt_pcbgroup.h" 36263198Srwatson 37263198Srwatson#ifndef PCBGROUP 38263198Srwatson#error "options RSS depends on options PCBGROUP" 39263198Srwatson#endif 40263198Srwatson 41263198Srwatson#include <sys/param.h> 42263198Srwatson#include <sys/mbuf.h> 43263198Srwatson#include <sys/socket.h> 44263198Srwatson#include <sys/priv.h> 45263198Srwatson#include <sys/kernel.h> 46263198Srwatson#include <sys/smp.h> 47263198Srwatson#include <sys/sysctl.h> 48266737Sadrian#include <sys/sbuf.h> 49263198Srwatson 50263198Srwatson#include <net/if.h> 51263198Srwatson#include <net/if_var.h> 52263198Srwatson#include <net/netisr.h> 53277331Sadrian#include <net/rss_config.h> 54277331Sadrian#include <net/toeplitz.h> 55263198Srwatson 56263198Srwatson/*- 57263198Srwatson * Operating system parts of receiver-side scaling (RSS), which allows 58263198Srwatson * network cards to direct flows to particular receive queues based on hashes 59263198Srwatson * of header tuples. This implementation aligns RSS buckets with connection 60263198Srwatson * groups at the TCP/IP layer, so each bucket is associated with exactly one 61263198Srwatson * group. As a result, the group lookup structures (and lock) should have an 62263198Srwatson * effective affinity with exactly one CPU. 63263198Srwatson * 64263198Srwatson * Network device drivers needing to configure RSS will query this framework 65263198Srwatson * for parameters, such as the current RSS key, hashing policies, number of 66263198Srwatson * bits, and indirection table mapping hashes to buckets and CPUs. They may 67263198Srwatson * provide their own supplementary information, such as queue<->CPU bindings. 68263198Srwatson * It is the responsibility of the network device driver to inject packets 69263198Srwatson * into the stack on as close to the right CPU as possible, if playing by RSS 70263198Srwatson * rules. 71263198Srwatson * 72263198Srwatson * TODO: 73263198Srwatson * 74263198Srwatson * - Synchronization for rss_key and other future-configurable parameters. 75263198Srwatson * - Event handler drivers can register to pick up RSS configuration changes. 76263198Srwatson * - Should we allow rss_basecpu to be configured? 77263198Srwatson * - Randomize key on boot. 78263198Srwatson * - IPv6 support. 79263198Srwatson * - Statistics on how often there's a misalignment between hardware 80263198Srwatson * placement and pcbgroup expectations. 81263198Srwatson */ 82263198Srwatson 83277331SadrianSYSCTL_DECL(_net_inet); 84263198SrwatsonSYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); 85263198Srwatson 86263198Srwatson/* 87263198Srwatson * Toeplitz is the only required hash function in the RSS spec, so use it by 88263198Srwatson * default. 89263198Srwatson */ 90263198Srwatsonstatic u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 91267992ShselaskySYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 92263198Srwatson "RSS hash algorithm"); 93263198Srwatson 94263198Srwatson/* 95263198Srwatson * Size of the indirection table; at most 128 entries per the RSS spec. We 96263198Srwatson * size it to at least 2 times the number of CPUs by default to allow useful 97263198Srwatson * rebalancing. If not set explicitly with a loader tunable, we tune based 98263198Srwatson * on the number of CPUs present. 99263198Srwatson * 100263198Srwatson * XXXRW: buckets might be better to use for the tunable than bits. 101263198Srwatson */ 102263198Srwatsonstatic u_int rss_bits; 103267992ShselaskySYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 104263198Srwatson "RSS bits"); 105263198Srwatson 106263198Srwatsonstatic u_int rss_mask; 107263198SrwatsonSYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 108263198Srwatson "RSS mask"); 109263198Srwatson 110263198Srwatsonstatic const u_int rss_maxbits = RSS_MAXBITS; 111263198SrwatsonSYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 112263198Srwatson __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 113263198Srwatson 114263198Srwatson/* 115263198Srwatson * RSS's own count of the number of CPUs it could be using for processing. 116263198Srwatson * Bounded to 64 by RSS constants. 117263198Srwatson */ 118263198Srwatsonstatic u_int rss_ncpus; 119263198SrwatsonSYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 120263198Srwatson "Number of CPUs available to RSS"); 121263198Srwatson 122263198Srwatson#define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 123263198Srwatsonstatic const u_int rss_maxcpus = RSS_MAXCPUS; 124263198SrwatsonSYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 125263198Srwatson __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 126263198Srwatson 127263198Srwatson/* 128263198Srwatson * Variable exists just for reporting rss_bits in a user-friendly way. 129263198Srwatson */ 130263198Srwatsonstatic u_int rss_buckets; 131263198SrwatsonSYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 132263198Srwatson "RSS buckets"); 133263198Srwatson 134263198Srwatson/* 135263198Srwatson * Base CPU number; devices will add this to all CPU numbers returned by the 136263198Srwatson * RSS indirection table. Currently unmodifable in FreeBSD. 137263198Srwatson */ 138263198Srwatsonstatic const u_int rss_basecpu; 139263198SrwatsonSYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 140263198Srwatson __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 141263198Srwatson 142263198Srwatson/* 143287245Sadrian * Print verbose debugging messages. 144287245Sadrian * 0 - disable 145287245Sadrian * non-zero - enable 146287245Sadrian */ 147287245Sadrianint rss_debug = 0; 148287245SadrianSYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0, 149287245Sadrian "RSS debug level"); 150287245Sadrian 151287245Sadrian/* 152263198Srwatson * RSS secret key, intended to prevent attacks on load-balancing. Its 153263198Srwatson * effectiveness may be limited by algorithm choice and available entropy 154263198Srwatson * during the boot. 155263198Srwatson * 156263198Srwatson * XXXRW: And that we don't randomize it yet! 157263198Srwatson * 158268909Sadrian * This is the default Microsoft RSS specification key which is also 159268909Sadrian * the Chelsio T5 firmware default key. 160263198Srwatson */ 161268837Sadrianstatic uint8_t rss_key[RSS_KEYSIZE] = { 162269391Sgrehan 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 163269391Sgrehan 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 164269391Sgrehan 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 165269391Sgrehan 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 166269391Sgrehan 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 167263198Srwatson}; 168263198Srwatson 169263198Srwatson/* 170263198Srwatson * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 171298995Spfg * Drivers may supplement this table with a separate CPU<->queue table when 172263198Srwatson * programming devices. 173263198Srwatson */ 174263198Srwatsonstruct rss_table_entry { 175263198Srwatson uint8_t rte_cpu; /* CPU affinity of bucket. */ 176263198Srwatson}; 177263198Srwatsonstatic struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 178263198Srwatson 179263198Srwatsonstatic void 180263198Srwatsonrss_init(__unused void *arg) 181263198Srwatson{ 182263198Srwatson u_int i; 183266537Sadrian u_int cpuid; 184263198Srwatson 185263198Srwatson /* 186263198Srwatson * Validate tunables, coerce to sensible values. 187263198Srwatson */ 188263198Srwatson switch (rss_hashalgo) { 189263198Srwatson case RSS_HASH_TOEPLITZ: 190263198Srwatson case RSS_HASH_NAIVE: 191263198Srwatson break; 192263198Srwatson 193263198Srwatson default: 194287245Sadrian RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n", 195287245Sadrian rss_hashalgo, RSS_HASH_TOEPLITZ); 196263198Srwatson rss_hashalgo = RSS_HASH_TOEPLITZ; 197263198Srwatson } 198263198Srwatson 199263198Srwatson /* 200263198Srwatson * Count available CPUs. 201263198Srwatson * 202263198Srwatson * XXXRW: Note incorrect assumptions regarding contiguity of this set 203263198Srwatson * elsewhere. 204263198Srwatson */ 205263198Srwatson rss_ncpus = 0; 206263198Srwatson for (i = 0; i <= mp_maxid; i++) { 207263198Srwatson if (CPU_ABSENT(i)) 208263198Srwatson continue; 209263198Srwatson rss_ncpus++; 210263198Srwatson } 211263198Srwatson if (rss_ncpus > RSS_MAXCPUS) 212263198Srwatson rss_ncpus = RSS_MAXCPUS; 213263198Srwatson 214263198Srwatson /* 215263198Srwatson * Tune RSS table entries to be no less than 2x the number of CPUs 216263198Srwatson * -- unless we're running uniprocessor, in which case there's not 217263198Srwatson * much point in having buckets to rearrange for load-balancing! 218263198Srwatson */ 219263198Srwatson if (rss_ncpus > 1) { 220263198Srwatson if (rss_bits == 0) 221263198Srwatson rss_bits = fls(rss_ncpus - 1) + 1; 222263198Srwatson 223263198Srwatson /* 224263198Srwatson * Microsoft limits RSS table entries to 128, so apply that 225263198Srwatson * limit to both auto-detected CPU counts and user-configured 226263198Srwatson * ones. 227263198Srwatson */ 228263198Srwatson if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 229287245Sadrian RSS_DEBUG("RSS bits %u not valid, coercing to %u\n", 230287245Sadrian rss_bits, RSS_MAXBITS); 231263198Srwatson rss_bits = RSS_MAXBITS; 232263198Srwatson } 233263198Srwatson 234263198Srwatson /* 235263198Srwatson * Figure out how many buckets to use; warn if less than the 236263198Srwatson * number of configured CPUs, although this is not a fatal 237263198Srwatson * problem. 238263198Srwatson */ 239263198Srwatson rss_buckets = (1 << rss_bits); 240263198Srwatson if (rss_buckets < rss_ncpus) 241287245Sadrian RSS_DEBUG("WARNING: rss_buckets (%u) less than " 242287245Sadrian "rss_ncpus (%u)\n", rss_buckets, rss_ncpus); 243263198Srwatson rss_mask = rss_buckets - 1; 244263198Srwatson } else { 245263198Srwatson rss_bits = 0; 246263198Srwatson rss_buckets = 1; 247263198Srwatson rss_mask = 0; 248263198Srwatson } 249263198Srwatson 250263198Srwatson /* 251263198Srwatson * Set up initial CPU assignments: round-robin by default. 252263198Srwatson */ 253266537Sadrian cpuid = CPU_FIRST(); 254266537Sadrian for (i = 0; i < rss_buckets; i++) { 255266537Sadrian rss_table[i].rte_cpu = cpuid; 256266537Sadrian cpuid = CPU_NEXT(cpuid); 257266537Sadrian } 258263198Srwatson 259263198Srwatson /* 260263198Srwatson * Randomize rrs_key. 261263198Srwatson * 262263198Srwatson * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 263263198Srwatson * loop to check for "bad" RSS keys. 264263198Srwatson */ 265263198Srwatson} 266263198SrwatsonSYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 267263198Srwatson 268263198Srwatsonstatic uint32_t 269263198Srwatsonrss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 270263198Srwatson const uint8_t *data) 271263198Srwatson{ 272263198Srwatson uint32_t v; 273263198Srwatson u_int i; 274263198Srwatson 275263198Srwatson v = 0; 276263198Srwatson for (i = 0; i < keylen; i++) 277263198Srwatson v += key[i]; 278263198Srwatson for (i = 0; i < datalen; i++) 279263198Srwatson v += data[i]; 280263198Srwatson return (v); 281263198Srwatson} 282263198Srwatson 283277331Sadrianuint32_t 284263198Srwatsonrss_hash(u_int datalen, const uint8_t *data) 285263198Srwatson{ 286263198Srwatson 287263198Srwatson switch (rss_hashalgo) { 288263198Srwatson case RSS_HASH_TOEPLITZ: 289263198Srwatson return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 290263198Srwatson data)); 291263198Srwatson 292263198Srwatson case RSS_HASH_NAIVE: 293263198Srwatson return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 294263198Srwatson data)); 295263198Srwatson 296263198Srwatson default: 297263198Srwatson panic("%s: unsupported/unknown hashalgo %d", __func__, 298263198Srwatson rss_hashalgo); 299263198Srwatson } 300263198Srwatson} 301263198Srwatson 302263198Srwatson/* 303263198Srwatson * Query the number of RSS bits in use. 304263198Srwatson */ 305263198Srwatsonu_int 306263198Srwatsonrss_getbits(void) 307263198Srwatson{ 308263198Srwatson 309263198Srwatson return (rss_bits); 310263198Srwatson} 311263198Srwatson 312263198Srwatson/* 313263198Srwatson * Query the RSS bucket associated with an RSS hash. 314263198Srwatson */ 315263198Srwatsonu_int 316263198Srwatsonrss_getbucket(u_int hash) 317263198Srwatson{ 318263198Srwatson 319263198Srwatson return (hash & rss_mask); 320263198Srwatson} 321263198Srwatson 322263198Srwatson/* 323267891Sadrian * Query the RSS layer bucket associated with the given 324267891Sadrian * entry in the RSS hash space. 325267891Sadrian * 326267891Sadrian * The RSS indirection table is 0 .. rss_buckets-1, 327267891Sadrian * covering the low 'rss_bits' of the total 128 slot 328267891Sadrian * RSS indirection table. So just mask off rss_bits and 329267891Sadrian * return that. 330267891Sadrian * 331267891Sadrian * NIC drivers can then iterate over the 128 slot RSS 332267891Sadrian * indirection table and fetch which RSS bucket to 333267891Sadrian * map it to. This will typically be a CPU queue 334267891Sadrian */ 335267891Sadrianu_int 336267891Sadrianrss_get_indirection_to_bucket(u_int index) 337267891Sadrian{ 338267891Sadrian 339267891Sadrian return (index & rss_mask); 340267891Sadrian} 341267891Sadrian 342267891Sadrian/* 343263198Srwatson * Query the RSS CPU associated with an RSS bucket. 344263198Srwatson */ 345263198Srwatsonu_int 346263198Srwatsonrss_getcpu(u_int bucket) 347263198Srwatson{ 348263198Srwatson 349263198Srwatson return (rss_table[bucket].rte_cpu); 350263198Srwatson} 351263198Srwatson 352263198Srwatson/* 353266419Sadrian * netisr CPU affinity lookup given just the hash and hashtype. 354263198Srwatson */ 355266419Sadrianu_int 356266419Sadrianrss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 357263198Srwatson{ 358263198Srwatson 359266419Sadrian switch (hash_type) { 360263198Srwatson case M_HASHTYPE_RSS_IPV4: 361263198Srwatson case M_HASHTYPE_RSS_TCP_IPV4: 362268912Sadrian case M_HASHTYPE_RSS_UDP_IPV4: 363268559Sadrian case M_HASHTYPE_RSS_IPV6: 364268559Sadrian case M_HASHTYPE_RSS_TCP_IPV6: 365268912Sadrian case M_HASHTYPE_RSS_UDP_IPV6: 366266419Sadrian return (rss_getcpu(rss_getbucket(hash_val))); 367263198Srwatson default: 368266419Sadrian return (NETISR_CPUID_NONE); 369263198Srwatson } 370263198Srwatson} 371263198Srwatson 372263198Srwatson/* 373266737Sadrian * Query the RSS bucket associated with the given hash value and 374266737Sadrian * type. 375266737Sadrian */ 376266737Sadrianint 377266737Sadrianrss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 378266737Sadrian{ 379266737Sadrian 380266737Sadrian switch (hash_type) { 381266737Sadrian case M_HASHTYPE_RSS_IPV4: 382266737Sadrian case M_HASHTYPE_RSS_TCP_IPV4: 383268912Sadrian case M_HASHTYPE_RSS_UDP_IPV4: 384268559Sadrian case M_HASHTYPE_RSS_IPV6: 385268559Sadrian case M_HASHTYPE_RSS_TCP_IPV6: 386268912Sadrian case M_HASHTYPE_RSS_UDP_IPV6: 387266737Sadrian *bucket_id = rss_getbucket(hash_val); 388266737Sadrian return (0); 389266737Sadrian default: 390266737Sadrian return (-1); 391266737Sadrian } 392266737Sadrian} 393266737Sadrian 394266737Sadrian/* 395266419Sadrian * netisr CPU affinity lookup routine for use by protocols. 396266419Sadrian */ 397266419Sadrianstruct mbuf * 398266419Sadrianrss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 399266419Sadrian{ 400266419Sadrian 401266419Sadrian M_ASSERTPKTHDR(m); 402266419Sadrian *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 403266419Sadrian return (m); 404266419Sadrian} 405266419Sadrian 406266737Sadrianint 407266737Sadrianrss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 408266737Sadrian{ 409266737Sadrian 410266737Sadrian M_ASSERTPKTHDR(m); 411266737Sadrian 412266737Sadrian return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 413266737Sadrian bucket_id)); 414266737Sadrian} 415266737Sadrian 416266419Sadrian/* 417263198Srwatson * Query the RSS hash algorithm. 418263198Srwatson */ 419263198Srwatsonu_int 420263198Srwatsonrss_gethashalgo(void) 421263198Srwatson{ 422263198Srwatson 423263198Srwatson return (rss_hashalgo); 424263198Srwatson} 425263198Srwatson 426263198Srwatson/* 427263198Srwatson * Query the current RSS key; likely to be used by device drivers when 428263198Srwatson * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 429263198Srwatson * 430263198Srwatson * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 431263198Srwatson */ 432263198Srwatsonvoid 433263198Srwatsonrss_getkey(uint8_t *key) 434263198Srwatson{ 435263198Srwatson 436263198Srwatson bcopy(rss_key, key, sizeof(rss_key)); 437263198Srwatson} 438263198Srwatson 439263198Srwatson/* 440263198Srwatson * Query the number of buckets; this may be used by both network device 441263198Srwatson * drivers, which will need to populate hardware shadows of the software 442263198Srwatson * indirection table, and the network stack itself (such as when deciding how 443263198Srwatson * many connection groups to allocate). 444263198Srwatson */ 445263198Srwatsonu_int 446263198Srwatsonrss_getnumbuckets(void) 447263198Srwatson{ 448263198Srwatson 449263198Srwatson return (rss_buckets); 450263198Srwatson} 451263198Srwatson 452263198Srwatson/* 453263198Srwatson * Query the number of CPUs in use by RSS; may be useful to device drivers 454263198Srwatson * trying to figure out how to map a larger number of CPUs into a smaller 455263198Srwatson * number of receive queues. 456263198Srwatson */ 457263198Srwatsonu_int 458263198Srwatsonrss_getnumcpus(void) 459263198Srwatson{ 460263198Srwatson 461263198Srwatson return (rss_ncpus); 462263198Srwatson} 463263198Srwatson 464277331Sadrian/* 465277331Sadrian * Return the supported RSS hash configuration. 466277331Sadrian * 467277331Sadrian * NICs should query this to determine what to configure in their redirection 468277331Sadrian * matching table. 469277331Sadrian */ 470277331Sadrianinline u_int 471277331Sadrianrss_gethashconfig(void) 472268911Sadrian{ 473271297Sadrian 474268911Sadrian /* Return 4-tuple for TCP; 2-tuple for others */ 475268911Sadrian /* 476268911Sadrian * UDP may fragment more often than TCP and thus we'll end up with 477268911Sadrian * NICs returning 2-tuple fragments. 478268911Sadrian * udp_init() and udplite_init() both currently initialise things 479268911Sadrian * as 2-tuple. 480268911Sadrian * So for now disable UDP 4-tuple hashing until all of the other 481268911Sadrian * pieces are in place. 482268911Sadrian */ 483268911Sadrian return ( 484268911Sadrian RSS_HASHTYPE_RSS_IPV4 485268911Sadrian | RSS_HASHTYPE_RSS_TCP_IPV4 486268911Sadrian | RSS_HASHTYPE_RSS_IPV6 487268911Sadrian | RSS_HASHTYPE_RSS_TCP_IPV6 488268911Sadrian | RSS_HASHTYPE_RSS_IPV6_EX 489268911Sadrian | RSS_HASHTYPE_RSS_TCP_IPV6_EX 490268911Sadrian#if 0 491268911Sadrian | RSS_HASHTYPE_RSS_UDP_IPV4 492268911Sadrian | RSS_HASHTYPE_RSS_UDP_IPV4_EX 493268911Sadrian | RSS_HASHTYPE_RSS_UDP_IPV6 494268911Sadrian | RSS_HASHTYPE_RSS_UDP_IPV6_EX 495268911Sadrian#endif 496268911Sadrian ); 497268911Sadrian} 498268911Sadrian 499268911Sadrian/* 500263198Srwatson * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 501263198Srwatson * it appearing in debugging output unnecessarily. 502263198Srwatson */ 503263198Srwatsonstatic int 504263198Srwatsonsysctl_rss_key(SYSCTL_HANDLER_ARGS) 505263198Srwatson{ 506263198Srwatson uint8_t temp_rss_key[RSS_KEYSIZE]; 507263198Srwatson int error; 508263198Srwatson 509263198Srwatson error = priv_check(req->td, PRIV_NETINET_HASHKEY); 510263198Srwatson if (error) 511263198Srwatson return (error); 512263198Srwatson 513263198Srwatson bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 514263198Srwatson error = sysctl_handle_opaque(oidp, temp_rss_key, 515263198Srwatson sizeof(temp_rss_key), req); 516263198Srwatson if (error) 517263198Srwatson return (error); 518263198Srwatson if (req->newptr != NULL) { 519263198Srwatson /* XXXRW: Not yet. */ 520263198Srwatson return (EINVAL); 521263198Srwatson } 522263198Srwatson return (0); 523263198Srwatson} 524263198SrwatsonSYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 525263198Srwatson CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 526263198Srwatson "", "RSS keying material"); 527266737Sadrian 528266737Sadrianstatic int 529266737Sadriansysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 530266737Sadrian{ 531266737Sadrian struct sbuf *sb; 532266737Sadrian int error; 533266737Sadrian int i; 534266737Sadrian 535266737Sadrian error = 0; 536266737Sadrian error = sysctl_wire_old_buffer(req, 0); 537266737Sadrian if (error != 0) 538266737Sadrian return (error); 539266737Sadrian sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 540266737Sadrian if (sb == NULL) 541266737Sadrian return (ENOMEM); 542266737Sadrian for (i = 0; i < rss_buckets; i++) { 543266737Sadrian sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 544266737Sadrian i, 545266737Sadrian rss_getcpu(i)); 546266737Sadrian } 547266737Sadrian error = sbuf_finish(sb); 548266737Sadrian sbuf_delete(sb); 549266737Sadrian 550266737Sadrian return (error); 551266737Sadrian} 552266737SadrianSYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 553266737Sadrian CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, 554266737Sadrian sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 555