1/*-
2 * Copyright (c) 2010-2011 Juniper Networks, Inc.
3 * All rights reserved.
4 *
5 * This software was developed by Robert N. M. Watson under contract
6 * to Juniper Networks, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31
32__FBSDID("$FreeBSD$");
33
34#include "opt_inet6.h"
35#include "opt_pcbgroup.h"
36
37#ifndef PCBGROUP
38#error "options RSS depends on options PCBGROUP"
39#endif
40
41#include <sys/param.h>
42#include <sys/mbuf.h>
43#include <sys/socket.h>
44#include <sys/priv.h>
45#include <sys/kernel.h>
46#include <sys/smp.h>
47#include <sys/sysctl.h>
48#include <sys/sbuf.h>
49
50#include <net/if.h>
51#include <net/if_var.h>
52#include <net/netisr.h>
53#include <net/rss_config.h>
54#include <net/toeplitz.h>
55
56/*-
57 * Operating system parts of receiver-side scaling (RSS), which allows
58 * network cards to direct flows to particular receive queues based on hashes
59 * of header tuples.  This implementation aligns RSS buckets with connection
60 * groups at the TCP/IP layer, so each bucket is associated with exactly one
61 * group.  As a result, the group lookup structures (and lock) should have an
62 * effective affinity with exactly one CPU.
63 *
64 * Network device drivers needing to configure RSS will query this framework
65 * for parameters, such as the current RSS key, hashing policies, number of
66 * bits, and indirection table mapping hashes to buckets and CPUs.  They may
67 * provide their own supplementary information, such as queue<->CPU bindings.
68 * It is the responsibility of the network device driver to inject packets
69 * into the stack on as close to the right CPU as possible, if playing by RSS
70 * rules.
71 *
72 * TODO:
73 *
74 * - Synchronization for rss_key and other future-configurable parameters.
75 * - Event handler drivers can register to pick up RSS configuration changes.
76 * - Should we allow rss_basecpu to be configured?
77 * - Randomize key on boot.
78 * - IPv6 support.
79 * - Statistics on how often there's a misalignment between hardware
80 *   placement and pcbgroup expectations.
81 */
82
83SYSCTL_DECL(_net_inet);
84SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
85    "Receive-side steering");
86
87/*
88 * Toeplitz is the only required hash function in the RSS spec, so use it by
89 * default.
90 */
91static u_int	rss_hashalgo = RSS_HASH_TOEPLITZ;
92SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
93    "RSS hash algorithm");
94
95/*
96 * Size of the indirection table; at most 128 entries per the RSS spec.  We
97 * size it to at least 2 times the number of CPUs by default to allow useful
98 * rebalancing.  If not set explicitly with a loader tunable, we tune based
99 * on the number of CPUs present.
100 *
101 * XXXRW: buckets might be better to use for the tunable than bits.
102 */
103static u_int	rss_bits;
104SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
105    "RSS bits");
106
107static u_int	rss_mask;
108SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
109    "RSS mask");
110
111static const u_int	rss_maxbits = RSS_MAXBITS;
112SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
113    __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
114
115/*
116 * RSS's own count of the number of CPUs it could be using for processing.
117 * Bounded to 64 by RSS constants.
118 */
119static u_int	rss_ncpus;
120SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
121    "Number of CPUs available to RSS");
122
123#define	RSS_MAXCPUS	(1 << (RSS_MAXBITS - 1))
124static const u_int	rss_maxcpus = RSS_MAXCPUS;
125SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
126    __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
127
128/*
129 * Variable exists just for reporting rss_bits in a user-friendly way.
130 */
131static u_int	rss_buckets;
132SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
133    "RSS buckets");
134
135/*
136 * Base CPU number; devices will add this to all CPU numbers returned by the
137 * RSS indirection table.  Currently unmodifable in FreeBSD.
138 */
139static const u_int	rss_basecpu;
140SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
141    __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
142
143/*
144 * Print verbose debugging messages.
145 * 0 - disable
146 * non-zero - enable
147 */
148int	rss_debug = 0;
149SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0,
150    "RSS debug level");
151
152/*
153 * RSS secret key, intended to prevent attacks on load-balancing.  Its
154 * effectiveness may be limited by algorithm choice and available entropy
155 * during the boot.
156 *
157 * XXXRW: And that we don't randomize it yet!
158 *
159 * This is the default Microsoft RSS specification key which is also
160 * the Chelsio T5 firmware default key.
161 */
162static uint8_t rss_key[RSS_KEYSIZE] = {
163	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
164	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
165	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
166	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
167	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
168};
169
170/*
171 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
172 * Drivers may supplement this table with a separate CPU<->queue table when
173 * programming devices.
174 */
175struct rss_table_entry {
176	uint8_t		rte_cpu;	/* CPU affinity of bucket. */
177};
178static struct rss_table_entry	rss_table[RSS_TABLE_MAXLEN];
179
180static void
181rss_init(__unused void *arg)
182{
183	u_int i;
184	u_int cpuid;
185
186	/*
187	 * Validate tunables, coerce to sensible values.
188	 */
189	switch (rss_hashalgo) {
190	case RSS_HASH_TOEPLITZ:
191	case RSS_HASH_NAIVE:
192		break;
193
194	default:
195		RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n",
196		    rss_hashalgo, RSS_HASH_TOEPLITZ);
197		rss_hashalgo = RSS_HASH_TOEPLITZ;
198	}
199
200	/*
201	 * Count available CPUs.
202	 *
203	 * XXXRW: Note incorrect assumptions regarding contiguity of this set
204	 * elsewhere.
205	 */
206	rss_ncpus = 0;
207	for (i = 0; i <= mp_maxid; i++) {
208		if (CPU_ABSENT(i))
209			continue;
210		rss_ncpus++;
211	}
212	if (rss_ncpus > RSS_MAXCPUS)
213		rss_ncpus = RSS_MAXCPUS;
214
215	/*
216	 * Tune RSS table entries to be no less than 2x the number of CPUs
217	 * -- unless we're running uniprocessor, in which case there's not
218	 * much point in having buckets to rearrange for load-balancing!
219	 */
220	if (rss_ncpus > 1) {
221		if (rss_bits == 0)
222			rss_bits = fls(rss_ncpus - 1) + 1;
223
224		/*
225		 * Microsoft limits RSS table entries to 128, so apply that
226		 * limit to both auto-detected CPU counts and user-configured
227		 * ones.
228		 */
229		if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
230			RSS_DEBUG("RSS bits %u not valid, coercing to %u\n",
231			    rss_bits, RSS_MAXBITS);
232			rss_bits = RSS_MAXBITS;
233		}
234
235		/*
236		 * Figure out how many buckets to use; warn if less than the
237		 * number of configured CPUs, although this is not a fatal
238		 * problem.
239		 */
240		rss_buckets = (1 << rss_bits);
241		if (rss_buckets < rss_ncpus)
242			RSS_DEBUG("WARNING: rss_buckets (%u) less than "
243			    "rss_ncpus (%u)\n", rss_buckets, rss_ncpus);
244		rss_mask = rss_buckets - 1;
245	} else {
246		rss_bits = 0;
247		rss_buckets = 1;
248		rss_mask = 0;
249	}
250
251	/*
252	 * Set up initial CPU assignments: round-robin by default.
253	 */
254	cpuid = CPU_FIRST();
255	for (i = 0; i < rss_buckets; i++) {
256		rss_table[i].rte_cpu = cpuid;
257		cpuid = CPU_NEXT(cpuid);
258	}
259
260	/*
261	 * Randomize rrs_key.
262	 *
263	 * XXXRW: Not yet.  If nothing else, will require an rss_isbadkey()
264	 * loop to check for "bad" RSS keys.
265	 */
266}
267SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
268
269static uint32_t
270rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
271    const uint8_t *data)
272{
273	uint32_t v;
274	u_int i;
275
276	v = 0;
277	for (i = 0; i < keylen; i++)
278		v += key[i];
279	for (i = 0; i < datalen; i++)
280		v += data[i];
281	return (v);
282}
283
284uint32_t
285rss_hash(u_int datalen, const uint8_t *data)
286{
287
288	switch (rss_hashalgo) {
289	case RSS_HASH_TOEPLITZ:
290		return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
291		    data));
292
293	case RSS_HASH_NAIVE:
294		return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
295		    data));
296
297	default:
298		panic("%s: unsupported/unknown hashalgo %d", __func__,
299		    rss_hashalgo);
300	}
301}
302
303/*
304 * Query the number of RSS bits in use.
305 */
306u_int
307rss_getbits(void)
308{
309
310	return (rss_bits);
311}
312
313/*
314 * Query the RSS bucket associated with an RSS hash.
315 */
316u_int
317rss_getbucket(u_int hash)
318{
319
320	return (hash & rss_mask);
321}
322
323/*
324 * Query the RSS layer bucket associated with the given
325 * entry in the RSS hash space.
326 *
327 * The RSS indirection table is 0 .. rss_buckets-1,
328 * covering the low 'rss_bits' of the total 128 slot
329 * RSS indirection table.  So just mask off rss_bits and
330 * return that.
331 *
332 * NIC drivers can then iterate over the 128 slot RSS
333 * indirection table and fetch which RSS bucket to
334 * map it to.  This will typically be a CPU queue
335 */
336u_int
337rss_get_indirection_to_bucket(u_int index)
338{
339
340	return (index & rss_mask);
341}
342
343/*
344 * Query the RSS CPU associated with an RSS bucket.
345 */
346u_int
347rss_getcpu(u_int bucket)
348{
349
350	return (rss_table[bucket].rte_cpu);
351}
352
353/*
354 * netisr CPU affinity lookup given just the hash and hashtype.
355 */
356u_int
357rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
358{
359
360	switch (hash_type) {
361	case M_HASHTYPE_RSS_IPV4:
362	case M_HASHTYPE_RSS_TCP_IPV4:
363	case M_HASHTYPE_RSS_UDP_IPV4:
364	case M_HASHTYPE_RSS_IPV6:
365	case M_HASHTYPE_RSS_TCP_IPV6:
366	case M_HASHTYPE_RSS_UDP_IPV6:
367		return (rss_getcpu(rss_getbucket(hash_val)));
368	default:
369		return (NETISR_CPUID_NONE);
370	}
371}
372
373/*
374 * Query the RSS bucket associated with the given hash value and
375 * type.
376 */
377int
378rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
379{
380
381	switch (hash_type) {
382	case M_HASHTYPE_RSS_IPV4:
383	case M_HASHTYPE_RSS_TCP_IPV4:
384	case M_HASHTYPE_RSS_UDP_IPV4:
385	case M_HASHTYPE_RSS_IPV6:
386	case M_HASHTYPE_RSS_TCP_IPV6:
387	case M_HASHTYPE_RSS_UDP_IPV6:
388		*bucket_id = rss_getbucket(hash_val);
389		return (0);
390	default:
391		return (-1);
392	}
393}
394
395/*
396 * netisr CPU affinity lookup routine for use by protocols.
397 */
398struct mbuf *
399rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
400{
401
402	M_ASSERTPKTHDR(m);
403	*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
404	return (m);
405}
406
407int
408rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
409{
410
411	M_ASSERTPKTHDR(m);
412
413	return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
414	    bucket_id));
415}
416
417/*
418 * Query the RSS hash algorithm.
419 */
420u_int
421rss_gethashalgo(void)
422{
423
424	return (rss_hashalgo);
425}
426
427/*
428 * Query the current RSS key; likely to be used by device drivers when
429 * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
430 *
431 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
432 */
433void
434rss_getkey(uint8_t *key)
435{
436
437	bcopy(rss_key, key, sizeof(rss_key));
438}
439
440/*
441 * Query the number of buckets; this may be used by both network device
442 * drivers, which will need to populate hardware shadows of the software
443 * indirection table, and the network stack itself (such as when deciding how
444 * many connection groups to allocate).
445 */
446u_int
447rss_getnumbuckets(void)
448{
449
450	return (rss_buckets);
451}
452
453/*
454 * Query the number of CPUs in use by RSS; may be useful to device drivers
455 * trying to figure out how to map a larger number of CPUs into a smaller
456 * number of receive queues.
457 */
458u_int
459rss_getnumcpus(void)
460{
461
462	return (rss_ncpus);
463}
464
465/*
466 * Return the supported RSS hash configuration.
467 *
468 * NICs should query this to determine what to configure in their redirection
469 * matching table.
470 */
471inline u_int
472rss_gethashconfig(void)
473{
474
475	/* Return 4-tuple for TCP; 2-tuple for others */
476	/*
477	 * UDP may fragment more often than TCP and thus we'll end up with
478	 * NICs returning 2-tuple fragments.
479	 * udp_init() and udplite_init() both currently initialise things
480	 * as 2-tuple.
481	 * So for now disable UDP 4-tuple hashing until all of the other
482	 * pieces are in place.
483	 */
484	return (
485	    RSS_HASHTYPE_RSS_IPV4
486	|    RSS_HASHTYPE_RSS_TCP_IPV4
487	|    RSS_HASHTYPE_RSS_IPV6
488	|    RSS_HASHTYPE_RSS_TCP_IPV6
489	|    RSS_HASHTYPE_RSS_IPV6_EX
490	|    RSS_HASHTYPE_RSS_TCP_IPV6_EX
491#if 0
492	|    RSS_HASHTYPE_RSS_UDP_IPV4
493	|    RSS_HASHTYPE_RSS_UDP_IPV6
494	|    RSS_HASHTYPE_RSS_UDP_IPV6_EX
495#endif
496	);
497}
498
499/*
500 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
501 * it appearing in debugging output unnecessarily.
502 */
503static int
504sysctl_rss_key(SYSCTL_HANDLER_ARGS)
505{
506	uint8_t temp_rss_key[RSS_KEYSIZE];
507	int error;
508
509	error = priv_check(req->td, PRIV_NETINET_HASHKEY);
510	if (error)
511		return (error);
512
513	bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
514	error = sysctl_handle_opaque(oidp, temp_rss_key,
515	    sizeof(temp_rss_key), req);
516	if (error)
517		return (error);
518	if (req->newptr != NULL) {
519		/* XXXRW: Not yet. */
520		return (EINVAL);
521	}
522	return (0);
523}
524SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
525    CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
526    "", "RSS keying material");
527
528static int
529sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
530{
531	struct sbuf *sb;
532	int error;
533	int i;
534
535	error = 0;
536	error = sysctl_wire_old_buffer(req, 0);
537	if (error != 0)
538		return (error);
539	sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
540	if (sb == NULL)
541		return (ENOMEM);
542	for (i = 0; i < rss_buckets; i++) {
543		sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
544		    i,
545		    rss_getcpu(i));
546	}
547	error = sbuf_finish(sb);
548	sbuf_delete(sb);
549
550	return (error);
551}
552SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
553    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
554    sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
555