1/*
2 * Copyright (c) 2016 Citrix Systems Inc.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version 2
6 * as published by the Free Softare Foundation; or, when distributed
7 * separately from the Linux kernel or incorporated into other
8 * software packages, subject to the following license:
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this source file (the "Software"), to deal in the Software without
12 * restriction, including without limitation the rights to use, copy, modify,
13 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
14 * and to permit persons to whom the Software is furnished to do so, subject to
15 * the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 * IN THE SOFTWARE.
27 */
28
29#define XEN_NETIF_DEFINE_TOEPLITZ
30
31#include "common.h"
32#include <linux/vmalloc.h>
33#include <linux/rculist.h>
34
35static void xenvif_add_hash(struct xenvif *vif, const u8 *tag,
36			    unsigned int len, u32 val)
37{
38	struct xenvif_hash_cache_entry *new, *entry, *oldest;
39	unsigned long flags;
40	bool found;
41
42	new = kmalloc(sizeof(*entry), GFP_ATOMIC);
43	if (!new)
44		return;
45
46	memcpy(new->tag, tag, len);
47	new->len = len;
48	new->val = val;
49
50	spin_lock_irqsave(&vif->hash.cache.lock, flags);
51
52	found = false;
53	oldest = NULL;
54	list_for_each_entry_rcu(entry, &vif->hash.cache.list, link,
55				lockdep_is_held(&vif->hash.cache.lock)) {
56		/* Make sure we don't add duplicate entries */
57		if (entry->len == len &&
58		    memcmp(entry->tag, tag, len) == 0)
59			found = true;
60		if (!oldest || entry->seq < oldest->seq)
61			oldest = entry;
62	}
63
64	if (!found) {
65		new->seq = atomic_inc_return(&vif->hash.cache.seq);
66		list_add_rcu(&new->link, &vif->hash.cache.list);
67
68		if (++vif->hash.cache.count > xenvif_hash_cache_size) {
69			list_del_rcu(&oldest->link);
70			vif->hash.cache.count--;
71			kfree_rcu(oldest, rcu);
72		}
73	}
74
75	spin_unlock_irqrestore(&vif->hash.cache.lock, flags);
76
77	if (found)
78		kfree(new);
79}
80
81static u32 xenvif_new_hash(struct xenvif *vif, const u8 *data,
82			   unsigned int len)
83{
84	u32 val;
85
86	val = xen_netif_toeplitz_hash(vif->hash.key,
87				      sizeof(vif->hash.key),
88				      data, len);
89
90	if (xenvif_hash_cache_size != 0)
91		xenvif_add_hash(vif, data, len, val);
92
93	return val;
94}
95
96static void xenvif_flush_hash(struct xenvif *vif)
97{
98	struct xenvif_hash_cache_entry *entry;
99	unsigned long flags;
100
101	if (xenvif_hash_cache_size == 0)
102		return;
103
104	spin_lock_irqsave(&vif->hash.cache.lock, flags);
105
106	list_for_each_entry_rcu(entry, &vif->hash.cache.list, link,
107				lockdep_is_held(&vif->hash.cache.lock)) {
108		list_del_rcu(&entry->link);
109		vif->hash.cache.count--;
110		kfree_rcu(entry, rcu);
111	}
112
113	spin_unlock_irqrestore(&vif->hash.cache.lock, flags);
114}
115
116static u32 xenvif_find_hash(struct xenvif *vif, const u8 *data,
117			    unsigned int len)
118{
119	struct xenvif_hash_cache_entry *entry;
120	u32 val;
121	bool found;
122
123	if (len >= XEN_NETBK_HASH_TAG_SIZE)
124		return 0;
125
126	if (xenvif_hash_cache_size == 0)
127		return xenvif_new_hash(vif, data, len);
128
129	rcu_read_lock();
130
131	found = false;
132
133	list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) {
134		if (entry->len == len &&
135		    memcmp(entry->tag, data, len) == 0) {
136			val = entry->val;
137			entry->seq = atomic_inc_return(&vif->hash.cache.seq);
138			found = true;
139			break;
140		}
141	}
142
143	rcu_read_unlock();
144
145	if (!found)
146		val = xenvif_new_hash(vif, data, len);
147
148	return val;
149}
150
151void xenvif_set_skb_hash(struct xenvif *vif, struct sk_buff *skb)
152{
153	struct flow_keys flow;
154	u32 hash = 0;
155	enum pkt_hash_types type = PKT_HASH_TYPE_NONE;
156	u32 flags = vif->hash.flags;
157	bool has_tcp_hdr;
158
159	/* Quick rejection test: If the network protocol doesn't
160	 * correspond to any enabled hash type then there's no point
161	 * in parsing the packet header.
162	 */
163	switch (skb->protocol) {
164	case htons(ETH_P_IP):
165		if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP |
166			     XEN_NETIF_CTRL_HASH_TYPE_IPV4))
167			break;
168
169		goto done;
170
171	case htons(ETH_P_IPV6):
172		if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP |
173			     XEN_NETIF_CTRL_HASH_TYPE_IPV6))
174			break;
175
176		goto done;
177
178	default:
179		goto done;
180	}
181
182	memset(&flow, 0, sizeof(flow));
183	if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
184		goto done;
185
186	has_tcp_hdr = (flow.basic.ip_proto == IPPROTO_TCP) &&
187		      !(flow.control.flags & FLOW_DIS_IS_FRAGMENT);
188
189	switch (skb->protocol) {
190	case htons(ETH_P_IP):
191		if (has_tcp_hdr &&
192		    (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP)) {
193			u8 data[12];
194
195			memcpy(&data[0], &flow.addrs.v4addrs.src, 4);
196			memcpy(&data[4], &flow.addrs.v4addrs.dst, 4);
197			memcpy(&data[8], &flow.ports.src, 2);
198			memcpy(&data[10], &flow.ports.dst, 2);
199
200			hash = xenvif_find_hash(vif, data, sizeof(data));
201			type = PKT_HASH_TYPE_L4;
202		} else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4) {
203			u8 data[8];
204
205			memcpy(&data[0], &flow.addrs.v4addrs.src, 4);
206			memcpy(&data[4], &flow.addrs.v4addrs.dst, 4);
207
208			hash = xenvif_find_hash(vif, data, sizeof(data));
209			type = PKT_HASH_TYPE_L3;
210		}
211
212		break;
213
214	case htons(ETH_P_IPV6):
215		if (has_tcp_hdr &&
216		    (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)) {
217			u8 data[36];
218
219			memcpy(&data[0], &flow.addrs.v6addrs.src, 16);
220			memcpy(&data[16], &flow.addrs.v6addrs.dst, 16);
221			memcpy(&data[32], &flow.ports.src, 2);
222			memcpy(&data[34], &flow.ports.dst, 2);
223
224			hash = xenvif_find_hash(vif, data, sizeof(data));
225			type = PKT_HASH_TYPE_L4;
226		} else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6) {
227			u8 data[32];
228
229			memcpy(&data[0], &flow.addrs.v6addrs.src, 16);
230			memcpy(&data[16], &flow.addrs.v6addrs.dst, 16);
231
232			hash = xenvif_find_hash(vif, data, sizeof(data));
233			type = PKT_HASH_TYPE_L3;
234		}
235
236		break;
237	}
238
239done:
240	if (type == PKT_HASH_TYPE_NONE)
241		skb_clear_hash(skb);
242	else
243		__skb_set_sw_hash(skb, hash, type == PKT_HASH_TYPE_L4);
244}
245
246u32 xenvif_set_hash_alg(struct xenvif *vif, u32 alg)
247{
248	switch (alg) {
249	case XEN_NETIF_CTRL_HASH_ALGORITHM_NONE:
250	case XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ:
251		break;
252
253	default:
254		return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
255	}
256
257	vif->hash.alg = alg;
258
259	return XEN_NETIF_CTRL_STATUS_SUCCESS;
260}
261
262u32 xenvif_get_hash_flags(struct xenvif *vif, u32 *flags)
263{
264	if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE)
265		return XEN_NETIF_CTRL_STATUS_NOT_SUPPORTED;
266
267	*flags = XEN_NETIF_CTRL_HASH_TYPE_IPV4 |
268		 XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP |
269		 XEN_NETIF_CTRL_HASH_TYPE_IPV6 |
270		 XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP;
271
272	return XEN_NETIF_CTRL_STATUS_SUCCESS;
273}
274
275u32 xenvif_set_hash_flags(struct xenvif *vif, u32 flags)
276{
277	if (flags & ~(XEN_NETIF_CTRL_HASH_TYPE_IPV4 |
278		      XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP |
279		      XEN_NETIF_CTRL_HASH_TYPE_IPV6 |
280		      XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP))
281		return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
282
283	if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE)
284		return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
285
286	vif->hash.flags = flags;
287
288	return XEN_NETIF_CTRL_STATUS_SUCCESS;
289}
290
291u32 xenvif_set_hash_key(struct xenvif *vif, u32 gref, u32 len)
292{
293	u8 *key = vif->hash.key;
294	struct gnttab_copy copy_op = {
295		.source.u.ref = gref,
296		.source.domid = vif->domid,
297		.dest.u.gmfn = virt_to_gfn(key),
298		.dest.domid = DOMID_SELF,
299		.dest.offset = xen_offset_in_page(key),
300		.len = len,
301		.flags = GNTCOPY_source_gref
302	};
303
304	if (len > XEN_NETBK_MAX_HASH_KEY_SIZE)
305		return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
306
307	if (copy_op.len != 0) {
308		gnttab_batch_copy(&copy_op, 1);
309
310		if (copy_op.status != GNTST_okay)
311			return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
312	}
313
314	/* Clear any remaining key octets */
315	if (len < XEN_NETBK_MAX_HASH_KEY_SIZE)
316		memset(key + len, 0, XEN_NETBK_MAX_HASH_KEY_SIZE - len);
317
318	xenvif_flush_hash(vif);
319
320	return XEN_NETIF_CTRL_STATUS_SUCCESS;
321}
322
323u32 xenvif_set_hash_mapping_size(struct xenvif *vif, u32 size)
324{
325	if (size > XEN_NETBK_MAX_HASH_MAPPING_SIZE)
326		return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
327
328	vif->hash.size = size;
329	memset(vif->hash.mapping[vif->hash.mapping_sel], 0,
330	       sizeof(u32) * size);
331
332	return XEN_NETIF_CTRL_STATUS_SUCCESS;
333}
334
335u32 xenvif_set_hash_mapping(struct xenvif *vif, u32 gref, u32 len,
336			    u32 off)
337{
338	u32 *mapping = vif->hash.mapping[!vif->hash.mapping_sel];
339	unsigned int nr = 1;
340	struct gnttab_copy copy_op[2] = {{
341		.source.u.ref = gref,
342		.source.domid = vif->domid,
343		.dest.domid = DOMID_SELF,
344		.len = len * sizeof(*mapping),
345		.flags = GNTCOPY_source_gref
346	}};
347
348	if ((off + len < off) || (off + len > vif->hash.size) ||
349	    len > XEN_PAGE_SIZE / sizeof(*mapping))
350		return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
351
352	copy_op[0].dest.u.gmfn = virt_to_gfn(mapping + off);
353	copy_op[0].dest.offset = xen_offset_in_page(mapping + off);
354	if (copy_op[0].dest.offset + copy_op[0].len > XEN_PAGE_SIZE) {
355		copy_op[1] = copy_op[0];
356		copy_op[1].source.offset = XEN_PAGE_SIZE - copy_op[0].dest.offset;
357		copy_op[1].dest.u.gmfn = virt_to_gfn(mapping + off + len);
358		copy_op[1].dest.offset = 0;
359		copy_op[1].len = copy_op[0].len - copy_op[1].source.offset;
360		copy_op[0].len = copy_op[1].source.offset;
361		nr = 2;
362	}
363
364	memcpy(mapping, vif->hash.mapping[vif->hash.mapping_sel],
365	       vif->hash.size * sizeof(*mapping));
366
367	if (copy_op[0].len != 0) {
368		gnttab_batch_copy(copy_op, nr);
369
370		if (copy_op[0].status != GNTST_okay ||
371		    copy_op[nr - 1].status != GNTST_okay)
372			return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
373	}
374
375	while (len-- != 0)
376		if (mapping[off++] >= vif->num_queues)
377			return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER;
378
379	vif->hash.mapping_sel = !vif->hash.mapping_sel;
380
381	return XEN_NETIF_CTRL_STATUS_SUCCESS;
382}
383
384#ifdef CONFIG_DEBUG_FS
385void xenvif_dump_hash_info(struct xenvif *vif, struct seq_file *m)
386{
387	unsigned int i;
388
389	switch (vif->hash.alg) {
390	case XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ:
391		seq_puts(m, "Hash Algorithm: TOEPLITZ\n");
392		break;
393
394	case XEN_NETIF_CTRL_HASH_ALGORITHM_NONE:
395		seq_puts(m, "Hash Algorithm: NONE\n");
396		fallthrough;
397	default:
398		return;
399	}
400
401	if (vif->hash.flags) {
402		seq_puts(m, "\nHash Flags:\n");
403
404		if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4)
405			seq_puts(m, "- IPv4\n");
406		if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP)
407			seq_puts(m, "- IPv4 + TCP\n");
408		if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6)
409			seq_puts(m, "- IPv6\n");
410		if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)
411			seq_puts(m, "- IPv6 + TCP\n");
412	}
413
414	seq_puts(m, "\nHash Key:\n");
415
416	for (i = 0; i < XEN_NETBK_MAX_HASH_KEY_SIZE; ) {
417		unsigned int j, n;
418
419		n = 8;
420		if (i + n >= XEN_NETBK_MAX_HASH_KEY_SIZE)
421			n = XEN_NETBK_MAX_HASH_KEY_SIZE - i;
422
423		seq_printf(m, "[%2u - %2u]: ", i, i + n - 1);
424
425		for (j = 0; j < n; j++, i++)
426			seq_printf(m, "%02x ", vif->hash.key[i]);
427
428		seq_puts(m, "\n");
429	}
430
431	if (vif->hash.size != 0) {
432		const u32 *mapping = vif->hash.mapping[vif->hash.mapping_sel];
433
434		seq_puts(m, "\nHash Mapping:\n");
435
436		for (i = 0; i < vif->hash.size; ) {
437			unsigned int j, n;
438
439			n = 8;
440			if (i + n >= vif->hash.size)
441				n = vif->hash.size - i;
442
443			seq_printf(m, "[%4u - %4u]: ", i, i + n - 1);
444
445			for (j = 0; j < n; j++, i++)
446				seq_printf(m, "%4u ", mapping[i]);
447
448			seq_puts(m, "\n");
449		}
450	}
451}
452#endif /* CONFIG_DEBUG_FS */
453
454void xenvif_init_hash(struct xenvif *vif)
455{
456	if (xenvif_hash_cache_size == 0)
457		return;
458
459	BUG_ON(vif->hash.cache.count);
460
461	spin_lock_init(&vif->hash.cache.lock);
462	INIT_LIST_HEAD(&vif->hash.cache.list);
463}
464
465void xenvif_deinit_hash(struct xenvif *vif)
466{
467	xenvif_flush_hash(vif);
468}
469