1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32#include <sys/cdefs.h>
33#include "opt_rss.h"
34
35#include "ena_rss.h"
36
37/*
38 * This function should generate unique key for the whole driver.
39 * If the key was already genereated in the previous call (for example
40 * for another adapter), then it should be returned instead.
41 */
42void
43ena_rss_key_fill(void *key, size_t size)
44{
45	static bool key_generated;
46	static uint8_t default_key[ENA_HASH_KEY_SIZE];
47
48	KASSERT(size <= ENA_HASH_KEY_SIZE,
49	    ("Requested more bytes than ENA RSS key can hold"));
50
51	if (!key_generated) {
52		arc4random_buf(default_key, ENA_HASH_KEY_SIZE);
53		key_generated = true;
54	}
55
56	memcpy(key, default_key, size);
57}
58
59/*
60 * ENA HW expects the key to be in reverse-byte order.
61 */
62static void
63ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size)
64{
65	int i;
66
67	key = key + key_size - 1;
68
69	for (i = 0; i < key_size; ++i)
70		*reordered_key++ = *key--;
71}
72
73int
74ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key)
75{
76	enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ;
77	u8 hw_key[ENA_HASH_KEY_SIZE];
78
79	ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE);
80
81	return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key,
82	    ENA_HASH_KEY_SIZE, 0x0));
83}
84
85int
86ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key)
87{
88	u8 hw_key[ENA_HASH_KEY_SIZE];
89	int rc;
90
91	rc = ena_com_get_hash_key(ena_dev, hw_key);
92	if (rc != 0)
93		return rc;
94
95	ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE);
96
97	return (0);
98}
99
100static int
101ena_rss_init_default(struct ena_adapter *adapter)
102{
103	struct ena_com_dev *ena_dev = adapter->ena_dev;
104	device_t dev = adapter->pdev;
105	int qid, rc, i;
106
107	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
108	if (unlikely(rc != 0)) {
109		ena_log(dev, ERR, "Cannot init indirect table\n");
110		return (rc);
111	}
112
113	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
114#ifdef RSS
115		qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues;
116#else
117		qid = i % adapter->num_io_queues;
118#endif
119		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
120		    ENA_IO_RXQ_IDX(qid));
121		if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
122			ena_log(dev, ERR, "Cannot fill indirect table\n");
123			goto err_rss_destroy;
124		}
125	}
126
127
128#ifdef RSS
129	uint8_t rss_algo = rss_gethashalgo();
130	if (rss_algo == RSS_HASH_TOEPLITZ) {
131		uint8_t hash_key[RSS_KEYSIZE];
132
133		rss_getkey(hash_key);
134		rc = ena_rss_set_hash(ena_dev, hash_key);
135	} else
136#endif
137		rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ,
138		    NULL, ENA_HASH_KEY_SIZE, 0x0);
139	if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
140		ena_log(dev, ERR, "Cannot fill hash function\n");
141		goto err_rss_destroy;
142	}
143
144	rc = ena_com_set_default_hash_ctrl(ena_dev);
145	if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
146		ena_log(dev, ERR, "Cannot fill hash control\n");
147		goto err_rss_destroy;
148	}
149
150	rc = ena_rss_indir_init(adapter);
151
152	return (rc == EOPNOTSUPP ? 0 : rc);
153
154err_rss_destroy:
155	ena_com_rss_destroy(ena_dev);
156	return (rc);
157}
158
159/* Configure the Rx forwarding */
160int
161ena_rss_configure(struct ena_adapter *adapter)
162{
163	struct ena_com_dev *ena_dev = adapter->ena_dev;
164	int rc;
165
166	/* In case the RSS table was destroyed */
167	if (!ena_dev->rss.tbl_log_size) {
168		rc = ena_rss_init_default(adapter);
169		if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
170			ena_log(adapter->pdev, ERR,
171			    "WARNING: RSS was not properly re-initialized,"
172			    " it will affect bandwidth\n");
173			ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
174			return (rc);
175		}
176	}
177
178	/* Set indirect table */
179	rc = ena_com_indirect_table_set(ena_dev);
180	if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
181		return (rc);
182
183	/* Configure hash function (if supported) */
184	rc = ena_com_set_hash_function(ena_dev);
185	if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
186		return (rc);
187
188	/* Configure hash inputs (if supported) */
189	rc = ena_com_set_hash_ctrl(ena_dev);
190	if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
191		return (rc);
192
193	return (0);
194}
195
196static void
197ena_rss_init_default_deferred(void *arg)
198{
199	struct ena_adapter *adapter;
200	devclass_t dc;
201	int max;
202	int rc;
203
204	dc = devclass_find("ena");
205	if (unlikely(dc == NULL)) {
206		ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__);
207		return;
208	}
209
210	max = devclass_get_maxunit(dc);
211	while (max-- >= 0) {
212		adapter = devclass_get_softc(dc, max);
213		if (adapter != NULL) {
214			rc = ena_rss_init_default(adapter);
215			ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
216			if (unlikely(rc != 0)) {
217				ena_log(adapter->pdev, WARN,
218				    "WARNING: RSS was not properly initialized,"
219				    " it will affect bandwidth\n");
220				ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE,
221				    adapter);
222			}
223		}
224	}
225}
226SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND,
227    ena_rss_init_default_deferred, NULL);
228
229int
230ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table)
231{
232	int rc, i;
233
234	rc = ena_com_indirect_table_get(adapter->ena_dev, table);
235	if (rc != 0) {
236		if (rc == EOPNOTSUPP)
237			device_printf(adapter->pdev,
238			    "Reading from indirection table not supported\n");
239		else
240			device_printf(adapter->pdev,
241			    "Unable to get indirection table\n");
242		return (rc);
243	}
244
245	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i)
246		table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]);
247
248	return (0);
249}
250
251int
252ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table)
253{
254	int rc, i;
255
256	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) {
257		rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i,
258		    ENA_IO_RXQ_IDX(table[i]));
259		if (rc != 0) {
260			device_printf(adapter->pdev,
261			    "Cannot fill indirection table entry %d\n", i);
262			return (rc);
263		}
264	}
265
266	rc = ena_com_indirect_table_set(adapter->ena_dev);
267	if (rc == EOPNOTSUPP)
268		device_printf(adapter->pdev,
269		    "Writing to indirection table not supported\n");
270	else if (rc != 0)
271		device_printf(adapter->pdev, "Cannot set indirection table\n");
272
273	return (rc);
274}
275
276int
277ena_rss_indir_init(struct ena_adapter *adapter)
278{
279	struct ena_indir *indir = adapter->rss_indir;
280	int rc;
281
282	if (indir == NULL) {
283		adapter->rss_indir = indir = malloc(sizeof(struct ena_indir),
284		    M_DEVBUF, M_WAITOK | M_ZERO);
285		if (indir == NULL)
286			return (ENOMEM);
287	}
288
289	rc = ena_rss_indir_get(adapter, indir->table);
290	if (rc != 0) {
291		free(adapter->rss_indir, M_DEVBUF);
292		adapter->rss_indir = NULL;
293
294		return (rc);
295	}
296
297	ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table);
298
299	return (0);
300}
301