ioat_test.c revision 289733 
1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/dev/ioat/ioat_test.c 289733 2015-10-22 04:38:05Z cem $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/bus.h>
33#include <sys/conf.h>
34#include <sys/ioccom.h>
35#include <sys/kernel.h>
36#include <sys/lock.h>
37#include <sys/malloc.h>
38#include <sys/module.h>
39#include <sys/mutex.h>
40#include <sys/rman.h>
41#include <sys/sysctl.h>
42#include <dev/pci/pcireg.h>
43#include <dev/pci/pcivar.h>
44#include <machine/bus.h>
45#include <machine/resource.h>
46#include <vm/vm.h>
47#include <vm/pmap.h>
48
49#include "ioat.h"
50#include "ioat_hw.h"
51#include "ioat_internal.h"
52#include "ioat_test.h"
53
54#ifndef time_after
55#define	time_after(a,b)		((long)(b) - (long)(a) < 0)
56#endif
57
58MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations");
59
60#define	IOAT_MAX_BUFS	256
61
62struct test_transaction {
63	void			*buf[IOAT_MAX_BUFS];
64	uint32_t		length;
65	uint32_t		depth;
66	struct ioat_test	*test;
67	TAILQ_ENTRY(test_transaction)	entry;
68};
69
70#define	IT_LOCK()	mtx_lock(&ioat_test_lk)
71#define	IT_UNLOCK()	mtx_unlock(&ioat_test_lk)
72#define	IT_ASSERT()	mtx_assert(&ioat_test_lk, MA_OWNED)
73static struct mtx ioat_test_lk;
74MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF);
75
76static int g_thread_index = 1;
77static struct cdev *g_ioat_cdev = NULL;
78
79static void
80ioat_test_transaction_destroy(struct test_transaction *tx)
81{
82	int i;
83
84	for (i = 0; i < IOAT_MAX_BUFS; i++) {
85		if (tx->buf[i] != NULL) {
86			contigfree(tx->buf[i], tx->length, M_IOAT_TEST);
87			tx->buf[i] = NULL;
88		}
89	}
90
91	free(tx, M_IOAT_TEST);
92}
93
94static struct
95test_transaction *ioat_test_transaction_create(unsigned num_buffers,
96    uint32_t buffer_size)
97{
98	struct test_transaction *tx;
99	unsigned i;
100
101	tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO);
102	if (tx == NULL)
103		return (NULL);
104
105	tx->length = buffer_size;
106
107	for (i = 0; i < num_buffers; i++) {
108		tx->buf[i] = contigmalloc(buffer_size, M_IOAT_TEST, M_NOWAIT,
109		    0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
110
111		if (tx->buf[i] == NULL) {
112			ioat_test_transaction_destroy(tx);
113			return (NULL);
114		}
115	}
116	return (tx);
117}
118
119static bool
120ioat_compare_ok(struct test_transaction *tx)
121{
122	uint32_t i;
123
124	for (i = 0; i < tx->depth; i++) {
125		if (memcmp(tx->buf[2*i], tx->buf[2*i+1], tx->length) != 0)
126			return (false);
127	}
128	return (true);
129}
130
131static void
132ioat_dma_test_callback(void *arg)
133{
134	struct test_transaction *tx;
135	struct ioat_test *test;
136
137	tx = arg;
138	test = tx->test;
139
140	if (test->verify && !ioat_compare_ok(tx)) {
141		ioat_log_message(0, "miscompare found\n");
142		atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);
143	} else if (!test->too_late)
144		atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);
145
146	IT_LOCK();
147	TAILQ_REMOVE(&test->pend_q, tx, entry);
148	TAILQ_INSERT_TAIL(&test->free_q, tx, entry);
149	wakeup(&test->free_q);
150	IT_UNLOCK();
151}
152
153static int
154ioat_test_prealloc_memory(struct ioat_test *test, int index)
155{
156	uint32_t i, j, k;
157	struct test_transaction *tx;
158
159	for (i = 0; i < test->transactions; i++) {
160		tx = ioat_test_transaction_create(test->chain_depth * 2,
161		    test->buffer_size);
162		if (tx == NULL) {
163			ioat_log_message(0, "tx == NULL - memory exhausted\n");
164			test->status[IOAT_TEST_NO_MEMORY]++;
165			return (ENOMEM);
166		}
167
168		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);
169
170		tx->test = test;
171		tx->depth = test->chain_depth;
172
173		/* fill in source buffers */
174		for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) {
175			uint32_t val = j + (index << 28);
176
177			for (k = 0; k < test->chain_depth; k++) {
178				((uint32_t *)tx->buf[2*k])[j] = ~val;
179				((uint32_t *)tx->buf[2*k+1])[j] = val;
180			}
181		}
182	}
183	return (0);
184}
185
186static void
187ioat_test_release_memory(struct ioat_test *test)
188{
189	struct test_transaction *tx, *s;
190
191	TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)
192		ioat_test_transaction_destroy(tx);
193	TAILQ_INIT(&test->free_q);
194
195	TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)
196		ioat_test_transaction_destroy(tx);
197	TAILQ_INIT(&test->pend_q);
198}
199
200static void
201ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma)
202{
203	struct test_transaction *tx;
204	struct bus_dmadesc *desc;
205	bus_dmaengine_callback_t cb;
206	bus_addr_t src, dest;
207	uint32_t i, flags;
208
209	IT_LOCK();
210	while (TAILQ_EMPTY(&test->free_q))
211		msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0);
212
213	tx = TAILQ_FIRST(&test->free_q);
214	TAILQ_REMOVE(&test->free_q, tx, entry);
215	TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);
216	IT_UNLOCK();
217
218	ioat_acquire(dma);
219	for (i = 0; i < tx->depth; i++) {
220		src = vtophys((vm_offset_t)tx->buf[2*i]);
221		dest = vtophys((vm_offset_t)tx->buf[2*i+1]);
222
223		if (i == tx->depth - 1) {
224			cb = ioat_dma_test_callback;
225			flags = DMA_INT_EN;
226		} else {
227			cb = NULL;
228			flags = 0;
229		}
230
231		desc = ioat_copy(dma, src, dest, tx->length, cb, tx, flags);
232		if (desc == NULL)
233			panic("Failed to allocate a ring slot "
234			    "-- this shouldn't happen!");
235	}
236	ioat_release(dma);
237}
238
239static void
240ioat_dma_test(void *arg)
241{
242	struct ioat_test *test;
243	bus_dmaengine_t dmaengine;
244	uint32_t loops;
245	int index, rc, start, end;
246
247	test = arg;
248	memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status));
249
250	if (test->buffer_size > 1024 * 1024) {
251		ioat_log_message(0, "Buffer size too large >1MB\n");
252		test->status[IOAT_TEST_NO_MEMORY]++;
253		return;
254	}
255
256	if (test->chain_depth * 2 > IOAT_MAX_BUFS) {
257		ioat_log_message(0, "Depth too large (> %u)\n",
258		    (unsigned)IOAT_MAX_BUFS / 2);
259		test->status[IOAT_TEST_NO_MEMORY]++;
260		return;
261	}
262
263	if (btoc((uint64_t)test->buffer_size * test->chain_depth *
264	    test->transactions) > (physmem / 4)) {
265		ioat_log_message(0, "Sanity check failed -- test would "
266		    "use more than 1/4 of phys mem.\n");
267		test->status[IOAT_TEST_NO_MEMORY]++;
268		return;
269	}
270
271	if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) {
272		ioat_log_message(0, "Sanity check failed -- test would "
273		    "use more than available IOAT ring space.\n");
274		test->status[IOAT_TEST_NO_MEMORY]++;
275		return;
276	}
277
278	dmaengine = ioat_get_dmaengine(test->channel_index);
279	if (dmaengine == NULL) {
280		ioat_log_message(0, "Couldn't acquire dmaengine\n");
281		test->status[IOAT_TEST_NO_DMA_ENGINE]++;
282		return;
283	}
284
285	index = g_thread_index++;
286	TAILQ_INIT(&test->free_q);
287	TAILQ_INIT(&test->pend_q);
288
289	if (test->duration == 0)
290		ioat_log_message(1, "Thread %d: num_loops remaining: 0x%08x\n",
291		    index, test->transactions);
292	else
293		ioat_log_message(1, "Thread %d: starting\n", index);
294
295	rc = ioat_test_prealloc_memory(test, index);
296	if (rc != 0) {
297		ioat_log_message(0, "prealloc_memory: %d\n", rc);
298		return;
299	}
300	wmb();
301
302	test->too_late = false;
303	start = ticks;
304	end = start + (((sbintime_t)test->duration * hz) / 1000);
305
306	for (loops = 0;; loops++) {
307		if (test->duration == 0 && loops >= test->transactions)
308			break;
309		else if (test->duration != 0 && time_after(ticks, end)) {
310			test->too_late = true;
311			break;
312		}
313
314		ioat_test_submit_1_tx(test, dmaengine);
315	}
316
317	ioat_log_message(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n",
318	    ticks - start, ticks - end, (ticks - start) / hz);
319
320	IT_LOCK();
321	while (!TAILQ_EMPTY(&test->pend_q))
322		msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz);
323	IT_UNLOCK();
324
325	ioat_log_message(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n",
326	    ticks - start, ticks - end, (ticks - start) / hz);
327
328	ioat_test_release_memory(test);
329}
330
331static int
332ioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td)
333{
334
335	return (0);
336}
337
338static int
339ioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td)
340{
341
342	return (0);
343}
344
345static int
346ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag,
347    struct thread *td)
348{
349
350	switch (cmd) {
351	case IOAT_DMATEST:
352		ioat_dma_test(arg);
353		break;
354	default:
355		return (EINVAL);
356	}
357	return (0);
358}
359
360static struct cdevsw ioat_cdevsw = {
361	.d_version =	D_VERSION,
362	.d_flags =	0,
363	.d_open =	ioat_test_open,
364	.d_close =	ioat_test_close,
365	.d_ioctl =	ioat_test_ioctl,
366	.d_name =	"ioat_test",
367};
368
369static int
370sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS)
371{
372	int error, enabled;
373
374	enabled = (g_ioat_cdev != NULL);
375	error = sysctl_handle_int(oidp, &enabled, 0, req);
376	if (error != 0 || req->newptr == NULL)
377		return (error);
378
379	if (enabled != 0 && g_ioat_cdev == NULL) {
380		g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL,
381		    0600, "ioat_test");
382	} else if (enabled == 0 && g_ioat_cdev != NULL) {
383		destroy_dev(g_ioat_cdev);
384		g_ioat_cdev = NULL;
385	}
386	return (0);
387}
388SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_INT | CTLFLAG_RW,
389    0, 0, sysctl_enable_ioat_test, "I",
390    "Non-zero: Enable the /dev/ioat_test device");
391