1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: stable/10/sys/dev/ioat/ioat_test.c 315071 2017-03-11 15:26:41Z avg $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/bus.h>
33#include <sys/conf.h>
34#include <sys/ioccom.h>
35#include <sys/kernel.h>
36#include <sys/lock.h>
37#include <sys/malloc.h>
38#include <sys/module.h>
39#include <sys/mutex.h>
40#include <sys/rman.h>
41#include <sys/sysctl.h>
42#include <dev/pci/pcireg.h>
43#include <dev/pci/pcivar.h>
44#include <machine/bus.h>
45#include <machine/resource.h>
46#include <machine/stdarg.h>
47#include <vm/vm.h>
48#include <vm/vm_param.h>
49#include <vm/pmap.h>
50
51#include "ioat.h"
52#include "ioat_hw.h"
53#include "ioat_internal.h"
54#include "ioat_test.h"
55
56#ifndef time_after
57#define	time_after(a,b)		((long)(b) - (long)(a) < 0)
58#endif
59
60MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations");
61
62#define	IOAT_MAX_BUFS	256
63
64struct test_transaction {
65	void			*buf[IOAT_MAX_BUFS];
66	uint32_t		length;
67	uint32_t		depth;
68	struct ioat_test	*test;
69	TAILQ_ENTRY(test_transaction)	entry;
70};
71
72#define	IT_LOCK()	mtx_lock(&ioat_test_lk)
73#define	IT_UNLOCK()	mtx_unlock(&ioat_test_lk)
74#define	IT_ASSERT()	mtx_assert(&ioat_test_lk, MA_OWNED)
75static struct mtx ioat_test_lk;
76MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF);
77
78static int g_thread_index = 1;
79static struct cdev *g_ioat_cdev = NULL;
80
81#define	ioat_test_log(v, ...)	_ioat_test_log((v), "ioat_test: " __VA_ARGS__)
82static void _ioat_test_log(int verbosity, const char *fmt, ...);
83
84static void
85ioat_test_transaction_destroy(struct test_transaction *tx)
86{
87	struct ioat_test *test;
88	int i;
89
90	test = tx->test;
91
92	for (i = 0; i < IOAT_MAX_BUFS; i++) {
93		if (tx->buf[i] != NULL) {
94			if (test->testkind == IOAT_TEST_DMA_8K)
95				free(tx->buf[i], M_IOAT_TEST);
96			else
97				contigfree(tx->buf[i], tx->length, M_IOAT_TEST);
98			tx->buf[i] = NULL;
99		}
100	}
101
102	free(tx, M_IOAT_TEST);
103}
104
105static struct
106test_transaction *ioat_test_transaction_create(struct ioat_test *test,
107    unsigned num_buffers)
108{
109	struct test_transaction *tx;
110	unsigned i;
111
112	tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO);
113	if (tx == NULL)
114		return (NULL);
115
116	tx->length = test->buffer_size;
117
118	for (i = 0; i < num_buffers; i++) {
119		if (test->testkind == IOAT_TEST_DMA_8K)
120			tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST,
121			    M_NOWAIT);
122		else
123			tx->buf[i] = contigmalloc(test->buffer_size,
124			    M_IOAT_TEST, M_NOWAIT, 0, BUS_SPACE_MAXADDR,
125			    PAGE_SIZE, 0);
126
127		if (tx->buf[i] == NULL) {
128			ioat_test_transaction_destroy(tx);
129			return (NULL);
130		}
131	}
132	return (tx);
133}
134
135static void
136dump_hex(void *p, size_t chunks)
137{
138	size_t i, j;
139
140	for (i = 0; i < chunks; i++) {
141		for (j = 0; j < 8; j++)
142			printf("%08x ", ((uint32_t *)p)[i * 8 + j]);
143		printf("\n");
144	}
145}
146
147static bool
148ioat_compare_ok(struct test_transaction *tx)
149{
150	struct ioat_test *test;
151	char *dst, *src;
152	uint32_t i, j;
153
154	test = tx->test;
155
156	for (i = 0; i < tx->depth; i++) {
157		dst = tx->buf[2 * i + 1];
158		src = tx->buf[2 * i];
159
160		if (test->testkind == IOAT_TEST_FILL) {
161			for (j = 0; j < tx->length; j += sizeof(uint64_t)) {
162				if (memcmp(src, &dst[j],
163					MIN(sizeof(uint64_t), tx->length - j))
164				    != 0)
165					return (false);
166			}
167		} else if (test->testkind == IOAT_TEST_DMA) {
168			if (memcmp(src, dst, tx->length) != 0)
169				return (false);
170		} else if (test->testkind == IOAT_TEST_RAW_DMA) {
171			if (test->raw_write)
172				dst = test->raw_vtarget;
173			dump_hex(dst, tx->length / 32);
174		}
175	}
176	return (true);
177}
178
179static void
180ioat_dma_test_callback(void *arg, int error)
181{
182	struct test_transaction *tx;
183	struct ioat_test *test;
184
185	if (error != 0)
186		ioat_test_log(0, "%s: Got error: %d\n", __func__, error);
187
188	tx = arg;
189	test = tx->test;
190
191	if (test->verify && !ioat_compare_ok(tx)) {
192		ioat_test_log(0, "miscompare found\n");
193		atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);
194	} else if (!test->too_late)
195		atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);
196
197	IT_LOCK();
198	TAILQ_REMOVE(&test->pend_q, tx, entry);
199	TAILQ_INSERT_TAIL(&test->free_q, tx, entry);
200	wakeup(&test->free_q);
201	IT_UNLOCK();
202}
203
204static int
205ioat_test_prealloc_memory(struct ioat_test *test, int index)
206{
207	uint32_t i, j, k;
208	struct test_transaction *tx;
209
210	for (i = 0; i < test->transactions; i++) {
211		tx = ioat_test_transaction_create(test, test->chain_depth * 2);
212		if (tx == NULL) {
213			ioat_test_log(0, "tx == NULL - memory exhausted\n");
214			test->status[IOAT_TEST_NO_MEMORY]++;
215			return (ENOMEM);
216		}
217
218		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);
219
220		tx->test = test;
221		tx->depth = test->chain_depth;
222
223		/* fill in source buffers */
224		for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) {
225			uint32_t val = j + (index << 28);
226
227			for (k = 0; k < test->chain_depth; k++) {
228				((uint32_t *)tx->buf[2*k])[j] = ~val;
229				((uint32_t *)tx->buf[2*k+1])[j] = val;
230			}
231		}
232	}
233	return (0);
234}
235
236static void
237ioat_test_release_memory(struct ioat_test *test)
238{
239	struct test_transaction *tx, *s;
240
241	TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)
242		ioat_test_transaction_destroy(tx);
243	TAILQ_INIT(&test->free_q);
244
245	TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)
246		ioat_test_transaction_destroy(tx);
247	TAILQ_INIT(&test->pend_q);
248}
249
250static void
251ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma)
252{
253	struct test_transaction *tx;
254	struct bus_dmadesc *desc;
255	bus_dmaengine_callback_t cb;
256	bus_addr_t src, dest;
257	uint64_t fillpattern;
258	uint32_t i, flags;
259
260	desc = NULL;
261
262	IT_LOCK();
263	while (TAILQ_EMPTY(&test->free_q))
264		msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0);
265
266	tx = TAILQ_FIRST(&test->free_q);
267	TAILQ_REMOVE(&test->free_q, tx, entry);
268	TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);
269	IT_UNLOCK();
270
271	if (test->testkind != IOAT_TEST_MEMCPY)
272		ioat_acquire(dma);
273	for (i = 0; i < tx->depth; i++) {
274		if (test->testkind == IOAT_TEST_MEMCPY) {
275			memcpy(tx->buf[2 * i + 1], tx->buf[2 * i], tx->length);
276			if (i == tx->depth - 1)
277				ioat_dma_test_callback(tx, 0);
278			continue;
279		}
280
281		src = vtophys((vm_offset_t)tx->buf[2*i]);
282		dest = vtophys((vm_offset_t)tx->buf[2*i+1]);
283
284		if (test->testkind == IOAT_TEST_RAW_DMA) {
285			if (test->raw_write)
286				dest = test->raw_target;
287			else
288				src = test->raw_target;
289		}
290
291		if (i == tx->depth - 1) {
292			cb = ioat_dma_test_callback;
293			flags = DMA_INT_EN;
294		} else {
295			cb = NULL;
296			flags = 0;
297		}
298
299		if (test->testkind == IOAT_TEST_DMA ||
300		    test->testkind == IOAT_TEST_RAW_DMA)
301			desc = ioat_copy(dma, dest, src, tx->length, cb, tx,
302			    flags);
303		else if (test->testkind == IOAT_TEST_FILL) {
304			fillpattern = *(uint64_t *)tx->buf[2*i];
305			desc = ioat_blockfill(dma, dest, fillpattern,
306			    tx->length, cb, tx, flags);
307		} else if (test->testkind == IOAT_TEST_DMA_8K) {
308			bus_addr_t src2, dst2;
309
310			src2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE);
311			dst2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE);
312
313			desc = ioat_copy_8k_aligned(dma, dest, dst2, src, src2,
314			    cb, tx, flags);
315		}
316		if (desc == NULL)
317			break;
318	}
319	if (test->testkind == IOAT_TEST_MEMCPY)
320		return;
321	ioat_release(dma);
322
323	/*
324	 * We couldn't issue an IO -- either the device is being detached or
325	 * the HW reset.  Essentially spin until the device comes back up or
326	 * our timer expires.
327	 */
328	if (desc == NULL && tx->depth > 0) {
329		atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth);
330		IT_LOCK();
331		TAILQ_REMOVE(&test->pend_q, tx, entry);
332		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);
333		IT_UNLOCK();
334	}
335}
336
337static void
338ioat_dma_test(void *arg)
339{
340	struct ioat_softc *ioat;
341	struct ioat_test *test;
342	bus_dmaengine_t dmaengine;
343	uint32_t loops;
344	int index, rc, start, end, error;
345
346	test = arg;
347	memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status));
348
349	if (test->testkind == IOAT_TEST_DMA_8K &&
350	    test->buffer_size != 2 * PAGE_SIZE) {
351		ioat_test_log(0, "Asked for 8k test and buffer size isn't 8k\n");
352		test->status[IOAT_TEST_INVALID_INPUT]++;
353		return;
354	}
355
356	if (test->buffer_size > 1024 * 1024) {
357		ioat_test_log(0, "Buffer size too large >1MB\n");
358		test->status[IOAT_TEST_NO_MEMORY]++;
359		return;
360	}
361
362	if (test->chain_depth * 2 > IOAT_MAX_BUFS) {
363		ioat_test_log(0, "Depth too large (> %u)\n",
364		    (unsigned)IOAT_MAX_BUFS / 2);
365		test->status[IOAT_TEST_NO_MEMORY]++;
366		return;
367	}
368
369	if (btoc((uint64_t)test->buffer_size * test->chain_depth *
370	    test->transactions) > (physmem / 4)) {
371		ioat_test_log(0, "Sanity check failed -- test would "
372		    "use more than 1/4 of phys mem.\n");
373		test->status[IOAT_TEST_NO_MEMORY]++;
374		return;
375	}
376
377	if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) {
378		ioat_test_log(0, "Sanity check failed -- test would "
379		    "use more than available IOAT ring space.\n");
380		test->status[IOAT_TEST_NO_MEMORY]++;
381		return;
382	}
383
384	if (test->testkind >= IOAT_NUM_TESTKINDS) {
385		ioat_test_log(0, "Invalid kind %u\n",
386		    (unsigned)test->testkind);
387		test->status[IOAT_TEST_INVALID_INPUT]++;
388		return;
389	}
390
391	dmaengine = ioat_get_dmaengine(test->channel_index, M_NOWAIT);
392	if (dmaengine == NULL) {
393		ioat_test_log(0, "Couldn't acquire dmaengine\n");
394		test->status[IOAT_TEST_NO_DMA_ENGINE]++;
395		return;
396	}
397	ioat = to_ioat_softc(dmaengine);
398
399	if (test->testkind == IOAT_TEST_FILL &&
400	    (ioat->capabilities & IOAT_DMACAP_BFILL) == 0)
401	{
402		ioat_test_log(0,
403		    "Hardware doesn't support block fill, aborting test\n");
404		test->status[IOAT_TEST_INVALID_INPUT]++;
405		goto out;
406	}
407
408	if (test->coalesce_period > ioat->intrdelay_max) {
409		ioat_test_log(0,
410		    "Hardware doesn't support intrdelay of %u us.\n",
411		    (unsigned)test->coalesce_period);
412		test->status[IOAT_TEST_INVALID_INPUT]++;
413		goto out;
414	}
415	error = ioat_set_interrupt_coalesce(dmaengine, test->coalesce_period);
416	if (error == ENODEV && test->coalesce_period == 0)
417		error = 0;
418	if (error != 0) {
419		ioat_test_log(0, "ioat_set_interrupt_coalesce: %d\n", error);
420		test->status[IOAT_TEST_INVALID_INPUT]++;
421		goto out;
422	}
423
424	if (test->zero_stats)
425		memset(&ioat->stats, 0, sizeof(ioat->stats));
426
427	if (test->testkind == IOAT_TEST_RAW_DMA) {
428		if (test->raw_is_virtual) {
429			test->raw_vtarget = (void *)test->raw_target;
430			test->raw_target = vtophys(test->raw_vtarget);
431		} else {
432			test->raw_vtarget = pmap_mapdev(test->raw_target,
433			    test->buffer_size);
434		}
435	}
436
437	index = g_thread_index++;
438	TAILQ_INIT(&test->free_q);
439	TAILQ_INIT(&test->pend_q);
440
441	if (test->duration == 0)
442		ioat_test_log(1, "Thread %d: num_loops remaining: 0x%08x\n",
443		    index, test->transactions);
444	else
445		ioat_test_log(1, "Thread %d: starting\n", index);
446
447	rc = ioat_test_prealloc_memory(test, index);
448	if (rc != 0) {
449		ioat_test_log(0, "prealloc_memory: %d\n", rc);
450		goto out;
451	}
452	wmb();
453
454	test->too_late = false;
455	start = ticks;
456	end = start + (((sbintime_t)test->duration * hz) / 1000);
457
458	for (loops = 0;; loops++) {
459		if (test->duration == 0 && loops >= test->transactions)
460			break;
461		else if (test->duration != 0 && time_after(ticks, end)) {
462			test->too_late = true;
463			break;
464		}
465
466		ioat_test_submit_1_tx(test, dmaengine);
467	}
468
469	ioat_test_log(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n",
470	    ticks - start, ticks - end, (ticks - start) / hz);
471
472	IT_LOCK();
473	while (!TAILQ_EMPTY(&test->pend_q))
474		msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz);
475	IT_UNLOCK();
476
477	ioat_test_log(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n",
478	    ticks - start, ticks - end, (ticks - start) / hz);
479
480	ioat_test_release_memory(test);
481out:
482	if (test->testkind == IOAT_TEST_RAW_DMA && !test->raw_is_virtual)
483		pmap_unmapdev((vm_offset_t)test->raw_vtarget,
484		    test->buffer_size);
485	ioat_put_dmaengine(dmaengine);
486}
487
488static int
489ioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td)
490{
491
492	return (0);
493}
494
495static int
496ioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td)
497{
498
499	return (0);
500}
501
502static int
503ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag,
504    struct thread *td)
505{
506
507	switch (cmd) {
508	case IOAT_DMATEST:
509		ioat_dma_test(arg);
510		break;
511	default:
512		return (EINVAL);
513	}
514	return (0);
515}
516
517static struct cdevsw ioat_cdevsw = {
518	.d_version =	D_VERSION,
519	.d_flags =	0,
520	.d_open =	ioat_test_open,
521	.d_close =	ioat_test_close,
522	.d_ioctl =	ioat_test_ioctl,
523	.d_name =	"ioat_test",
524};
525
526static int
527enable_ioat_test(bool enable)
528{
529
530	mtx_assert(&Giant, MA_OWNED);
531
532	if (enable && g_ioat_cdev == NULL) {
533		g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL,
534		    0600, "ioat_test");
535	} else if (!enable && g_ioat_cdev != NULL) {
536		destroy_dev(g_ioat_cdev);
537		g_ioat_cdev = NULL;
538	}
539	return (0);
540}
541
542static int
543sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS)
544{
545	int error, enabled;
546
547	enabled = (g_ioat_cdev != NULL);
548	error = sysctl_handle_int(oidp, &enabled, 0, req);
549	if (error != 0 || req->newptr == NULL)
550		return (error);
551
552	enable_ioat_test(enabled);
553	return (0);
554}
555SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_INT | CTLFLAG_RW,
556    0, 0, sysctl_enable_ioat_test, "I",
557    "Non-zero: Enable the /dev/ioat_test device");
558
559void
560ioat_test_attach(void)
561{
562#ifdef notyet
563	char *val;
564
565	val = kern_getenv("hw.ioat.enable_ioat_test");
566	if (val != NULL && strcmp(val, "0") != 0) {
567#else
568	int val = 0;
569
570	TUNABLE_INT_FETCH("hw.ioat.enable_ioat_test", &val);
571	if (val != 0) {
572#endif
573		mtx_lock(&Giant);
574		enable_ioat_test(true);
575		mtx_unlock(&Giant);
576	}
577#ifdef notyet
578	freeenv(val);
579#endif
580}
581
582void
583ioat_test_detach(void)
584{
585
586	mtx_lock(&Giant);
587	enable_ioat_test(false);
588	mtx_unlock(&Giant);
589}
590
591static void
592_ioat_test_log(int verbosity, const char *fmt, ...)
593{
594	va_list argp;
595
596	if (verbosity > g_ioat_debug_level)
597		return;
598
599	va_start(argp, fmt);
600	vprintf(fmt, argp);
601	va_end(argp);
602}
603