ioat_test.c revision 289733
1287117Scem/*- 2287117Scem * Copyright (C) 2012 Intel Corporation 3287117Scem * All rights reserved. 4287117Scem * 5287117Scem * Redistribution and use in source and binary forms, with or without 6287117Scem * modification, are permitted provided that the following conditions 7287117Scem * are met: 8287117Scem * 1. Redistributions of source code must retain the above copyright 9287117Scem * notice, this list of conditions and the following disclaimer. 10287117Scem * 2. Redistributions in binary form must reproduce the above copyright 11287117Scem * notice, this list of conditions and the following disclaimer in the 12287117Scem * documentation and/or other materials provided with the distribution. 13287117Scem * 14287117Scem * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15287117Scem * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16287117Scem * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17287117Scem * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18287117Scem * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19287117Scem * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20287117Scem * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21287117Scem * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22287117Scem * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23287117Scem * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24287117Scem * SUCH DAMAGE. 25287117Scem */ 26287117Scem 27287117Scem#include <sys/cdefs.h> 28287117Scem__FBSDID("$FreeBSD: head/sys/dev/ioat/ioat_test.c 289733 2015-10-22 04:38:05Z cem $"); 29287117Scem 30287117Scem#include <sys/param.h> 31287117Scem#include <sys/systm.h> 32287117Scem#include <sys/bus.h> 33287117Scem#include <sys/conf.h> 34287117Scem#include <sys/ioccom.h> 35287117Scem#include <sys/kernel.h> 36287117Scem#include <sys/lock.h> 37287117Scem#include <sys/malloc.h> 38287117Scem#include <sys/module.h> 39287117Scem#include <sys/mutex.h> 40287117Scem#include <sys/rman.h> 41287117Scem#include <sys/sysctl.h> 42287117Scem#include <dev/pci/pcireg.h> 43287117Scem#include <dev/pci/pcivar.h> 44287117Scem#include <machine/bus.h> 45287117Scem#include <machine/resource.h> 46287117Scem#include <vm/vm.h> 47287117Scem#include <vm/pmap.h> 48287117Scem 49287117Scem#include "ioat.h" 50287117Scem#include "ioat_hw.h" 51287117Scem#include "ioat_internal.h" 52287117Scem#include "ioat_test.h" 53287117Scem 54289733Scem#ifndef time_after 55289733Scem#define time_after(a,b) ((long)(b) - (long)(a) < 0) 56289733Scem#endif 57289733Scem 58287117ScemMALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations"); 59287117Scem 60289733Scem#define IOAT_MAX_BUFS 256 61287117Scem 62287117Scemstruct test_transaction { 63287117Scem void *buf[IOAT_MAX_BUFS]; 64287117Scem uint32_t length; 65289733Scem uint32_t depth; 66287117Scem struct ioat_test *test; 67289733Scem TAILQ_ENTRY(test_transaction) entry; 68287117Scem}; 69287117Scem 70289733Scem#define IT_LOCK() mtx_lock(&ioat_test_lk) 71289733Scem#define IT_UNLOCK() mtx_unlock(&ioat_test_lk) 72289733Scem#define IT_ASSERT() mtx_assert(&ioat_test_lk, MA_OWNED) 73289733Scemstatic struct mtx ioat_test_lk; 74289733ScemMTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF); 75289733Scem 76287117Scemstatic int g_thread_index = 1; 77287117Scemstatic struct cdev *g_ioat_cdev = NULL; 78287117Scem 79287117Scemstatic void 80287117Scemioat_test_transaction_destroy(struct test_transaction *tx) 81287117Scem{ 82287117Scem int i; 83287117Scem 84287117Scem for (i = 0; i < IOAT_MAX_BUFS; i++) { 85287117Scem if (tx->buf[i] != NULL) { 86289733Scem contigfree(tx->buf[i], tx->length, M_IOAT_TEST); 87287117Scem tx->buf[i] = NULL; 88287117Scem } 89287117Scem } 90287117Scem 91287117Scem free(tx, M_IOAT_TEST); 92287117Scem} 93287117Scem 94287117Scemstatic struct 95289733Scemtest_transaction *ioat_test_transaction_create(unsigned num_buffers, 96287117Scem uint32_t buffer_size) 97287117Scem{ 98287117Scem struct test_transaction *tx; 99289733Scem unsigned i; 100287117Scem 101289733Scem tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO); 102287117Scem if (tx == NULL) 103287117Scem return (NULL); 104287117Scem 105287117Scem tx->length = buffer_size; 106287117Scem 107287117Scem for (i = 0; i < num_buffers; i++) { 108287117Scem tx->buf[i] = contigmalloc(buffer_size, M_IOAT_TEST, M_NOWAIT, 109287117Scem 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 110287117Scem 111287117Scem if (tx->buf[i] == NULL) { 112287117Scem ioat_test_transaction_destroy(tx); 113287117Scem return (NULL); 114287117Scem } 115287117Scem } 116287117Scem return (tx); 117287117Scem} 118287117Scem 119289733Scemstatic bool 120289733Scemioat_compare_ok(struct test_transaction *tx) 121289733Scem{ 122289733Scem uint32_t i; 123289733Scem 124289733Scem for (i = 0; i < tx->depth; i++) { 125289733Scem if (memcmp(tx->buf[2*i], tx->buf[2*i+1], tx->length) != 0) 126289733Scem return (false); 127289733Scem } 128289733Scem return (true); 129289733Scem} 130289733Scem 131287117Scemstatic void 132287117Scemioat_dma_test_callback(void *arg) 133287117Scem{ 134287117Scem struct test_transaction *tx; 135287117Scem struct ioat_test *test; 136287117Scem 137287117Scem tx = arg; 138287117Scem test = tx->test; 139287117Scem 140289733Scem if (test->verify && !ioat_compare_ok(tx)) { 141287117Scem ioat_log_message(0, "miscompare found\n"); 142289733Scem atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth); 143289733Scem } else if (!test->too_late) 144289733Scem atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth); 145289733Scem 146289733Scem IT_LOCK(); 147289733Scem TAILQ_REMOVE(&test->pend_q, tx, entry); 148289733Scem TAILQ_INSERT_TAIL(&test->free_q, tx, entry); 149289733Scem wakeup(&test->free_q); 150289733Scem IT_UNLOCK(); 151289733Scem} 152289733Scem 153289733Scemstatic int 154289733Scemioat_test_prealloc_memory(struct ioat_test *test, int index) 155289733Scem{ 156289733Scem uint32_t i, j, k; 157289733Scem struct test_transaction *tx; 158289733Scem 159289733Scem for (i = 0; i < test->transactions; i++) { 160289733Scem tx = ioat_test_transaction_create(test->chain_depth * 2, 161289733Scem test->buffer_size); 162289733Scem if (tx == NULL) { 163289733Scem ioat_log_message(0, "tx == NULL - memory exhausted\n"); 164289733Scem test->status[IOAT_TEST_NO_MEMORY]++; 165289733Scem return (ENOMEM); 166289733Scem } 167289733Scem 168289733Scem TAILQ_INSERT_HEAD(&test->free_q, tx, entry); 169289733Scem 170289733Scem tx->test = test; 171289733Scem tx->depth = test->chain_depth; 172289733Scem 173289733Scem /* fill in source buffers */ 174289733Scem for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) { 175289733Scem uint32_t val = j + (index << 28); 176289733Scem 177289733Scem for (k = 0; k < test->chain_depth; k++) { 178289733Scem ((uint32_t *)tx->buf[2*k])[j] = ~val; 179289733Scem ((uint32_t *)tx->buf[2*k+1])[j] = val; 180289733Scem } 181289733Scem } 182287117Scem } 183289733Scem return (0); 184287117Scem} 185287117Scem 186287117Scemstatic void 187289733Scemioat_test_release_memory(struct ioat_test *test) 188289733Scem{ 189289733Scem struct test_transaction *tx, *s; 190289733Scem 191289733Scem TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s) 192289733Scem ioat_test_transaction_destroy(tx); 193289733Scem TAILQ_INIT(&test->free_q); 194289733Scem 195289733Scem TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s) 196289733Scem ioat_test_transaction_destroy(tx); 197289733Scem TAILQ_INIT(&test->pend_q); 198289733Scem} 199289733Scem 200289733Scemstatic void 201289733Scemioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma) 202289733Scem{ 203289733Scem struct test_transaction *tx; 204289733Scem struct bus_dmadesc *desc; 205289733Scem bus_dmaengine_callback_t cb; 206289733Scem bus_addr_t src, dest; 207289733Scem uint32_t i, flags; 208289733Scem 209289733Scem IT_LOCK(); 210289733Scem while (TAILQ_EMPTY(&test->free_q)) 211289733Scem msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0); 212289733Scem 213289733Scem tx = TAILQ_FIRST(&test->free_q); 214289733Scem TAILQ_REMOVE(&test->free_q, tx, entry); 215289733Scem TAILQ_INSERT_HEAD(&test->pend_q, tx, entry); 216289733Scem IT_UNLOCK(); 217289733Scem 218289733Scem ioat_acquire(dma); 219289733Scem for (i = 0; i < tx->depth; i++) { 220289733Scem src = vtophys((vm_offset_t)tx->buf[2*i]); 221289733Scem dest = vtophys((vm_offset_t)tx->buf[2*i+1]); 222289733Scem 223289733Scem if (i == tx->depth - 1) { 224289733Scem cb = ioat_dma_test_callback; 225289733Scem flags = DMA_INT_EN; 226289733Scem } else { 227289733Scem cb = NULL; 228289733Scem flags = 0; 229289733Scem } 230289733Scem 231289733Scem desc = ioat_copy(dma, src, dest, tx->length, cb, tx, flags); 232289733Scem if (desc == NULL) 233289733Scem panic("Failed to allocate a ring slot " 234289733Scem "-- this shouldn't happen!"); 235289733Scem } 236289733Scem ioat_release(dma); 237289733Scem} 238289733Scem 239289733Scemstatic void 240287117Scemioat_dma_test(void *arg) 241287117Scem{ 242287117Scem struct ioat_test *test; 243287117Scem bus_dmaengine_t dmaengine; 244287117Scem uint32_t loops; 245289733Scem int index, rc, start, end; 246287117Scem 247287117Scem test = arg; 248289733Scem memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status)); 249287117Scem 250289733Scem if (test->buffer_size > 1024 * 1024) { 251289733Scem ioat_log_message(0, "Buffer size too large >1MB\n"); 252289733Scem test->status[IOAT_TEST_NO_MEMORY]++; 253289733Scem return; 254289733Scem } 255287117Scem 256289733Scem if (test->chain_depth * 2 > IOAT_MAX_BUFS) { 257289733Scem ioat_log_message(0, "Depth too large (> %u)\n", 258289733Scem (unsigned)IOAT_MAX_BUFS / 2); 259289733Scem test->status[IOAT_TEST_NO_MEMORY]++; 260289733Scem return; 261289733Scem } 262289733Scem 263289733Scem if (btoc((uint64_t)test->buffer_size * test->chain_depth * 264289733Scem test->transactions) > (physmem / 4)) { 265289733Scem ioat_log_message(0, "Sanity check failed -- test would " 266289733Scem "use more than 1/4 of phys mem.\n"); 267289733Scem test->status[IOAT_TEST_NO_MEMORY]++; 268289733Scem return; 269289733Scem } 270289733Scem 271289733Scem if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) { 272289733Scem ioat_log_message(0, "Sanity check failed -- test would " 273289733Scem "use more than available IOAT ring space.\n"); 274289733Scem test->status[IOAT_TEST_NO_MEMORY]++; 275289733Scem return; 276289733Scem } 277289733Scem 278287117Scem dmaengine = ioat_get_dmaengine(test->channel_index); 279287117Scem if (dmaengine == NULL) { 280287117Scem ioat_log_message(0, "Couldn't acquire dmaengine\n"); 281289733Scem test->status[IOAT_TEST_NO_DMA_ENGINE]++; 282287117Scem return; 283287117Scem } 284287117Scem 285289733Scem index = g_thread_index++; 286289733Scem TAILQ_INIT(&test->free_q); 287289733Scem TAILQ_INIT(&test->pend_q); 288287117Scem 289289733Scem if (test->duration == 0) 290289733Scem ioat_log_message(1, "Thread %d: num_loops remaining: 0x%08x\n", 291289733Scem index, test->transactions); 292289733Scem else 293289733Scem ioat_log_message(1, "Thread %d: starting\n", index); 294287117Scem 295289733Scem rc = ioat_test_prealloc_memory(test, index); 296289733Scem if (rc != 0) { 297289733Scem ioat_log_message(0, "prealloc_memory: %d\n", rc); 298289733Scem return; 299289733Scem } 300289733Scem wmb(); 301287117Scem 302289733Scem test->too_late = false; 303289733Scem start = ticks; 304289733Scem end = start + (((sbintime_t)test->duration * hz) / 1000); 305289733Scem 306289733Scem for (loops = 0;; loops++) { 307289733Scem if (test->duration == 0 && loops >= test->transactions) 308289733Scem break; 309289733Scem else if (test->duration != 0 && time_after(ticks, end)) { 310289733Scem test->too_late = true; 311289733Scem break; 312287117Scem } 313287117Scem 314289733Scem ioat_test_submit_1_tx(test, dmaengine); 315289733Scem } 316287117Scem 317289733Scem ioat_log_message(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n", 318289733Scem ticks - start, ticks - end, (ticks - start) / hz); 319287117Scem 320289733Scem IT_LOCK(); 321289733Scem while (!TAILQ_EMPTY(&test->pend_q)) 322289733Scem msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz); 323289733Scem IT_UNLOCK(); 324287117Scem 325289733Scem ioat_log_message(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n", 326289733Scem ticks - start, ticks - end, (ticks - start) / hz); 327287117Scem 328289733Scem ioat_test_release_memory(test); 329287117Scem} 330287117Scem 331287117Scemstatic int 332287117Scemioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td) 333287117Scem{ 334287117Scem 335287117Scem return (0); 336287117Scem} 337287117Scem 338287117Scemstatic int 339287117Scemioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td) 340287117Scem{ 341287117Scem 342287117Scem return (0); 343287117Scem} 344287117Scem 345287117Scemstatic int 346287117Scemioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag, 347287117Scem struct thread *td) 348287117Scem{ 349287117Scem 350287117Scem switch (cmd) { 351287117Scem case IOAT_DMATEST: 352287117Scem ioat_dma_test(arg); 353287117Scem break; 354287117Scem default: 355287117Scem return (EINVAL); 356287117Scem } 357287117Scem return (0); 358287117Scem} 359287117Scem 360287117Scemstatic struct cdevsw ioat_cdevsw = { 361287117Scem .d_version = D_VERSION, 362287117Scem .d_flags = 0, 363287117Scem .d_open = ioat_test_open, 364287117Scem .d_close = ioat_test_close, 365287117Scem .d_ioctl = ioat_test_ioctl, 366287117Scem .d_name = "ioat_test", 367287117Scem}; 368287117Scem 369287117Scemstatic int 370287117Scemsysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS) 371287117Scem{ 372287117Scem int error, enabled; 373287117Scem 374287117Scem enabled = (g_ioat_cdev != NULL); 375287117Scem error = sysctl_handle_int(oidp, &enabled, 0, req); 376287117Scem if (error != 0 || req->newptr == NULL) 377287117Scem return (error); 378287117Scem 379287117Scem if (enabled != 0 && g_ioat_cdev == NULL) { 380287117Scem g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL, 381287117Scem 0600, "ioat_test"); 382287117Scem } else if (enabled == 0 && g_ioat_cdev != NULL) { 383287117Scem destroy_dev(g_ioat_cdev); 384287117Scem g_ioat_cdev = NULL; 385287117Scem } 386287117Scem return (0); 387287117Scem} 388287117ScemSYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_INT | CTLFLAG_RW, 389287117Scem 0, 0, sysctl_enable_ioat_test, "I", 390287117Scem "Non-zero: Enable the /dev/ioat_test device"); 391