1240616Sjimharris/*- 2253112Sjimharris * Copyright (C) 2012-2013 Intel Corporation 3240616Sjimharris * All rights reserved. 4240616Sjimharris * 5240616Sjimharris * Redistribution and use in source and binary forms, with or without 6240616Sjimharris * modification, are permitted provided that the following conditions 7240616Sjimharris * are met: 8240616Sjimharris * 1. Redistributions of source code must retain the above copyright 9240616Sjimharris * notice, this list of conditions and the following disclaimer. 10240616Sjimharris * 2. Redistributions in binary form must reproduce the above copyright 11240616Sjimharris * notice, this list of conditions and the following disclaimer in the 12240616Sjimharris * documentation and/or other materials provided with the distribution. 13240616Sjimharris * 14240616Sjimharris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15240616Sjimharris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16240616Sjimharris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17240616Sjimharris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18240616Sjimharris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19240616Sjimharris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20240616Sjimharris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21240616Sjimharris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22240616Sjimharris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23240616Sjimharris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24240616Sjimharris * SUCH DAMAGE. 25240616Sjimharris */ 26240616Sjimharris 27240616Sjimharris#include <sys/cdefs.h> 28240616Sjimharris__FBSDID("$FreeBSD$"); 29240616Sjimharris 30240616Sjimharris#include <sys/param.h> 31240616Sjimharris#include <sys/bio.h> 32240616Sjimharris#include <sys/conf.h> 33240616Sjimharris#include <sys/fcntl.h> 34240616Sjimharris#include <sys/kthread.h> 35240616Sjimharris#include <sys/module.h> 36240616Sjimharris#include <sys/proc.h> 37240616Sjimharris#include <sys/syscallsubr.h> 38240616Sjimharris#include <sys/sysctl.h> 39240616Sjimharris#include <sys/sysproto.h> 40240616Sjimharris#include <sys/systm.h> 41240616Sjimharris#include <sys/unistd.h> 42240616Sjimharris 43240616Sjimharris#include <geom/geom.h> 44240616Sjimharris 45240616Sjimharris#include "nvme_private.h" 46240616Sjimharris 47240616Sjimharrisstruct nvme_io_test_thread { 48240616Sjimharris 49240616Sjimharris uint32_t idx; 50240616Sjimharris struct nvme_namespace *ns; 51240616Sjimharris enum nvme_nvm_opcode opc; 52240616Sjimharris struct timeval start; 53240616Sjimharris void *buf; 54240616Sjimharris uint32_t size; 55240616Sjimharris uint32_t time; 56256152Sjimharris uint64_t io_completed; 57240616Sjimharris}; 58240616Sjimharris 59240616Sjimharrisstruct nvme_io_test_internal { 60240616Sjimharris 61240616Sjimharris struct nvme_namespace *ns; 62240616Sjimharris enum nvme_nvm_opcode opc; 63240616Sjimharris struct timeval start; 64240616Sjimharris uint32_t time; 65240616Sjimharris uint32_t size; 66240616Sjimharris uint32_t td_active; 67240616Sjimharris uint32_t td_idx; 68240616Sjimharris uint32_t flags; 69256152Sjimharris uint64_t io_completed[NVME_TEST_MAX_THREADS]; 70240616Sjimharris}; 71240616Sjimharris 72240616Sjimharrisstatic void 73240616Sjimharrisnvme_ns_bio_test_cb(struct bio *bio) 74240616Sjimharris{ 75240616Sjimharris struct mtx *mtx; 76240616Sjimharris 77240616Sjimharris mtx = mtx_pool_find(mtxpool_sleep, bio); 78240616Sjimharris mtx_lock(mtx); 79240616Sjimharris wakeup(bio); 80240616Sjimharris mtx_unlock(mtx); 81240616Sjimharris} 82240616Sjimharris 83240616Sjimharrisstatic void 84240616Sjimharrisnvme_ns_bio_test(void *arg) 85240616Sjimharris{ 86240616Sjimharris struct nvme_io_test_internal *io_test = arg; 87240616Sjimharris struct cdevsw *csw; 88240616Sjimharris struct mtx *mtx; 89240616Sjimharris struct bio *bio; 90240616Sjimharris struct cdev *dev; 91240616Sjimharris void *buf; 92240616Sjimharris struct timeval t; 93256152Sjimharris uint64_t io_completed = 0, offset; 94256152Sjimharris uint32_t idx; 95240616Sjimharris#if __FreeBSD_version >= 900017 96240616Sjimharris int ref; 97240616Sjimharris#endif 98240616Sjimharris 99248770Sjimharris buf = malloc(io_test->size, M_NVME, M_WAITOK); 100240616Sjimharris idx = atomic_fetchadd_int(&io_test->td_idx, 1); 101240616Sjimharris dev = io_test->ns->cdev; 102240616Sjimharris 103240616Sjimharris offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns); 104240616Sjimharris 105240616Sjimharris while (1) { 106240616Sjimharris 107240616Sjimharris bio = g_alloc_bio(); 108240616Sjimharris 109240616Sjimharris memset(bio, 0, sizeof(*bio)); 110240616Sjimharris bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ? 111240616Sjimharris BIO_READ : BIO_WRITE; 112240616Sjimharris bio->bio_done = nvme_ns_bio_test_cb; 113240616Sjimharris bio->bio_dev = dev; 114240616Sjimharris bio->bio_offset = offset; 115240616Sjimharris bio->bio_data = buf; 116240616Sjimharris bio->bio_bcount = io_test->size; 117240616Sjimharris 118240616Sjimharris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 119240616Sjimharris#if __FreeBSD_version >= 900017 120240616Sjimharris csw = dev_refthread(dev, &ref); 121240616Sjimharris#else 122240616Sjimharris csw = dev_refthread(dev); 123240616Sjimharris#endif 124240616Sjimharris } else 125240616Sjimharris csw = dev->si_devsw; 126240616Sjimharris 127240616Sjimharris mtx = mtx_pool_find(mtxpool_sleep, bio); 128240616Sjimharris mtx_lock(mtx); 129240616Sjimharris (*csw->d_strategy)(bio); 130240616Sjimharris msleep(bio, mtx, PRIBIO, "biotestwait", 0); 131240616Sjimharris mtx_unlock(mtx); 132240616Sjimharris 133240616Sjimharris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 134240616Sjimharris#if __FreeBSD_version >= 900017 135240616Sjimharris dev_relthread(dev, ref); 136240616Sjimharris#else 137240616Sjimharris dev_relthread(dev); 138240616Sjimharris#endif 139240616Sjimharris } 140240616Sjimharris 141240616Sjimharris if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0)) 142240616Sjimharris break; 143240616Sjimharris 144240616Sjimharris g_destroy_bio(bio); 145240616Sjimharris 146240616Sjimharris io_completed++; 147240616Sjimharris 148240616Sjimharris getmicrouptime(&t); 149240616Sjimharris timevalsub(&t, &io_test->start); 150240616Sjimharris 151240616Sjimharris if (t.tv_sec >= io_test->time) 152240616Sjimharris break; 153240616Sjimharris 154240616Sjimharris offset += io_test->size; 155240616Sjimharris if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns)) 156240616Sjimharris offset = 0; 157240616Sjimharris } 158240616Sjimharris 159240616Sjimharris io_test->io_completed[idx] = io_completed; 160240616Sjimharris wakeup_one(io_test); 161240616Sjimharris 162240616Sjimharris free(buf, M_NVME); 163240616Sjimharris 164240616Sjimharris atomic_subtract_int(&io_test->td_active, 1); 165240616Sjimharris mb(); 166240616Sjimharris 167240616Sjimharris#if __FreeBSD_version >= 800000 168240616Sjimharris kthread_exit(); 169240616Sjimharris#else 170240616Sjimharris kthread_exit(0); 171240616Sjimharris#endif 172240616Sjimharris} 173240616Sjimharris 174240616Sjimharrisstatic void 175248756Sjimharrisnvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl) 176240616Sjimharris{ 177240616Sjimharris struct nvme_io_test_thread *tth = arg; 178240616Sjimharris struct timeval t; 179240616Sjimharris 180240616Sjimharris tth->io_completed++; 181240616Sjimharris 182248756Sjimharris if (nvme_completion_is_error(cpl)) { 183240616Sjimharris printf("%s: error occurred\n", __func__); 184240616Sjimharris wakeup_one(tth); 185240616Sjimharris return; 186240616Sjimharris } 187240616Sjimharris 188240616Sjimharris getmicrouptime(&t); 189240616Sjimharris timevalsub(&t, &tth->start); 190240616Sjimharris 191240616Sjimharris if (t.tv_sec >= tth->time) { 192240616Sjimharris wakeup_one(tth); 193240616Sjimharris return; 194240616Sjimharris } 195240616Sjimharris 196240616Sjimharris switch (tth->opc) { 197240616Sjimharris case NVME_OPC_WRITE: 198240616Sjimharris nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048, 199240616Sjimharris tth->size/nvme_ns_get_sector_size(tth->ns), 200240616Sjimharris nvme_ns_io_test_cb, tth); 201240616Sjimharris break; 202240616Sjimharris case NVME_OPC_READ: 203240616Sjimharris nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048, 204240616Sjimharris tth->size/nvme_ns_get_sector_size(tth->ns), 205240616Sjimharris nvme_ns_io_test_cb, tth); 206240616Sjimharris break; 207240616Sjimharris default: 208240616Sjimharris break; 209240616Sjimharris } 210240616Sjimharris} 211240616Sjimharris 212240616Sjimharrisstatic void 213240616Sjimharrisnvme_ns_io_test(void *arg) 214240616Sjimharris{ 215240616Sjimharris struct nvme_io_test_internal *io_test = arg; 216240616Sjimharris struct nvme_io_test_thread *tth; 217240616Sjimharris struct nvme_completion cpl; 218240616Sjimharris int error; 219240616Sjimharris 220248770Sjimharris tth = malloc(sizeof(*tth), M_NVME, M_WAITOK | M_ZERO); 221240616Sjimharris tth->ns = io_test->ns; 222240616Sjimharris tth->opc = io_test->opc; 223240616Sjimharris memcpy(&tth->start, &io_test->start, sizeof(tth->start)); 224248770Sjimharris tth->buf = malloc(io_test->size, M_NVME, M_WAITOK); 225240616Sjimharris tth->size = io_test->size; 226240616Sjimharris tth->time = io_test->time; 227240616Sjimharris tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1); 228240616Sjimharris 229240616Sjimharris memset(&cpl, 0, sizeof(cpl)); 230240616Sjimharris 231240616Sjimharris nvme_ns_io_test_cb(tth, &cpl); 232240616Sjimharris 233240616Sjimharris error = tsleep(tth, 0, "test_wait", tth->time*hz*2); 234240616Sjimharris 235240616Sjimharris if (error) 236240616Sjimharris printf("%s: error = %d\n", __func__, error); 237240616Sjimharris 238240616Sjimharris io_test->io_completed[tth->idx] = tth->io_completed; 239240616Sjimharris wakeup_one(io_test); 240240616Sjimharris 241240616Sjimharris free(tth->buf, M_NVME); 242240616Sjimharris free(tth, M_NVME); 243240616Sjimharris 244240616Sjimharris atomic_subtract_int(&io_test->td_active, 1); 245240616Sjimharris mb(); 246240616Sjimharris 247240616Sjimharris#if __FreeBSD_version >= 800004 248240616Sjimharris kthread_exit(); 249240616Sjimharris#else 250240616Sjimharris kthread_exit(0); 251240616Sjimharris#endif 252240616Sjimharris} 253240616Sjimharris 254240616Sjimharrisvoid 255240616Sjimharrisnvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg) 256240616Sjimharris{ 257240616Sjimharris struct nvme_io_test *io_test; 258240616Sjimharris struct nvme_io_test_internal *io_test_internal; 259240616Sjimharris void (*fn)(void *); 260240616Sjimharris int i; 261240616Sjimharris 262240616Sjimharris io_test = (struct nvme_io_test *)arg; 263240616Sjimharris 264240616Sjimharris if ((io_test->opc != NVME_OPC_READ) && 265240616Sjimharris (io_test->opc != NVME_OPC_WRITE)) 266240616Sjimharris return; 267240616Sjimharris 268240616Sjimharris if (io_test->size % nvme_ns_get_sector_size(ns)) 269240616Sjimharris return; 270240616Sjimharris 271240616Sjimharris io_test_internal = malloc(sizeof(*io_test_internal), M_NVME, 272248770Sjimharris M_WAITOK | M_ZERO); 273240616Sjimharris io_test_internal->opc = io_test->opc; 274240616Sjimharris io_test_internal->ns = ns; 275240616Sjimharris io_test_internal->td_active = io_test->num_threads; 276240616Sjimharris io_test_internal->time = io_test->time; 277240616Sjimharris io_test_internal->size = io_test->size; 278240616Sjimharris io_test_internal->flags = io_test->flags; 279240616Sjimharris 280240616Sjimharris if (cmd == NVME_IO_TEST) 281240616Sjimharris fn = nvme_ns_io_test; 282240616Sjimharris else 283240616Sjimharris fn = nvme_ns_bio_test; 284240616Sjimharris 285240616Sjimharris getmicrouptime(&io_test_internal->start); 286240616Sjimharris 287240616Sjimharris for (i = 0; i < io_test->num_threads; i++) 288240616Sjimharris#if __FreeBSD_version >= 800004 289240616Sjimharris kthread_add(fn, io_test_internal, 290245136Sjimharris NULL, NULL, 0, 0, "nvme_io_test[%d]", i); 291240616Sjimharris#else 292240616Sjimharris kthread_create(fn, io_test_internal, 293245136Sjimharris NULL, 0, 0, "nvme_io_test[%d]", i); 294240616Sjimharris#endif 295240616Sjimharris 296240616Sjimharris tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz); 297240616Sjimharris 298240616Sjimharris while (io_test_internal->td_active > 0) 299240616Sjimharris DELAY(10); 300240616Sjimharris 301240616Sjimharris memcpy(io_test->io_completed, io_test_internal->io_completed, 302240616Sjimharris sizeof(io_test->io_completed)); 303240616Sjimharris 304240616Sjimharris free(io_test_internal, M_NVME); 305240616Sjimharris} 306