1240616Sjimharris/*- 2253112Sjimharris * Copyright (C) 2012-2013 Intel Corporation 3240616Sjimharris * All rights reserved. 4240616Sjimharris * 5240616Sjimharris * Redistribution and use in source and binary forms, with or without 6240616Sjimharris * modification, are permitted provided that the following conditions 7240616Sjimharris * are met: 8240616Sjimharris * 1. Redistributions of source code must retain the above copyright 9240616Sjimharris * notice, this list of conditions and the following disclaimer. 10240616Sjimharris * 2. Redistributions in binary form must reproduce the above copyright 11240616Sjimharris * notice, this list of conditions and the following disclaimer in the 12240616Sjimharris * documentation and/or other materials provided with the distribution. 13240616Sjimharris * 14240616Sjimharris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15240616Sjimharris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16240616Sjimharris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17240616Sjimharris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18240616Sjimharris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19240616Sjimharris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20240616Sjimharris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21240616Sjimharris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22240616Sjimharris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23240616Sjimharris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24240616Sjimharris * SUCH DAMAGE. 25240616Sjimharris */ 26240616Sjimharris 27240616Sjimharris#include <sys/cdefs.h> 28240616Sjimharris__FBSDID("$FreeBSD: stable/11/sys/dev/nvme/nvme_test.c 346243 2019-04-15 16:27:53Z mav $"); 29240616Sjimharris 30240616Sjimharris#include <sys/param.h> 31240616Sjimharris#include <sys/bio.h> 32240616Sjimharris#include <sys/conf.h> 33240616Sjimharris#include <sys/fcntl.h> 34240616Sjimharris#include <sys/kthread.h> 35240616Sjimharris#include <sys/module.h> 36240616Sjimharris#include <sys/proc.h> 37240616Sjimharris#include <sys/syscallsubr.h> 38240616Sjimharris#include <sys/sysctl.h> 39240616Sjimharris#include <sys/sysproto.h> 40240616Sjimharris#include <sys/systm.h> 41240616Sjimharris#include <sys/unistd.h> 42240616Sjimharris 43240616Sjimharris#include <geom/geom.h> 44240616Sjimharris 45240616Sjimharris#include "nvme_private.h" 46240616Sjimharris 47240616Sjimharrisstruct nvme_io_test_thread { 48240616Sjimharris 49240616Sjimharris uint32_t idx; 50240616Sjimharris struct nvme_namespace *ns; 51240616Sjimharris enum nvme_nvm_opcode opc; 52240616Sjimharris struct timeval start; 53240616Sjimharris void *buf; 54240616Sjimharris uint32_t size; 55240616Sjimharris uint32_t time; 56256152Sjimharris uint64_t io_completed; 57240616Sjimharris}; 58240616Sjimharris 59240616Sjimharrisstruct nvme_io_test_internal { 60240616Sjimharris 61240616Sjimharris struct nvme_namespace *ns; 62240616Sjimharris enum nvme_nvm_opcode opc; 63240616Sjimharris struct timeval start; 64240616Sjimharris uint32_t time; 65240616Sjimharris uint32_t size; 66240616Sjimharris uint32_t td_active; 67240616Sjimharris uint32_t td_idx; 68240616Sjimharris uint32_t flags; 69256152Sjimharris uint64_t io_completed[NVME_TEST_MAX_THREADS]; 70240616Sjimharris}; 71240616Sjimharris 72240616Sjimharrisstatic void 73240616Sjimharrisnvme_ns_bio_test_cb(struct bio *bio) 74240616Sjimharris{ 75240616Sjimharris struct mtx *mtx; 76240616Sjimharris 77240616Sjimharris mtx = mtx_pool_find(mtxpool_sleep, bio); 78240616Sjimharris mtx_lock(mtx); 79240616Sjimharris wakeup(bio); 80240616Sjimharris mtx_unlock(mtx); 81240616Sjimharris} 82240616Sjimharris 83240616Sjimharrisstatic void 84240616Sjimharrisnvme_ns_bio_test(void *arg) 85240616Sjimharris{ 86240616Sjimharris struct nvme_io_test_internal *io_test = arg; 87240616Sjimharris struct cdevsw *csw; 88240616Sjimharris struct mtx *mtx; 89240616Sjimharris struct bio *bio; 90240616Sjimharris struct cdev *dev; 91240616Sjimharris void *buf; 92240616Sjimharris struct timeval t; 93256152Sjimharris uint64_t io_completed = 0, offset; 94256152Sjimharris uint32_t idx; 95240616Sjimharris int ref; 96240616Sjimharris 97248770Sjimharris buf = malloc(io_test->size, M_NVME, M_WAITOK); 98240616Sjimharris idx = atomic_fetchadd_int(&io_test->td_idx, 1); 99240616Sjimharris dev = io_test->ns->cdev; 100240616Sjimharris 101240616Sjimharris offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns); 102240616Sjimharris 103240616Sjimharris while (1) { 104240616Sjimharris 105240616Sjimharris bio = g_alloc_bio(); 106240616Sjimharris 107240616Sjimharris memset(bio, 0, sizeof(*bio)); 108240616Sjimharris bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ? 109240616Sjimharris BIO_READ : BIO_WRITE; 110240616Sjimharris bio->bio_done = nvme_ns_bio_test_cb; 111240616Sjimharris bio->bio_dev = dev; 112240616Sjimharris bio->bio_offset = offset; 113240616Sjimharris bio->bio_data = buf; 114240616Sjimharris bio->bio_bcount = io_test->size; 115240616Sjimharris 116240616Sjimharris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 117240616Sjimharris csw = dev_refthread(dev, &ref); 118240616Sjimharris } else 119240616Sjimharris csw = dev->si_devsw; 120240616Sjimharris 121240616Sjimharris mtx = mtx_pool_find(mtxpool_sleep, bio); 122240616Sjimharris mtx_lock(mtx); 123240616Sjimharris (*csw->d_strategy)(bio); 124240616Sjimharris msleep(bio, mtx, PRIBIO, "biotestwait", 0); 125240616Sjimharris mtx_unlock(mtx); 126240616Sjimharris 127240616Sjimharris if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 128240616Sjimharris dev_relthread(dev, ref); 129240616Sjimharris } 130240616Sjimharris 131240616Sjimharris if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0)) 132240616Sjimharris break; 133240616Sjimharris 134240616Sjimharris g_destroy_bio(bio); 135240616Sjimharris 136240616Sjimharris io_completed++; 137240616Sjimharris 138240616Sjimharris getmicrouptime(&t); 139240616Sjimharris timevalsub(&t, &io_test->start); 140240616Sjimharris 141240616Sjimharris if (t.tv_sec >= io_test->time) 142240616Sjimharris break; 143240616Sjimharris 144240616Sjimharris offset += io_test->size; 145240616Sjimharris if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns)) 146240616Sjimharris offset = 0; 147240616Sjimharris } 148240616Sjimharris 149240616Sjimharris io_test->io_completed[idx] = io_completed; 150240616Sjimharris wakeup_one(io_test); 151240616Sjimharris 152240616Sjimharris free(buf, M_NVME); 153240616Sjimharris 154240616Sjimharris atomic_subtract_int(&io_test->td_active, 1); 155240616Sjimharris mb(); 156240616Sjimharris 157240616Sjimharris kthread_exit(); 158240616Sjimharris} 159240616Sjimharris 160240616Sjimharrisstatic void 161248756Sjimharrisnvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl) 162240616Sjimharris{ 163240616Sjimharris struct nvme_io_test_thread *tth = arg; 164240616Sjimharris struct timeval t; 165240616Sjimharris 166240616Sjimharris tth->io_completed++; 167240616Sjimharris 168248756Sjimharris if (nvme_completion_is_error(cpl)) { 169240616Sjimharris printf("%s: error occurred\n", __func__); 170240616Sjimharris wakeup_one(tth); 171240616Sjimharris return; 172240616Sjimharris } 173240616Sjimharris 174240616Sjimharris getmicrouptime(&t); 175240616Sjimharris timevalsub(&t, &tth->start); 176240616Sjimharris 177240616Sjimharris if (t.tv_sec >= tth->time) { 178240616Sjimharris wakeup_one(tth); 179240616Sjimharris return; 180240616Sjimharris } 181240616Sjimharris 182240616Sjimharris switch (tth->opc) { 183240616Sjimharris case NVME_OPC_WRITE: 184240616Sjimharris nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048, 185240616Sjimharris tth->size/nvme_ns_get_sector_size(tth->ns), 186240616Sjimharris nvme_ns_io_test_cb, tth); 187240616Sjimharris break; 188240616Sjimharris case NVME_OPC_READ: 189240616Sjimharris nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048, 190240616Sjimharris tth->size/nvme_ns_get_sector_size(tth->ns), 191240616Sjimharris nvme_ns_io_test_cb, tth); 192240616Sjimharris break; 193240616Sjimharris default: 194240616Sjimharris break; 195240616Sjimharris } 196240616Sjimharris} 197240616Sjimharris 198240616Sjimharrisstatic void 199240616Sjimharrisnvme_ns_io_test(void *arg) 200240616Sjimharris{ 201240616Sjimharris struct nvme_io_test_internal *io_test = arg; 202240616Sjimharris struct nvme_io_test_thread *tth; 203240616Sjimharris struct nvme_completion cpl; 204240616Sjimharris int error; 205240616Sjimharris 206248770Sjimharris tth = malloc(sizeof(*tth), M_NVME, M_WAITOK | M_ZERO); 207240616Sjimharris tth->ns = io_test->ns; 208240616Sjimharris tth->opc = io_test->opc; 209240616Sjimharris memcpy(&tth->start, &io_test->start, sizeof(tth->start)); 210248770Sjimharris tth->buf = malloc(io_test->size, M_NVME, M_WAITOK); 211240616Sjimharris tth->size = io_test->size; 212240616Sjimharris tth->time = io_test->time; 213240616Sjimharris tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1); 214240616Sjimharris 215240616Sjimharris memset(&cpl, 0, sizeof(cpl)); 216240616Sjimharris 217240616Sjimharris nvme_ns_io_test_cb(tth, &cpl); 218240616Sjimharris 219240616Sjimharris error = tsleep(tth, 0, "test_wait", tth->time*hz*2); 220240616Sjimharris 221240616Sjimharris if (error) 222240616Sjimharris printf("%s: error = %d\n", __func__, error); 223240616Sjimharris 224240616Sjimharris io_test->io_completed[tth->idx] = tth->io_completed; 225240616Sjimharris wakeup_one(io_test); 226240616Sjimharris 227240616Sjimharris free(tth->buf, M_NVME); 228240616Sjimharris free(tth, M_NVME); 229240616Sjimharris 230240616Sjimharris atomic_subtract_int(&io_test->td_active, 1); 231240616Sjimharris mb(); 232240616Sjimharris 233240616Sjimharris kthread_exit(); 234240616Sjimharris} 235240616Sjimharris 236240616Sjimharrisvoid 237240616Sjimharrisnvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg) 238240616Sjimharris{ 239240616Sjimharris struct nvme_io_test *io_test; 240240616Sjimharris struct nvme_io_test_internal *io_test_internal; 241240616Sjimharris void (*fn)(void *); 242240616Sjimharris int i; 243240616Sjimharris 244240616Sjimharris io_test = (struct nvme_io_test *)arg; 245240616Sjimharris 246240616Sjimharris if ((io_test->opc != NVME_OPC_READ) && 247240616Sjimharris (io_test->opc != NVME_OPC_WRITE)) 248240616Sjimharris return; 249240616Sjimharris 250240616Sjimharris if (io_test->size % nvme_ns_get_sector_size(ns)) 251240616Sjimharris return; 252240616Sjimharris 253240616Sjimharris io_test_internal = malloc(sizeof(*io_test_internal), M_NVME, 254248770Sjimharris M_WAITOK | M_ZERO); 255240616Sjimharris io_test_internal->opc = io_test->opc; 256240616Sjimharris io_test_internal->ns = ns; 257240616Sjimharris io_test_internal->td_active = io_test->num_threads; 258240616Sjimharris io_test_internal->time = io_test->time; 259240616Sjimharris io_test_internal->size = io_test->size; 260240616Sjimharris io_test_internal->flags = io_test->flags; 261240616Sjimharris 262240616Sjimharris if (cmd == NVME_IO_TEST) 263240616Sjimharris fn = nvme_ns_io_test; 264240616Sjimharris else 265240616Sjimharris fn = nvme_ns_bio_test; 266240616Sjimharris 267240616Sjimharris getmicrouptime(&io_test_internal->start); 268240616Sjimharris 269240616Sjimharris for (i = 0; i < io_test->num_threads; i++) 270240616Sjimharris kthread_add(fn, io_test_internal, 271245136Sjimharris NULL, NULL, 0, 0, "nvme_io_test[%d]", i); 272240616Sjimharris 273240616Sjimharris tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz); 274240616Sjimharris 275240616Sjimharris while (io_test_internal->td_active > 0) 276240616Sjimharris DELAY(10); 277240616Sjimharris 278240616Sjimharris memcpy(io_test->io_completed, io_test_internal->io_completed, 279240616Sjimharris sizeof(io_test->io_completed)); 280240616Sjimharris 281240616Sjimharris free(io_test_internal, M_NVME); 282240616Sjimharris} 283