nvd.c revision 241394
1/*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/nvd/nvd.c 241394 2012-10-10 08:36:38Z kevlo $"); 29 30#include <sys/param.h> 31#include <sys/bio.h> 32#include <sys/kernel.h> 33#include <sys/malloc.h> 34#include <sys/module.h> 35#include <sys/systm.h> 36#include <sys/taskqueue.h> 37 38#include <geom/geom.h> 39#include <geom/geom_disk.h> 40 41#include <dev/nvme/nvme.h> 42 43struct nvd_disk; 44 45static disk_ioctl_t nvd_ioctl; 46static disk_strategy_t nvd_strategy; 47 48static void create_geom_disk(void *, struct nvme_namespace *ns); 49static void destroy_geom_disk(struct nvd_disk *ndisk); 50 51static int nvd_load(void); 52static void nvd_unload(void); 53 54MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations"); 55 56struct nvme_consumer *consumer_handle; 57 58struct nvd_disk { 59 60 struct bio_queue_head bioq; 61 struct task bioqtask; 62 struct mtx bioqlock; 63 64 struct disk *disk; 65 struct taskqueue *tq; 66 struct nvme_namespace *ns; 67 68 uint32_t cur_depth; 69 70 TAILQ_ENTRY(nvd_disk) tailq; 71}; 72 73TAILQ_HEAD(, nvd_disk) nvd_head; 74 75static int nvd_modevent(module_t mod, int type, void *arg) 76{ 77 int error = 0; 78 79 switch (type) { 80 case MOD_LOAD: 81 error = nvd_load(); 82 break; 83 case MOD_UNLOAD: 84 nvd_unload(); 85 break; 86 default: 87 break; 88 } 89 90 return (error); 91} 92 93moduledata_t nvd_mod = { 94 "nvd", 95 (modeventhand_t)nvd_modevent, 96 0 97}; 98 99DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 100MODULE_VERSION(nvd, 1); 101MODULE_DEPEND(nvd, nvme, 1, 1, 1); 102 103static int 104nvd_load() 105{ 106 107 TAILQ_INIT(&nvd_head); 108 consumer_handle = nvme_register_consumer(create_geom_disk, NULL); 109 110 return (consumer_handle != NULL ? 0 : -1); 111} 112 113static void 114nvd_unload() 115{ 116 struct nvd_disk *nvd; 117 118 while (!TAILQ_EMPTY(&nvd_head)) { 119 nvd = TAILQ_FIRST(&nvd_head); 120 TAILQ_REMOVE(&nvd_head, nvd, tailq); 121 destroy_geom_disk(nvd); 122 free(nvd, M_NVD); 123 } 124 125 nvme_unregister_consumer(consumer_handle); 126} 127 128static void 129nvd_strategy(struct bio *bp) 130{ 131 struct nvd_disk *ndisk; 132 133 ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1; 134 135 mtx_lock(&ndisk->bioqlock); 136 bioq_insert_tail(&ndisk->bioq, bp); 137 mtx_unlock(&ndisk->bioqlock); 138 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 139} 140 141static int 142nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag, 143 struct thread *td) 144{ 145 int ret = 0; 146 147 switch (cmd) { 148 default: 149 ret = EIO; 150 } 151 152 return (ret); 153} 154 155static void 156nvd_done(void *arg, const struct nvme_completion *status) 157{ 158 struct bio *bp; 159 struct nvd_disk *ndisk; 160 161 bp = (struct bio *)arg; 162 163 ndisk = bp->bio_disk->d_drv1; 164 165 if (atomic_fetchadd_int(&ndisk->cur_depth, -1) == NVME_QD) 166 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 167 168 /* 169 * TODO: add more extensive translation of NVMe status codes 170 * to different bio error codes (i.e. EIO, EINVAL, etc.) 171 */ 172 if (status->sf_sc || status->sf_sct) { 173 bp->bio_error = EIO; 174 bp->bio_flags |= BIO_ERROR; 175 bp->bio_resid = bp->bio_bcount; 176 } else 177 bp->bio_resid = 0; 178 179 biodone(bp); 180} 181 182static void 183nvd_bioq_process(void *arg, int pending) 184{ 185 struct nvd_disk *ndisk = arg; 186 struct bio *bp; 187 int err; 188 189 for (;;) { 190 if (atomic_load_acq_int(&ndisk->cur_depth) >= NVME_QD) 191 break; 192 193 mtx_lock(&ndisk->bioqlock); 194 bp = bioq_takefirst(&ndisk->bioq); 195 mtx_unlock(&ndisk->bioqlock); 196 if (bp == NULL) 197 break; 198 199#ifdef BIO_ORDERED 200 /* 201 * BIO_ORDERED flag dictates that all outstanding bios 202 * must be completed before processing the bio with 203 * BIO_ORDERED flag set. 204 */ 205 if (bp->bio_flags & BIO_ORDERED) { 206 while (ndisk->cur_depth > 0) { 207 pause("nvd flush", 1); 208 } 209 } 210#endif 211 212 bp->bio_driver1 = NULL; 213 atomic_add_acq_int(&ndisk->cur_depth, 1); 214 215 err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); 216 217 if (err) { 218 atomic_add_acq_int(&ndisk->cur_depth, -1); 219 bp->bio_error = EIO; 220 bp->bio_flags |= BIO_ERROR; 221 bp->bio_resid = bp->bio_bcount; 222 biodone(bp); 223 } 224 225#ifdef BIO_ORDERED 226 /* 227 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED 228 * flag set must be completed before proceeding with 229 * additional bios. 230 */ 231 if (bp->bio_flags & BIO_ORDERED) { 232 while (ndisk->cur_depth > 0) { 233 pause("nvd flush", 1); 234 } 235 } 236#endif 237 } 238} 239 240static void 241create_geom_disk(void *arg, struct nvme_namespace *ns) 242{ 243 struct nvd_disk *ndisk; 244 struct disk *disk; 245 246 ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_NOWAIT); 247 248 disk = disk_alloc(); 249 disk->d_strategy = nvd_strategy; 250 disk->d_ioctl = nvd_ioctl; 251 disk->d_name = "nvd"; 252 disk->d_drv1 = ndisk; 253 254 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns); 255 disk->d_sectorsize = nvme_ns_get_sector_size(ns); 256 disk->d_mediasize = (off_t)nvme_ns_get_size(ns); 257 258 if (TAILQ_EMPTY(&nvd_head)) 259 disk->d_unit = 0; 260 else 261 disk->d_unit = TAILQ_FIRST(&nvd_head)->disk->d_unit + 1; 262 263 disk->d_flags = 0; 264 265 if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED) 266 disk->d_flags |= DISKFLAG_CANDELETE; 267 268 if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED) 269 disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 270 271 strlcpy(disk->d_ident, nvme_ns_get_serial_number(ns), 272 sizeof(disk->d_ident)); 273 274#if __FreeBSD_version >= 900034 275 strlcpy(disk->d_descr, nvme_ns_get_model_number(ns), 276 sizeof(disk->d_descr)); 277#endif 278 279 disk_create(disk, DISK_VERSION); 280 281 ndisk->ns = ns; 282 ndisk->disk = disk; 283 ndisk->cur_depth = 0; 284 285 mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF); 286 bioq_init(&ndisk->bioq); 287 288 TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk); 289 ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK, 290 taskqueue_thread_enqueue, &ndisk->tq); 291 taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq"); 292 293 TAILQ_INSERT_HEAD(&nvd_head, ndisk, tailq); 294} 295 296static void 297destroy_geom_disk(struct nvd_disk *ndisk) 298{ 299 struct bio *bp; 300 301 taskqueue_free(ndisk->tq); 302 disk_destroy(ndisk->disk); 303 304 mtx_lock(&ndisk->bioqlock); 305 for (;;) { 306 bp = bioq_takefirst(&ndisk->bioq); 307 if (bp == NULL) 308 break; 309 bp->bio_error = EIO; 310 bp->bio_flags |= BIO_ERROR; 311 bp->bio_resid = bp->bio_bcount; 312 313 biodone(bp); 314 } 315 mtx_unlock(&ndisk->bioqlock); 316 317 mtx_destroy(&ndisk->bioqlock); 318} 319