nvd.c revision 248765
1/*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/nvd/nvd.c 248765 2013-03-26 21:45:37Z jimharris $"); 29 30#include <sys/param.h> 31#include <sys/bio.h> 32#include <sys/kernel.h> 33#include <sys/malloc.h> 34#include <sys/module.h> 35#include <sys/systm.h> 36#include <sys/taskqueue.h> 37 38#include <geom/geom.h> 39#include <geom/geom_disk.h> 40 41#include <dev/nvme/nvme.h> 42 43struct nvd_disk; 44 45static disk_ioctl_t nvd_ioctl; 46static disk_strategy_t nvd_strategy; 47 48static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr); 49static void destroy_geom_disk(struct nvd_disk *ndisk); 50 51static void *nvd_new_controller(struct nvme_controller *ctrlr); 52 53static int nvd_load(void); 54static void nvd_unload(void); 55 56MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations"); 57 58struct nvme_consumer *consumer_handle; 59 60struct nvd_disk { 61 62 struct bio_queue_head bioq; 63 struct task bioqtask; 64 struct mtx bioqlock; 65 66 struct disk *disk; 67 struct taskqueue *tq; 68 struct nvme_namespace *ns; 69 70 uint32_t cur_depth; 71 72 TAILQ_ENTRY(nvd_disk) global_tailq; 73 TAILQ_ENTRY(nvd_disk) ctrlr_tailq; 74}; 75 76struct nvd_controller { 77 78 TAILQ_ENTRY(nvd_controller) tailq; 79 TAILQ_HEAD(, nvd_disk) disk_head; 80}; 81 82static TAILQ_HEAD(, nvd_controller) ctrlr_head; 83static TAILQ_HEAD(disk_list, nvd_disk) disk_head; 84 85static int nvd_modevent(module_t mod, int type, void *arg) 86{ 87 int error = 0; 88 89 switch (type) { 90 case MOD_LOAD: 91 error = nvd_load(); 92 break; 93 case MOD_UNLOAD: 94 nvd_unload(); 95 break; 96 default: 97 break; 98 } 99 100 return (error); 101} 102 103moduledata_t nvd_mod = { 104 "nvd", 105 (modeventhand_t)nvd_modevent, 106 0 107}; 108 109DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 110MODULE_VERSION(nvd, 1); 111MODULE_DEPEND(nvd, nvme, 1, 1, 1); 112 113static int 114nvd_load() 115{ 116 117 TAILQ_INIT(&ctrlr_head); 118 TAILQ_INIT(&disk_head); 119 120 consumer_handle = nvme_register_consumer(nvd_new_disk, 121 nvd_new_controller, NULL); 122 123 return (consumer_handle != NULL ? 0 : -1); 124} 125 126static void 127nvd_unload() 128{ 129 struct nvd_controller *ctrlr; 130 struct nvd_disk *disk; 131 132 while (!TAILQ_EMPTY(&ctrlr_head)) { 133 ctrlr = TAILQ_FIRST(&ctrlr_head); 134 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 135 free(ctrlr, M_NVD); 136 } 137 138 while (!TAILQ_EMPTY(&disk_head)) { 139 disk = TAILQ_FIRST(&disk_head); 140 TAILQ_REMOVE(&disk_head, disk, global_tailq); 141 destroy_geom_disk(disk); 142 free(disk, M_NVD); 143 } 144 145 nvme_unregister_consumer(consumer_handle); 146} 147 148static void 149nvd_strategy(struct bio *bp) 150{ 151 struct nvd_disk *ndisk; 152 153 ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1; 154 155 mtx_lock(&ndisk->bioqlock); 156 bioq_insert_tail(&ndisk->bioq, bp); 157 mtx_unlock(&ndisk->bioqlock); 158 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 159} 160 161static int 162nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag, 163 struct thread *td) 164{ 165 int ret = 0; 166 167 switch (cmd) { 168 default: 169 ret = EIO; 170 } 171 172 return (ret); 173} 174 175static void 176nvd_done(void *arg, const struct nvme_completion *cpl) 177{ 178 struct bio *bp; 179 struct nvd_disk *ndisk; 180 181 bp = (struct bio *)arg; 182 183 ndisk = bp->bio_disk->d_drv1; 184 185 atomic_add_int(&ndisk->cur_depth, -1); 186 187 /* 188 * TODO: add more extensive translation of NVMe status codes 189 * to different bio error codes (i.e. EIO, EINVAL, etc.) 190 */ 191 if (nvme_completion_is_error(cpl)) { 192 bp->bio_error = EIO; 193 bp->bio_flags |= BIO_ERROR; 194 bp->bio_resid = bp->bio_bcount; 195 } else 196 bp->bio_resid = 0; 197 198 biodone(bp); 199} 200 201static void 202nvd_bioq_process(void *arg, int pending) 203{ 204 struct nvd_disk *ndisk = arg; 205 struct bio *bp; 206 int err; 207 208 for (;;) { 209 mtx_lock(&ndisk->bioqlock); 210 bp = bioq_takefirst(&ndisk->bioq); 211 mtx_unlock(&ndisk->bioqlock); 212 if (bp == NULL) 213 break; 214 215#ifdef BIO_ORDERED 216 /* 217 * BIO_ORDERED flag dictates that all outstanding bios 218 * must be completed before processing the bio with 219 * BIO_ORDERED flag set. 220 */ 221 if (bp->bio_flags & BIO_ORDERED) { 222 while (ndisk->cur_depth > 0) { 223 pause("nvd flush", 1); 224 } 225 } 226#endif 227 228 bp->bio_driver1 = NULL; 229 atomic_add_int(&ndisk->cur_depth, 1); 230 231 err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); 232 233 if (err) { 234 atomic_add_int(&ndisk->cur_depth, -1); 235 bp->bio_error = err; 236 bp->bio_flags |= BIO_ERROR; 237 bp->bio_resid = bp->bio_bcount; 238 biodone(bp); 239 } 240 241#ifdef BIO_ORDERED 242 /* 243 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED 244 * flag set must be completed before proceeding with 245 * additional bios. 246 */ 247 if (bp->bio_flags & BIO_ORDERED) { 248 while (ndisk->cur_depth > 0) { 249 pause("nvd flush", 1); 250 } 251 } 252#endif 253 } 254} 255 256static void * 257nvd_new_controller(struct nvme_controller *ctrlr) 258{ 259 struct nvd_controller *nvd_ctrlr; 260 261 nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD, 262 M_ZERO | M_NOWAIT); 263 264 TAILQ_INIT(&nvd_ctrlr->disk_head); 265 TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq); 266 267 return (nvd_ctrlr); 268} 269 270static void * 271nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg) 272{ 273 struct nvd_disk *ndisk; 274 struct disk *disk; 275 struct nvd_controller *ctrlr = ctrlr_arg; 276 277 ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_NOWAIT); 278 279 disk = disk_alloc(); 280 disk->d_strategy = nvd_strategy; 281 disk->d_ioctl = nvd_ioctl; 282 disk->d_name = "nvd"; 283 disk->d_drv1 = ndisk; 284 285 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns); 286 disk->d_sectorsize = nvme_ns_get_sector_size(ns); 287 disk->d_mediasize = (off_t)nvme_ns_get_size(ns); 288 289 if (TAILQ_EMPTY(&disk_head)) 290 disk->d_unit = 0; 291 else 292 disk->d_unit = 293 TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1; 294 295 disk->d_flags = 0; 296 297 if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED) 298 disk->d_flags |= DISKFLAG_CANDELETE; 299 300 if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED) 301 disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 302 303 strlcpy(disk->d_ident, nvme_ns_get_serial_number(ns), 304 sizeof(disk->d_ident)); 305 306#if __FreeBSD_version >= 900034 307 strlcpy(disk->d_descr, nvme_ns_get_model_number(ns), 308 sizeof(disk->d_descr)); 309#endif 310 311 disk_create(disk, DISK_VERSION); 312 313 ndisk->ns = ns; 314 ndisk->disk = disk; 315 ndisk->cur_depth = 0; 316 317 mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF); 318 bioq_init(&ndisk->bioq); 319 320 TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk); 321 ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK, 322 taskqueue_thread_enqueue, &ndisk->tq); 323 taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq"); 324 325 TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq); 326 TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq); 327 328 return (NULL); 329} 330 331static void 332destroy_geom_disk(struct nvd_disk *ndisk) 333{ 334 struct bio *bp; 335 336 taskqueue_free(ndisk->tq); 337 disk_destroy(ndisk->disk); 338 339 mtx_lock(&ndisk->bioqlock); 340 for (;;) { 341 bp = bioq_takefirst(&ndisk->bioq); 342 if (bp == NULL) 343 break; 344 bp->bio_error = EIO; 345 bp->bio_flags |= BIO_ERROR; 346 bp->bio_resid = bp->bio_bcount; 347 348 biodone(bp); 349 } 350 mtx_unlock(&ndisk->bioqlock); 351 352 mtx_destroy(&ndisk->bioqlock); 353} 354