nvd.c revision 330897 
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (C) 2012-2016 Intel Corporation
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/11/sys/dev/nvd/nvd.c 330897 2018-03-14 03:19:51Z eadler $");
31
32#include <sys/param.h>
33#include <sys/bio.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/module.h>
37#include <sys/sysctl.h>
38#include <sys/systm.h>
39#include <sys/taskqueue.h>
40
41#include <geom/geom.h>
42#include <geom/geom_disk.h>
43
44#include <dev/nvme/nvme.h>
45
46#define NVD_STR		"nvd"
47
48struct nvd_disk;
49
50static disk_ioctl_t nvd_ioctl;
51static disk_strategy_t nvd_strategy;
52static dumper_t nvd_dump;
53
54static void nvd_done(void *arg, const struct nvme_completion *cpl);
55
56static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr);
57static void destroy_geom_disk(struct nvd_disk *ndisk);
58
59static void *nvd_new_controller(struct nvme_controller *ctrlr);
60static void nvd_controller_fail(void *ctrlr);
61
62static int nvd_load(void);
63static void nvd_unload(void);
64
65MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations");
66
67struct nvme_consumer *consumer_handle;
68
69struct nvd_disk {
70
71	struct bio_queue_head	bioq;
72	struct task		bioqtask;
73	struct mtx		bioqlock;
74
75	struct disk		*disk;
76	struct taskqueue	*tq;
77	struct nvme_namespace	*ns;
78
79	uint32_t		cur_depth;
80	uint32_t		ordered_in_flight;
81
82	TAILQ_ENTRY(nvd_disk)	global_tailq;
83	TAILQ_ENTRY(nvd_disk)	ctrlr_tailq;
84};
85
86struct nvd_controller {
87
88	TAILQ_ENTRY(nvd_controller)	tailq;
89	TAILQ_HEAD(, nvd_disk)		disk_head;
90};
91
92static TAILQ_HEAD(, nvd_controller)	ctrlr_head;
93static TAILQ_HEAD(disk_list, nvd_disk)	disk_head;
94
95static SYSCTL_NODE(_hw, OID_AUTO, nvd, CTLFLAG_RD, 0, "nvd driver parameters");
96/*
97 * The NVMe specification does not define a maximum or optimal delete size, so
98 *  technically max delete size is min(full size of the namespace, 2^32 - 1
99 *  LBAs).  A single delete for a multi-TB NVMe namespace though may take much
100 *  longer to complete than the nvme(4) I/O timeout period.  So choose a sensible
101 *  default here that is still suitably large to minimize the number of overall
102 *  delete operations.
103 */
104static uint64_t nvd_delete_max = (1024 * 1024 * 1024);  /* 1GB */
105SYSCTL_UQUAD(_hw_nvd, OID_AUTO, delete_max, CTLFLAG_RDTUN, &nvd_delete_max, 0,
106	     "nvd maximum BIO_DELETE size in bytes");
107
108static int nvd_modevent(module_t mod, int type, void *arg)
109{
110	int error = 0;
111
112	switch (type) {
113	case MOD_LOAD:
114		error = nvd_load();
115		break;
116	case MOD_UNLOAD:
117		nvd_unload();
118		break;
119	default:
120		break;
121	}
122
123	return (error);
124}
125
126moduledata_t nvd_mod = {
127	NVD_STR,
128	(modeventhand_t)nvd_modevent,
129	0
130};
131
132DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
133MODULE_VERSION(nvd, 1);
134MODULE_DEPEND(nvd, nvme, 1, 1, 1);
135
136static int
137nvd_load()
138{
139	if (!nvme_use_nvd)
140		return 0;
141
142	TAILQ_INIT(&ctrlr_head);
143	TAILQ_INIT(&disk_head);
144
145	consumer_handle = nvme_register_consumer(nvd_new_disk,
146	    nvd_new_controller, NULL, nvd_controller_fail);
147
148	return (consumer_handle != NULL ? 0 : -1);
149}
150
151static void
152nvd_unload()
153{
154	struct nvd_controller	*ctrlr;
155	struct nvd_disk		*disk;
156
157	if (!nvme_use_nvd)
158		return;
159
160	while (!TAILQ_EMPTY(&ctrlr_head)) {
161		ctrlr = TAILQ_FIRST(&ctrlr_head);
162		TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
163		free(ctrlr, M_NVD);
164	}
165
166	while (!TAILQ_EMPTY(&disk_head)) {
167		disk = TAILQ_FIRST(&disk_head);
168		TAILQ_REMOVE(&disk_head, disk, global_tailq);
169		destroy_geom_disk(disk);
170		free(disk, M_NVD);
171	}
172
173	nvme_unregister_consumer(consumer_handle);
174}
175
176static int
177nvd_bio_submit(struct nvd_disk *ndisk, struct bio *bp)
178{
179	int err;
180
181	bp->bio_driver1 = NULL;
182	atomic_add_int(&ndisk->cur_depth, 1);
183	err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done);
184	if (err) {
185		atomic_add_int(&ndisk->cur_depth, -1);
186		if (__predict_false(bp->bio_flags & BIO_ORDERED))
187			atomic_add_int(&ndisk->ordered_in_flight, -1);
188		bp->bio_error = err;
189		bp->bio_flags |= BIO_ERROR;
190		bp->bio_resid = bp->bio_bcount;
191		biodone(bp);
192		return (-1);
193	}
194
195	return (0);
196}
197
198static void
199nvd_strategy(struct bio *bp)
200{
201	struct nvd_disk *ndisk;
202
203	ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1;
204
205	if (__predict_false(bp->bio_flags & BIO_ORDERED))
206		atomic_add_int(&ndisk->ordered_in_flight, 1);
207
208	if (__predict_true(ndisk->ordered_in_flight == 0)) {
209		nvd_bio_submit(ndisk, bp);
210		return;
211	}
212
213	/*
214	 * There are ordered bios in flight, so we need to submit
215	 *  bios through the task queue to enforce ordering.
216	 */
217	mtx_lock(&ndisk->bioqlock);
218	bioq_insert_tail(&ndisk->bioq, bp);
219	mtx_unlock(&ndisk->bioqlock);
220	taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask);
221}
222
223static int
224nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag,
225    struct thread *td)
226{
227	int ret = 0;
228
229	switch (cmd) {
230	default:
231		ret = EIO;
232	}
233
234	return (ret);
235}
236
237static int
238nvd_dump(void *arg, void *virt, vm_offset_t phys, off_t offset, size_t len)
239{
240	struct nvd_disk *ndisk;
241	struct disk *dp;
242
243	dp = arg;
244	ndisk = dp->d_drv1;
245
246	return (nvme_ns_dump(ndisk->ns, virt, offset, len));
247}
248
249static void
250nvd_done(void *arg, const struct nvme_completion *cpl)
251{
252	struct bio *bp;
253	struct nvd_disk *ndisk;
254
255	bp = (struct bio *)arg;
256
257	ndisk = bp->bio_disk->d_drv1;
258
259	atomic_add_int(&ndisk->cur_depth, -1);
260	if (__predict_false(bp->bio_flags & BIO_ORDERED))
261		atomic_add_int(&ndisk->ordered_in_flight, -1);
262
263	biodone(bp);
264}
265
266static void
267nvd_bioq_process(void *arg, int pending)
268{
269	struct nvd_disk *ndisk = arg;
270	struct bio *bp;
271
272	for (;;) {
273		mtx_lock(&ndisk->bioqlock);
274		bp = bioq_takefirst(&ndisk->bioq);
275		mtx_unlock(&ndisk->bioqlock);
276		if (bp == NULL)
277			break;
278
279		if (nvd_bio_submit(ndisk, bp) != 0) {
280			continue;
281		}
282
283#ifdef BIO_ORDERED
284		/*
285		 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED
286		 *  flag set must be completed before proceeding with
287		 *  additional bios.
288		 */
289		if (bp->bio_flags & BIO_ORDERED) {
290			while (ndisk->cur_depth > 0) {
291				pause("nvd flush", 1);
292			}
293		}
294#endif
295	}
296}
297
298static void *
299nvd_new_controller(struct nvme_controller *ctrlr)
300{
301	struct nvd_controller	*nvd_ctrlr;
302
303	nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD,
304	    M_ZERO | M_WAITOK);
305
306	TAILQ_INIT(&nvd_ctrlr->disk_head);
307	TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq);
308
309	return (nvd_ctrlr);
310}
311
312static void *
313nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg)
314{
315	uint8_t			descr[NVME_MODEL_NUMBER_LENGTH+1];
316	struct nvd_disk		*ndisk;
317	struct disk		*disk;
318	struct nvd_controller	*ctrlr = ctrlr_arg;
319
320	ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK);
321
322	disk = disk_alloc();
323	disk->d_strategy = nvd_strategy;
324	disk->d_ioctl = nvd_ioctl;
325	disk->d_dump = nvd_dump;
326	disk->d_name = NVD_STR;
327	disk->d_drv1 = ndisk;
328
329	disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns);
330	disk->d_sectorsize = nvme_ns_get_sector_size(ns);
331	disk->d_mediasize = (off_t)nvme_ns_get_size(ns);
332	disk->d_delmaxsize = (off_t)nvme_ns_get_size(ns);
333	if (disk->d_delmaxsize > nvd_delete_max)
334		disk->d_delmaxsize = nvd_delete_max;
335	disk->d_stripesize = nvme_ns_get_stripesize(ns);
336
337	if (TAILQ_EMPTY(&disk_head))
338		disk->d_unit = 0;
339	else
340		disk->d_unit =
341		    TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1;
342
343	disk->d_flags = DISKFLAG_DIRECT_COMPLETION;
344
345	if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED)
346		disk->d_flags |= DISKFLAG_CANDELETE;
347
348	if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED)
349		disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
350
351/* ifdef used here to ease porting to stable branches at a later point. */
352#ifdef DISKFLAG_UNMAPPED_BIO
353	disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
354#endif
355
356	/*
357	 * d_ident and d_descr are both far bigger than the length of either
358	 *  the serial or model number strings.
359	 */
360	nvme_strvis(disk->d_ident, nvme_ns_get_serial_number(ns),
361	    sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH);
362	nvme_strvis(descr, nvme_ns_get_model_number(ns), sizeof(descr),
363	    NVME_MODEL_NUMBER_LENGTH);
364	strlcpy(disk->d_descr, descr, sizeof(descr));
365
366	disk->d_rotation_rate = DISK_RR_NON_ROTATING;
367
368	ndisk->ns = ns;
369	ndisk->disk = disk;
370	ndisk->cur_depth = 0;
371	ndisk->ordered_in_flight = 0;
372
373	mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF);
374	bioq_init(&ndisk->bioq);
375
376	TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk);
377	ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK,
378	    taskqueue_thread_enqueue, &ndisk->tq);
379	taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq");
380
381	TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq);
382	TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq);
383
384	disk_create(disk, DISK_VERSION);
385
386	printf(NVD_STR"%u: <%s> NVMe namespace\n", disk->d_unit, descr);
387	printf(NVD_STR"%u: %juMB (%ju %u byte sectors)\n", disk->d_unit,
388		(uintmax_t)disk->d_mediasize / (1024*1024),
389		(uintmax_t)disk->d_mediasize / disk->d_sectorsize,
390		disk->d_sectorsize);
391
392	return (NULL);
393}
394
395static void
396destroy_geom_disk(struct nvd_disk *ndisk)
397{
398	struct bio	*bp;
399	struct disk	*disk;
400	uint32_t	unit;
401	int		cnt = 0;
402
403	disk = ndisk->disk;
404	unit = disk->d_unit;
405	taskqueue_free(ndisk->tq);
406
407	disk_destroy(ndisk->disk);
408
409	mtx_lock(&ndisk->bioqlock);
410	for (;;) {
411		bp = bioq_takefirst(&ndisk->bioq);
412		if (bp == NULL)
413			break;
414		bp->bio_error = EIO;
415		bp->bio_flags |= BIO_ERROR;
416		bp->bio_resid = bp->bio_bcount;
417		cnt++;
418		biodone(bp);
419	}
420
421	printf(NVD_STR"%u: lost device - %d outstanding\n", unit, cnt);
422	printf(NVD_STR"%u: removing device entry\n", unit);
423
424	mtx_unlock(&ndisk->bioqlock);
425
426	mtx_destroy(&ndisk->bioqlock);
427}
428
429static void
430nvd_controller_fail(void *ctrlr_arg)
431{
432	struct nvd_controller	*ctrlr = ctrlr_arg;
433	struct nvd_disk		*disk;
434
435	while (!TAILQ_EMPTY(&ctrlr->disk_head)) {
436		disk = TAILQ_FIRST(&ctrlr->disk_head);
437		TAILQ_REMOVE(&disk_head, disk, global_tailq);
438		TAILQ_REMOVE(&ctrlr->disk_head, disk, ctrlr_tailq);
439		destroy_geom_disk(disk);
440		free(disk, M_NVD);
441	}
442
443	TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
444	free(ctrlr, M_NVD);
445}
446
447