dev/nvd/nvd.c

63 static void nvd_gone(struct nvd_disk *ndisk);
169 	struct nvd_disk		*ndisk;
177 		TAILQ_FOREACH(ndisk, &ctrlr->disk_head, ctrlr_tailq)
178 			nvd_gone(ndisk);
191 nvd_bio_submit(struct nvd_disk *ndisk, struct bio *bp)
197 		atomic_add_int(&ndisk->cur_depth, NVD_ODEPTH);
199 		atomic_add_int(&ndisk->cur_depth, 1);
200 	err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done);
203 			atomic_add_int(&ndisk->cur_depth, -NVD_ODEPTH);
204 			atomic_add_int(&ndisk->ordered_in_flight, -1);
205 			wakeup(&ndisk->cur_depth);
207 			if (atomic_fetchadd_int(&ndisk->cur_depth, -1) == 1 &&
208 			    __predict_false(ndisk->ordered_in_flight != 0))
209 				wakeup(&ndisk->cur_depth);
221 	struct nvd_disk *ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1;
228 		if (atomic_fetchadd_int(&ndisk->ordered_in_flight, 1) == 0 &&
229 		    ndisk->cur_depth == 0 && bioq_first(&ndisk->bioq) == NULL) {
230 			nvd_bio_submit(ndisk, bp);
233 	} else if (__predict_true(ndisk->ordered_in_flight == 0)) {
234 		nvd_bio_submit(ndisk, bp);
242 	mtx_lock(&ndisk->bioqlock);
243 	bioq_insert_tail(&ndisk->bioq, bp);
244 	mtx_unlock(&ndisk->bioqlock);
245 	taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask);
249 nvd_gone(struct nvd_disk *ndisk)
253 	printf(NVD_STR"%u: detached\n", ndisk->unit);
254 	mtx_lock(&ndisk->bioqlock);
255 	disk_gone(ndisk->disk);
256 	while ((bp = bioq_takefirst(&ndisk->bioq)) != NULL) {
258 			atomic_add_int(&ndisk->ordered_in_flight, -1);
264 	mtx_unlock(&ndisk->bioqlock);
270 	struct nvd_disk *ndisk = (struct nvd_disk *)dp->d_drv1;
272 	disk_destroy(ndisk->disk);
274 	TAILQ_REMOVE(&disk_head, ndisk, global_tailq);
275 	TAILQ_REMOVE(&ndisk->ctrlr->disk_head, ndisk, ctrlr_tailq);
276 	if (TAILQ_EMPTY(&ndisk->ctrlr->disk_head))
277 		wakeup(&ndisk->ctrlr->disk_head);
279 	taskqueue_free(ndisk->tq);
280 	mtx_destroy(&ndisk->bioqlock);
281 	free(ndisk, M_NVD);
288 	struct nvd_disk		*ndisk = dp->d_drv1;
290 	return (nvme_ns_ioctl_process(ndisk->ns, cmd, data, fflag, td));
297 	struct nvd_disk *ndisk = dp->d_drv1;
299 	return (nvme_ns_dump(ndisk->ns, virt, offset, len));
305 	struct nvd_disk *ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1;
310 		nsdata = nvme_ns_get_data(ndisk->ns);
351 	struct nvd_disk *ndisk = bp->bio_disk->d_drv1;
354 		atomic_add_int(&ndisk->cur_depth, -NVD_ODEPTH);
355 		atomic_add_int(&ndisk->ordered_in_flight, -1);
356 		wakeup(&ndisk->cur_depth);
358 		if (atomic_fetchadd_int(&ndisk->cur_depth, -1) == 1 &&
359 		    __predict_false(ndisk->ordered_in_flight != 0))
360 			wakeup(&ndisk->cur_depth);
369 	struct nvd_disk *ndisk = arg;
373 		mtx_lock(&ndisk->bioqlock);
374 		bp = bioq_takefirst(&ndisk->bioq);
375 		mtx_unlock(&ndisk->bioqlock);
384 			while (ndisk->cur_depth > 0)
385 				tsleep(&ndisk->cur_depth, 0, "nvdorb", 1);
391 			while (ndisk->cur_depth >= NVD_ODEPTH)
392 				tsleep(&ndisk->cur_depth, 0, "nvdora", 1);
395 		nvd_bio_submit(ndisk, bp);
420 	struct nvd_disk		*ndisk, *tnd;
426 	ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK);
427 	ndisk->ctrlr = ctrlr;
428 	ndisk->ns = ns;
429 	ndisk->cur_depth = 0;
430 	ndisk->ordered_in_flight = 0;
431 	mtx_init(&ndisk->bioqlock, "nvd bioq lock", NULL, MTX_DEF);
432 	bioq_init(&ndisk->bioq);
433 	TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk);
442 	ndisk->unit = unit;
444 		TAILQ_INSERT_BEFORE(tnd, ndisk, global_tailq);
446 		TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq);
447 	TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq);
450 	ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK,
451 	    taskqueue_thread_enqueue, &ndisk->tq);
452 	taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq");
454 	disk = ndisk->disk = disk_alloc();
461 	disk->d_unit = ndisk->unit;
462 	disk->d_drv1 = ndisk;
503 	return (ndisk);
510 	struct nvd_disk		*ndisk;
514 	TAILQ_FOREACH(ndisk, &ctrlr->disk_head, ctrlr_tailq)
515 		nvd_gone(ndisk);