Deleted Added
sdiff udiff text old ( 228301 ) new ( 234270 )
full compact
virtio_blk.c (228301) virtio_blk.c (234270)
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 13 unchanged lines hidden (view full) ---

22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for VirtIO block devices. */
28
29#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 13 unchanged lines hidden (view full) ---

22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for VirtIO block devices. */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: head/sys/dev/virtio/block/virtio_blk.c 228301 2011-12-06 06:28:32Z grehan $");
30__FBSDID("$FreeBSD: head/sys/dev/virtio/block/virtio_blk.c 234270 2012-04-14 05:48:04Z grehan $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/bio.h>
36#include <sys/malloc.h>
37#include <sys/module.h>
38#include <sys/sglist.h>

--- 26 unchanged lines hidden (view full) ---

65
66struct vtblk_softc {
67 device_t vtblk_dev;
68 struct mtx vtblk_mtx;
69 uint64_t vtblk_features;
70 uint32_t vtblk_flags;
71#define VTBLK_FLAG_INDIRECT 0x0001
72#define VTBLK_FLAG_READONLY 0x0002
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/bio.h>
36#include <sys/malloc.h>
37#include <sys/module.h>
38#include <sys/sglist.h>

--- 26 unchanged lines hidden (view full) ---

65
66struct vtblk_softc {
67 device_t vtblk_dev;
68 struct mtx vtblk_mtx;
69 uint64_t vtblk_features;
70 uint32_t vtblk_flags;
71#define VTBLK_FLAG_INDIRECT 0x0001
72#define VTBLK_FLAG_READONLY 0x0002
73#define VTBLK_FLAG_DETACHING 0x0004
74#define VTBLK_FLAG_SUSPENDED 0x0008
73#define VTBLK_FLAG_DETACH 0x0004
74#define VTBLK_FLAG_SUSPEND 0x0008
75#define VTBLK_FLAG_DUMPING 0x0010
76
77 struct virtqueue *vtblk_vq;
78 struct sglist *vtblk_sglist;
79 struct disk *vtblk_disk;
80
81 struct bio_queue_head vtblk_bioq;
82 TAILQ_HEAD(, vtblk_request)
83 vtblk_req_free;
84 TAILQ_HEAD(, vtblk_request)
75#define VTBLK_FLAG_DUMPING 0x0010
76
77 struct virtqueue *vtblk_vq;
78 struct sglist *vtblk_sglist;
79 struct disk *vtblk_disk;
80
81 struct bio_queue_head vtblk_bioq;
82 TAILQ_HEAD(, vtblk_request)
83 vtblk_req_free;
84 TAILQ_HEAD(, vtblk_request)
85 vtblk_req_ready;
85 vtblk_req_ready;
86
87 struct taskqueue *vtblk_tq;
88 struct task vtblk_intr_task;
89
90 int vtblk_max_nsegs;
91 int vtblk_request_count;
92
93 struct vtblk_request vtblk_dump_request;

--- 17 unchanged lines hidden (view full) ---

111
112static int vtblk_probe(device_t);
113static int vtblk_attach(device_t);
114static int vtblk_detach(device_t);
115static int vtblk_suspend(device_t);
116static int vtblk_resume(device_t);
117static int vtblk_shutdown(device_t);
118
86
87 struct taskqueue *vtblk_tq;
88 struct task vtblk_intr_task;
89
90 int vtblk_max_nsegs;
91 int vtblk_request_count;
92
93 struct vtblk_request vtblk_dump_request;

--- 17 unchanged lines hidden (view full) ---

111
112static int vtblk_probe(device_t);
113static int vtblk_attach(device_t);
114static int vtblk_detach(device_t);
115static int vtblk_suspend(device_t);
116static int vtblk_resume(device_t);
117static int vtblk_shutdown(device_t);
118
119static int vtblk_open(struct disk *);
120static int vtblk_close(struct disk *);
121static int vtblk_ioctl(struct disk *, u_long, void *, int,
122 struct thread *);
123static int vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
124static void vtblk_strategy(struct bio *);
125
119static void vtblk_negotiate_features(struct vtblk_softc *);
120static int vtblk_maximum_segments(struct vtblk_softc *,
121 struct virtio_blk_config *);
122static int vtblk_alloc_virtqueue(struct vtblk_softc *);
123static void vtblk_alloc_disk(struct vtblk_softc *,
124 struct virtio_blk_config *);
125static void vtblk_create_disk(struct vtblk_softc *);
126
126static void vtblk_negotiate_features(struct vtblk_softc *);
127static int vtblk_maximum_segments(struct vtblk_softc *,
128 struct virtio_blk_config *);
129static int vtblk_alloc_virtqueue(struct vtblk_softc *);
130static void vtblk_alloc_disk(struct vtblk_softc *,
131 struct virtio_blk_config *);
132static void vtblk_create_disk(struct vtblk_softc *);
133
127static int vtblk_open(struct disk *);
128static int vtblk_close(struct disk *);
129static int vtblk_ioctl(struct disk *, u_long, void *, int,
130 struct thread *);
131static int vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
132static void vtblk_strategy(struct bio *);
133
134static int vtblk_quiesce(struct vtblk_softc *);
134static void vtblk_startio(struct vtblk_softc *);
135static struct vtblk_request * vtblk_bio_request(struct vtblk_softc *);
136static int vtblk_execute_request(struct vtblk_softc *,
137 struct vtblk_request *);
138
139static int vtblk_vq_intr(void *);
140static void vtblk_intr_task(void *, int);
141
142static void vtblk_stop(struct vtblk_softc *);
143
144static void vtblk_get_ident(struct vtblk_softc *);
145static void vtblk_prepare_dump(struct vtblk_softc *);
146static int vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
147static int vtblk_flush_dump(struct vtblk_softc *);
148static int vtblk_poll_request(struct vtblk_softc *,
149 struct vtblk_request *);
150
135static void vtblk_startio(struct vtblk_softc *);
136static struct vtblk_request * vtblk_bio_request(struct vtblk_softc *);
137static int vtblk_execute_request(struct vtblk_softc *,
138 struct vtblk_request *);
139
140static int vtblk_vq_intr(void *);
141static void vtblk_intr_task(void *, int);
142
143static void vtblk_stop(struct vtblk_softc *);
144
145static void vtblk_get_ident(struct vtblk_softc *);
146static void vtblk_prepare_dump(struct vtblk_softc *);
147static int vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
148static int vtblk_flush_dump(struct vtblk_softc *);
149static int vtblk_poll_request(struct vtblk_softc *,
150 struct vtblk_request *);
151
152static void vtblk_finish_completed(struct vtblk_softc *);
151static void vtblk_drain_vq(struct vtblk_softc *, int);
152static void vtblk_drain(struct vtblk_softc *);
153
154static int vtblk_alloc_requests(struct vtblk_softc *);
155static void vtblk_free_requests(struct vtblk_softc *);
156static struct vtblk_request * vtblk_dequeue_request(struct vtblk_softc *);
157static void vtblk_enqueue_request(struct vtblk_softc *,
158 struct vtblk_request *);
159
160static struct vtblk_request * vtblk_dequeue_ready(struct vtblk_softc *);
161static void vtblk_enqueue_ready(struct vtblk_softc *,
162 struct vtblk_request *);
163
153static void vtblk_drain_vq(struct vtblk_softc *, int);
154static void vtblk_drain(struct vtblk_softc *);
155
156static int vtblk_alloc_requests(struct vtblk_softc *);
157static void vtblk_free_requests(struct vtblk_softc *);
158static struct vtblk_request * vtblk_dequeue_request(struct vtblk_softc *);
159static void vtblk_enqueue_request(struct vtblk_softc *,
160 struct vtblk_request *);
161
162static struct vtblk_request * vtblk_dequeue_ready(struct vtblk_softc *);
163static void vtblk_enqueue_ready(struct vtblk_softc *,
164 struct vtblk_request *);
165
164static void vtblk_bio_error(struct bio *, int);
166static int vtblk_request_error(struct vtblk_request *);
167static void vtblk_finish_bio(struct bio *, int);
165
166/* Tunables. */
167static int vtblk_no_ident = 0;
168TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
169
170/* Features desired/implemented by this driver. */
171#define VTBLK_FEATURES \
172 (VIRTIO_BLK_F_BARRIER | \

--- 11 unchanged lines hidden (view full) ---

184 "VTBLK Lock", MTX_DEF)
185#define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc)))
186#define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc)))
187#define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc)))
188#define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED)
189#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \
190 mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED)
191
168
169/* Tunables. */
170static int vtblk_no_ident = 0;
171TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
172
173/* Features desired/implemented by this driver. */
174#define VTBLK_FEATURES \
175 (VIRTIO_BLK_F_BARRIER | \

--- 11 unchanged lines hidden (view full) ---

187 "VTBLK Lock", MTX_DEF)
188#define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc)))
189#define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc)))
190#define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc)))
191#define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED)
192#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \
193 mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED)
194
192#define VTBLK_BIO_SEGMENTS(_bp) sglist_count((_bp)->bio_data, (_bp)->bio_bcount)
193
194#define VTBLK_DISK_NAME "vtbd"
195#define VTBLK_DISK_NAME "vtbd"
196#define VTBLK_QUIESCE_TIMEOUT (30 * hz)
195
196/*
197 * Each block request uses at least two segments - one for the header
198 * and one for the status.
199 */
200#define VTBLK_MIN_SEGMENTS 2
201
202static uma_zone_t vtblk_req_zone;
203
204static device_method_t vtblk_methods[] = {
205 /* Device methods. */
206 DEVMETHOD(device_probe, vtblk_probe),
207 DEVMETHOD(device_attach, vtblk_attach),
208 DEVMETHOD(device_detach, vtblk_detach),
209 DEVMETHOD(device_suspend, vtblk_suspend),
210 DEVMETHOD(device_resume, vtblk_resume),
211 DEVMETHOD(device_shutdown, vtblk_shutdown),
212
197
198/*
199 * Each block request uses at least two segments - one for the header
200 * and one for the status.
201 */
202#define VTBLK_MIN_SEGMENTS 2
203
204static uma_zone_t vtblk_req_zone;
205
206static device_method_t vtblk_methods[] = {
207 /* Device methods. */
208 DEVMETHOD(device_probe, vtblk_probe),
209 DEVMETHOD(device_attach, vtblk_attach),
210 DEVMETHOD(device_detach, vtblk_detach),
211 DEVMETHOD(device_suspend, vtblk_suspend),
212 DEVMETHOD(device_resume, vtblk_resume),
213 DEVMETHOD(device_shutdown, vtblk_shutdown),
214
213 { 0, 0 }
215 DEVMETHOD_END
214};
215
216static driver_t vtblk_driver = {
217 "vtblk",
218 vtblk_methods,
219 sizeof(struct vtblk_softc)
220};
221static devclass_t vtblk_devclass;

--- 87 unchanged lines hidden (view full) ---

309 error = ENOTSUP;
310 device_printf(dev, "host requires unsupported "
311 "maximum segment size feature\n");
312 goto fail;
313 }
314 }
315
316 sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
216};
217
218static driver_t vtblk_driver = {
219 "vtblk",
220 vtblk_methods,
221 sizeof(struct vtblk_softc)
222};
223static devclass_t vtblk_devclass;

--- 87 unchanged lines hidden (view full) ---

311 error = ENOTSUP;
312 device_printf(dev, "host requires unsupported "
313 "maximum segment size feature\n");
314 goto fail;
315 }
316 }
317
318 sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
319 if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
320 error = EINVAL;
321 device_printf(dev, "fewer than minimum number of segments "
322 "allowed: %d\n", sc->vtblk_max_nsegs);
323 goto fail;
324 }
317
325
318 /*
319 * Allocate working sglist. The number of segments may be too
320 * large to safely store on the stack.
321 */
322 sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
323 if (sc->vtblk_sglist == NULL) {
324 error = ENOMEM;
325 device_printf(dev, "cannot allocate sglist\n");
326 goto fail;
327 }
328
329 error = vtblk_alloc_virtqueue(sc);

--- 41 unchanged lines hidden (view full) ---

371static int
372vtblk_detach(device_t dev)
373{
374 struct vtblk_softc *sc;
375
376 sc = device_get_softc(dev);
377
378 VTBLK_LOCK(sc);
326 sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
327 if (sc->vtblk_sglist == NULL) {
328 error = ENOMEM;
329 device_printf(dev, "cannot allocate sglist\n");
330 goto fail;
331 }
332
333 error = vtblk_alloc_virtqueue(sc);

--- 41 unchanged lines hidden (view full) ---

375static int
376vtblk_detach(device_t dev)
377{
378 struct vtblk_softc *sc;
379
380 sc = device_get_softc(dev);
381
382 VTBLK_LOCK(sc);
379 sc->vtblk_flags |= VTBLK_FLAG_DETACHING;
383 sc->vtblk_flags |= VTBLK_FLAG_DETACH;
380 if (device_is_attached(dev))
381 vtblk_stop(sc);
382 VTBLK_UNLOCK(sc);
383
384 if (sc->vtblk_tq != NULL) {
385 taskqueue_drain(sc->vtblk_tq, &sc->vtblk_intr_task);
386 taskqueue_free(sc->vtblk_tq);
387 sc->vtblk_tq = NULL;

--- 15 unchanged lines hidden (view full) ---

403
404 return (0);
405}
406
407static int
408vtblk_suspend(device_t dev)
409{
410 struct vtblk_softc *sc;
384 if (device_is_attached(dev))
385 vtblk_stop(sc);
386 VTBLK_UNLOCK(sc);
387
388 if (sc->vtblk_tq != NULL) {
389 taskqueue_drain(sc->vtblk_tq, &sc->vtblk_intr_task);
390 taskqueue_free(sc->vtblk_tq);
391 sc->vtblk_tq = NULL;

--- 15 unchanged lines hidden (view full) ---

407
408 return (0);
409}
410
411static int
412vtblk_suspend(device_t dev)
413{
414 struct vtblk_softc *sc;
415 int error;
411
412 sc = device_get_softc(dev);
413
414 VTBLK_LOCK(sc);
416
417 sc = device_get_softc(dev);
418
419 VTBLK_LOCK(sc);
415 sc->vtblk_flags |= VTBLK_FLAG_SUSPENDED;
416 /* TODO Wait for any inflight IO to complete? */
420 sc->vtblk_flags |= VTBLK_FLAG_SUSPEND;
421 /* XXX BMV: virtio_stop(), etc needed here? */
422 error = vtblk_quiesce(sc);
423 if (error)
424 sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
417 VTBLK_UNLOCK(sc);
418
425 VTBLK_UNLOCK(sc);
426
419 return (0);
427 return (error);
420}
421
422static int
423vtblk_resume(device_t dev)
424{
425 struct vtblk_softc *sc;
426
427 sc = device_get_softc(dev);
428
429 VTBLK_LOCK(sc);
428}
429
430static int
431vtblk_resume(device_t dev)
432{
433 struct vtblk_softc *sc;
434
435 sc = device_get_softc(dev);
436
437 VTBLK_LOCK(sc);
430 sc->vtblk_flags &= ~VTBLK_FLAG_SUSPENDED;
431 /* TODO Resume IO? */
438 /* XXX BMV: virtio_reinit(), etc needed here? */
439 sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
440 vtblk_startio(sc);
432 VTBLK_UNLOCK(sc);
433
434 return (0);
435}
436
437static int
438vtblk_shutdown(device_t dev)
439{

--- 4 unchanged lines hidden (view full) ---

444static int
445vtblk_open(struct disk *dp)
446{
447 struct vtblk_softc *sc;
448
449 if ((sc = dp->d_drv1) == NULL)
450 return (ENXIO);
451
441 VTBLK_UNLOCK(sc);
442
443 return (0);
444}
445
446static int
447vtblk_shutdown(device_t dev)
448{

--- 4 unchanged lines hidden (view full) ---

453static int
454vtblk_open(struct disk *dp)
455{
456 struct vtblk_softc *sc;
457
458 if ((sc = dp->d_drv1) == NULL)
459 return (ENXIO);
460
452 return (sc->vtblk_flags & VTBLK_FLAG_DETACHING ? ENXIO : 0);
461 return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0);
453}
454
455static int
456vtblk_close(struct disk *dp)
457{
458 struct vtblk_softc *sc;
459
460 if ((sc = dp->d_drv1) == NULL)

--- 23 unchanged lines hidden (view full) ---

484 int error;
485
486 dp = arg;
487 error = 0;
488
489 if ((sc = dp->d_drv1) == NULL)
490 return (ENXIO);
491
462}
463
464static int
465vtblk_close(struct disk *dp)
466{
467 struct vtblk_softc *sc;
468
469 if ((sc = dp->d_drv1) == NULL)

--- 23 unchanged lines hidden (view full) ---

493 int error;
494
495 dp = arg;
496 error = 0;
497
498 if ((sc = dp->d_drv1) == NULL)
499 return (ENXIO);
500
501 VTBLK_LOCK(sc);
502
492 if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
493 vtblk_prepare_dump(sc);
494 sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
495 }
496
497 if (length > 0)
498 error = vtblk_write_dump(sc, virtual, offset, length);
499 else if (virtual == NULL && offset == 0)
500 error = vtblk_flush_dump(sc);
503 if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
504 vtblk_prepare_dump(sc);
505 sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
506 }
507
508 if (length > 0)
509 error = vtblk_write_dump(sc, virtual, offset, length);
510 else if (virtual == NULL && offset == 0)
511 error = vtblk_flush_dump(sc);
512 else {
513 error = EINVAL;
514 sc->vtblk_flags &= ~VTBLK_FLAG_DUMPING;
515 }
501
502 VTBLK_UNLOCK(sc);
503
504 return (error);
505}
506
507static void
508vtblk_strategy(struct bio *bp)
509{
510 struct vtblk_softc *sc;
511
512 if ((sc = bp->bio_disk->d_drv1) == NULL) {
516
517 VTBLK_UNLOCK(sc);
518
519 return (error);
520}
521
522static void
523vtblk_strategy(struct bio *bp)
524{
525 struct vtblk_softc *sc;
526
527 if ((sc = bp->bio_disk->d_drv1) == NULL) {
513 vtblk_bio_error(bp, EINVAL);
528 vtblk_finish_bio(bp, EINVAL);
514 return;
515 }
516
517 /*
518 * Fail any write if RO. Unfortunately, there does not seem to
519 * be a better way to report our readonly'ness to GEOM above.
520 */
521 if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
522 (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
529 return;
530 }
531
532 /*
533 * Fail any write if RO. Unfortunately, there does not seem to
534 * be a better way to report our readonly'ness to GEOM above.
535 */
536 if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
537 (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
523 vtblk_bio_error(bp, EROFS);
538 vtblk_finish_bio(bp, EROFS);
524 return;
525 }
526
539 return;
540 }
541
542#ifdef INVARIANTS
527 /*
528 * Prevent read/write buffers spanning too many segments from
529 * getting into the queue. This should only trip if d_maxsize
530 * was incorrectly set.
531 */
532 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
543 /*
544 * Prevent read/write buffers spanning too many segments from
545 * getting into the queue. This should only trip if d_maxsize
546 * was incorrectly set.
547 */
548 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
533 KASSERT(VTBLK_BIO_SEGMENTS(bp) <= sc->vtblk_max_nsegs -
534 VTBLK_MIN_SEGMENTS,
549 int nsegs, max_nsegs;
550
551 nsegs = sglist_count(bp->bio_data, bp->bio_bcount);
552 max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS;
553
554 KASSERT(nsegs <= max_nsegs,
535 ("bio spanned too many segments: %d, max: %d",
555 ("bio spanned too many segments: %d, max: %d",
536 VTBLK_BIO_SEGMENTS(bp),
537 sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS));
556 nsegs, max_nsegs));
538 }
557 }
558#endif
539
540 VTBLK_LOCK(sc);
559
560 VTBLK_LOCK(sc);
541 if ((sc->vtblk_flags & VTBLK_FLAG_DETACHING) == 0) {
561 if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
562 vtblk_finish_bio(bp, ENXIO);
563 else {
542 bioq_disksort(&sc->vtblk_bioq, bp);
564 bioq_disksort(&sc->vtblk_bioq, bp);
543 vtblk_startio(sc);
544 } else
545 vtblk_bio_error(bp, ENXIO);
565
566 if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0)
567 vtblk_startio(sc);
568 }
546 VTBLK_UNLOCK(sc);
547}
548
549static void
550vtblk_negotiate_features(struct vtblk_softc *sc)
551{
552 device_t dev;
553 uint64_t features;

--- 110 unchanged lines hidden (view full) ---

664 device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n",
665 (uintmax_t) dp->d_mediasize >> 20,
666 (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
667 dp->d_sectorsize);
668
669 disk_create(dp, DISK_VERSION);
670}
671
569 VTBLK_UNLOCK(sc);
570}
571
572static void
573vtblk_negotiate_features(struct vtblk_softc *sc)
574{
575 device_t dev;
576 uint64_t features;

--- 110 unchanged lines hidden (view full) ---

687 device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n",
688 (uintmax_t) dp->d_mediasize >> 20,
689 (uintmax_t) dp->d_mediasize / dp->d_sectorsize,
690 dp->d_sectorsize);
691
692 disk_create(dp, DISK_VERSION);
693}
694
695static int
696vtblk_quiesce(struct vtblk_softc *sc)
697{
698 int error;
699
700 error = 0;
701
702 VTBLK_LOCK_ASSERT(sc);
703
704 while (!virtqueue_empty(sc->vtblk_vq)) {
705 if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq",
706 VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) {
707 error = EBUSY;
708 break;
709 }
710 }
711
712 return (error);
713}
714
672static void
673vtblk_startio(struct vtblk_softc *sc)
674{
675 struct virtqueue *vq;
676 struct vtblk_request *req;
677 int enq;
678
679 vq = sc->vtblk_vq;
680 enq = 0;
681
682 VTBLK_LOCK_ASSERT(sc);
683
715static void
716vtblk_startio(struct vtblk_softc *sc)
717{
718 struct virtqueue *vq;
719 struct vtblk_request *req;
720 int enq;
721
722 vq = sc->vtblk_vq;
723 enq = 0;
724
725 VTBLK_LOCK_ASSERT(sc);
726
684 if (sc->vtblk_flags & VTBLK_FLAG_SUSPENDED)
685 return;
686
687 while (!virtqueue_full(vq)) {
688 if ((req = vtblk_dequeue_ready(sc)) == NULL)
689 req = vtblk_bio_request(sc);
690 if (req == NULL)
691 break;
692
693 if (vtblk_execute_request(sc, req) != 0) {
694 vtblk_enqueue_ready(sc, req);

--- 36 unchanged lines hidden (view full) ---

731 req->vbr_hdr.type = VIRTIO_BLK_T_IN;
732 req->vbr_hdr.sector = bp->bio_offset / 512;
733 break;
734 case BIO_WRITE:
735 req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
736 req->vbr_hdr.sector = bp->bio_offset / 512;
737 break;
738 default:
727 while (!virtqueue_full(vq)) {
728 if ((req = vtblk_dequeue_ready(sc)) == NULL)
729 req = vtblk_bio_request(sc);
730 if (req == NULL)
731 break;
732
733 if (vtblk_execute_request(sc, req) != 0) {
734 vtblk_enqueue_ready(sc, req);

--- 36 unchanged lines hidden (view full) ---

771 req->vbr_hdr.type = VIRTIO_BLK_T_IN;
772 req->vbr_hdr.sector = bp->bio_offset / 512;
773 break;
774 case BIO_WRITE:
775 req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
776 req->vbr_hdr.sector = bp->bio_offset / 512;
777 break;
778 default:
739 KASSERT(0, ("bio with unhandled cmd: %d", bp->bio_cmd));
740 req->vbr_hdr.type = -1;
741 break;
779 panic("%s: bio with unhandled cmd: %d", __FUNCTION__,
780 bp->bio_cmd);
742 }
743
744 if (bp->bio_flags & BIO_ORDERED)
745 req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
746
747 return (req);
748}
749
750static int
751vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
752{
753 struct sglist *sg;
754 struct bio *bp;
781 }
782
783 if (bp->bio_flags & BIO_ORDERED)
784 req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
785
786 return (req);
787}
788
789static int
790vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
791{
792 struct sglist *sg;
793 struct bio *bp;
755 int writable, error;
794 int readable, writable, error;
756
757 sg = sc->vtblk_sglist;
758 bp = req->vbr_bp;
759 writable = 0;
760
761 VTBLK_LOCK_ASSERT(sc);
762
763 sglist_reset(sg);

--- 14 unchanged lines hidden (view full) ---

778
779 error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
780 KASSERT(error == 0, ("error adding ack to sglist"));
781 writable++;
782
783 KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
784 ("fewer than min segments: %d", sg->sg_nseg));
785
795
796 sg = sc->vtblk_sglist;
797 bp = req->vbr_bp;
798 writable = 0;
799
800 VTBLK_LOCK_ASSERT(sc);
801
802 sglist_reset(sg);

--- 14 unchanged lines hidden (view full) ---

817
818 error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
819 KASSERT(error == 0, ("error adding ack to sglist"));
820 writable++;
821
822 KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
823 ("fewer than min segments: %d", sg->sg_nseg));
824
786 error = virtqueue_enqueue(sc->vtblk_vq, req, sg,
787 sg->sg_nseg - writable, writable);
825 readable = sg->sg_nseg - writable;
788
826
789 return (error);
827 return (virtqueue_enqueue(sc->vtblk_vq, req, sg, readable, writable));
790}
791
792static int
793vtblk_vq_intr(void *xsc)
794{
795 struct vtblk_softc *sc;
796
797 sc = xsc;
798
799 virtqueue_disable_intr(sc->vtblk_vq);
800 taskqueue_enqueue_fast(sc->vtblk_tq, &sc->vtblk_intr_task);
801
802 return (1);
803}
804
805static void
806vtblk_intr_task(void *arg, int pending)
807{
808 struct vtblk_softc *sc;
828}
829
830static int
831vtblk_vq_intr(void *xsc)
832{
833 struct vtblk_softc *sc;
834
835 sc = xsc;
836
837 virtqueue_disable_intr(sc->vtblk_vq);
838 taskqueue_enqueue_fast(sc->vtblk_tq, &sc->vtblk_intr_task);
839
840 return (1);
841}
842
843static void
844vtblk_intr_task(void *arg, int pending)
845{
846 struct vtblk_softc *sc;
809 struct vtblk_request *req;
810 struct virtqueue *vq;
847 struct virtqueue *vq;
811 struct bio *bp;
812
813 sc = arg;
814 vq = sc->vtblk_vq;
815
816 VTBLK_LOCK(sc);
848
849 sc = arg;
850 vq = sc->vtblk_vq;
851
852 VTBLK_LOCK(sc);
817 if (sc->vtblk_flags & VTBLK_FLAG_DETACHING) {
853 if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
818 VTBLK_UNLOCK(sc);
819 return;
820 }
821
854 VTBLK_UNLOCK(sc);
855 return;
856 }
857
822 while ((req = virtqueue_dequeue(vq, NULL)) != NULL) {
823 bp = req->vbr_bp;
858 vtblk_finish_completed(sc);
824
859
825 if (req->vbr_ack == VIRTIO_BLK_S_OK)
826 bp->bio_resid = 0;
827 else {
828 bp->bio_flags |= BIO_ERROR;
829 if (req->vbr_ack == VIRTIO_BLK_S_UNSUPP)
830 bp->bio_error = ENOTSUP;
831 else
832 bp->bio_error = EIO;
833 }
860 if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0)
861 vtblk_startio(sc);
862 else
863 wakeup(&sc->vtblk_vq);
834
864
835 biodone(bp);
836 vtblk_enqueue_request(sc, req);
837 }
838
839 vtblk_startio(sc);
840
841 if (virtqueue_enable_intr(vq) != 0) {
842 virtqueue_disable_intr(vq);
843 VTBLK_UNLOCK(sc);
844 taskqueue_enqueue_fast(sc->vtblk_tq,
845 &sc->vtblk_intr_task);
846 return;
847 }
848

--- 119 unchanged lines hidden (view full) ---

968 return (vtblk_poll_request(sc, req));
969}
970
971static int
972vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
973{
974 device_t dev;
975 struct virtqueue *vq;
865 if (virtqueue_enable_intr(vq) != 0) {
866 virtqueue_disable_intr(vq);
867 VTBLK_UNLOCK(sc);
868 taskqueue_enqueue_fast(sc->vtblk_tq,
869 &sc->vtblk_intr_task);
870 return;
871 }
872

--- 119 unchanged lines hidden (view full) ---

992 return (vtblk_poll_request(sc, req));
993}
994
995static int
996vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
997{
998 device_t dev;
999 struct virtqueue *vq;
976 struct vtblk_request *r;
977 int error;
978
979 dev = sc->vtblk_dev;
980 vq = sc->vtblk_vq;
981
982 if (!virtqueue_empty(vq))
983 return (EBUSY);
984
985 error = vtblk_execute_request(sc, req);
986 if (error)
987 return (error);
988
989 virtqueue_notify(vq);
990
1000 int error;
1001
1002 dev = sc->vtblk_dev;
1003 vq = sc->vtblk_vq;
1004
1005 if (!virtqueue_empty(vq))
1006 return (EBUSY);
1007
1008 error = vtblk_execute_request(sc, req);
1009 if (error)
1010 return (error);
1011
1012 virtqueue_notify(vq);
1013
991 r = virtqueue_poll(vq, NULL);
992 KASSERT(r == req, ("unexpected request response"));
1014 req = virtqueue_poll(vq, NULL);
993
1015
994 if (req->vbr_ack != VIRTIO_BLK_S_OK) {
995 error = req->vbr_ack == VIRTIO_BLK_S_UNSUPP ? ENOTSUP : EIO;
996 if (bootverbose)
997 device_printf(dev,
998 "vtblk_poll_request: IO error: %d\n", error);
1016 error = vtblk_request_error(req);
1017 if (error && bootverbose) {
1018 device_printf(dev, "vtblk_poll_request: IO error: %d\n",
1019 error);
999 }
1000
1001 return (error);
1002}
1003
1004static void
1020 }
1021
1022 return (error);
1023}
1024
1025static void
1026vtblk_finish_completed(struct vtblk_softc *sc)
1027{
1028 struct vtblk_request *req;
1029 struct bio *bp;
1030 int error;
1031
1032 while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) {
1033 bp = req->vbr_bp;
1034
1035 error = vtblk_request_error(req);
1036 if (error)
1037 disk_err(bp, "hard error", -1, 1);
1038
1039 vtblk_finish_bio(bp, error);
1040 vtblk_enqueue_request(sc, req);
1041 }
1042}
1043
1044static void
1005vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
1006{
1007 struct virtqueue *vq;
1008 struct vtblk_request *req;
1009 int last;
1010
1011 vq = sc->vtblk_vq;
1012 last = 0;
1013
1014 while ((req = virtqueue_drain(vq, &last)) != NULL) {
1015 if (!skip_done)
1045vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
1046{
1047 struct virtqueue *vq;
1048 struct vtblk_request *req;
1049 int last;
1050
1051 vq = sc->vtblk_vq;
1052 last = 0;
1053
1054 while ((req = virtqueue_drain(vq, &last)) != NULL) {
1055 if (!skip_done)
1016 vtblk_bio_error(req->vbr_bp, ENXIO);
1056 vtblk_finish_bio(req->vbr_bp, ENXIO);
1017
1018 vtblk_enqueue_request(sc, req);
1019 }
1020
1021 KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
1022}
1023
1024static void
1025vtblk_drain(struct vtblk_softc *sc)
1026{
1027 struct bio_queue_head *bioq;
1028 struct vtblk_request *req;
1029 struct bio *bp;
1030
1031 bioq = &sc->vtblk_bioq;
1032
1057
1058 vtblk_enqueue_request(sc, req);
1059 }
1060
1061 KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
1062}
1063
1064static void
1065vtblk_drain(struct vtblk_softc *sc)
1066{
1067 struct bio_queue_head *bioq;
1068 struct vtblk_request *req;
1069 struct bio *bp;
1070
1071 bioq = &sc->vtblk_bioq;
1072
1033 if (sc->vtblk_vq != NULL)
1073 if (sc->vtblk_vq != NULL) {
1074 vtblk_finish_completed(sc);
1034 vtblk_drain_vq(sc, 0);
1075 vtblk_drain_vq(sc, 0);
1076 }
1035
1036 while ((req = vtblk_dequeue_ready(sc)) != NULL) {
1077
1078 while ((req = vtblk_dequeue_ready(sc)) != NULL) {
1037 vtblk_bio_error(req->vbr_bp, ENXIO);
1079 vtblk_finish_bio(req->vbr_bp, ENXIO);
1038 vtblk_enqueue_request(sc, req);
1039 }
1040
1041 while (bioq_first(bioq) != NULL) {
1042 bp = bioq_takefirst(bioq);
1080 vtblk_enqueue_request(sc, req);
1081 }
1082
1083 while (bioq_first(bioq) != NULL) {
1084 bp = bioq_takefirst(bioq);
1043 vtblk_bio_error(bp, ENXIO);
1085 vtblk_finish_bio(bp, ENXIO);
1044 }
1045
1046 vtblk_free_requests(sc);
1047}
1048
1049static int
1050vtblk_alloc_requests(struct vtblk_softc *sc)
1051{
1052 struct vtblk_request *req;
1086 }
1087
1088 vtblk_free_requests(sc);
1089}
1090
1091static int
1092vtblk_alloc_requests(struct vtblk_softc *sc)
1093{
1094 struct vtblk_request *req;
1053 int i, size;
1095 int i, nreqs;
1054
1096
1055 size = virtqueue_size(sc->vtblk_vq);
1097 nreqs = virtqueue_size(sc->vtblk_vq);
1056
1057 /*
1058 * Preallocate sufficient requests to keep the virtqueue full. Each
1059 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
1060 * the number allocated when indirect descriptors are not available.
1061 */
1062 if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
1098
1099 /*
1100 * Preallocate sufficient requests to keep the virtqueue full. Each
1101 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
1102 * the number allocated when indirect descriptors are not available.
1103 */
1104 if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
1063 size /= VTBLK_MIN_SEGMENTS;
1105 nreqs /= VTBLK_MIN_SEGMENTS;
1064
1106
1065 for (i = 0; i < size; i++) {
1107 for (i = 0; i < nreqs; i++) {
1066 req = uma_zalloc(vtblk_req_zone, M_NOWAIT);
1067 if (req == NULL)
1068 return (ENOMEM);
1069
1070 sc->vtblk_request_count++;
1071 vtblk_enqueue_request(sc, req);
1072 }
1073
1074 return (0);
1075}
1076
1077static void
1078vtblk_free_requests(struct vtblk_softc *sc)
1079{
1080 struct vtblk_request *req;
1081
1108 req = uma_zalloc(vtblk_req_zone, M_NOWAIT);
1109 if (req == NULL)
1110 return (ENOMEM);
1111
1112 sc->vtblk_request_count++;
1113 vtblk_enqueue_request(sc, req);
1114 }
1115
1116 return (0);
1117}
1118
1119static void
1120vtblk_free_requests(struct vtblk_softc *sc)
1121{
1122 struct vtblk_request *req;
1123
1124 KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready),
1125 ("ready requests left on queue"));
1126
1082 while ((req = vtblk_dequeue_request(sc)) != NULL) {
1083 sc->vtblk_request_count--;
1084 uma_zfree(vtblk_req_zone, req);
1085 }
1086
1087 KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
1088}
1089

--- 31 unchanged lines hidden (view full) ---

1121
1122static void
1123vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
1124{
1125
1126 TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
1127}
1128
1127 while ((req = vtblk_dequeue_request(sc)) != NULL) {
1128 sc->vtblk_request_count--;
1129 uma_zfree(vtblk_req_zone, req);
1130 }
1131
1132 KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
1133}
1134

--- 31 unchanged lines hidden (view full) ---

1166
1167static void
1168vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
1169{
1170
1171 TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
1172}
1173
1174static int
1175vtblk_request_error(struct vtblk_request *req)
1176{
1177 int error;
1178
1179 switch (req->vbr_ack) {
1180 case VIRTIO_BLK_S_OK:
1181 error = 0;
1182 break;
1183 case VIRTIO_BLK_S_UNSUPP:
1184 error = ENOTSUP;
1185 break;
1186 default:
1187 error = EIO;
1188 break;
1189 }
1190
1191 return (error);
1192}
1193
1129static void
1194static void
1130vtblk_bio_error(struct bio *bp, int error)
1195vtblk_finish_bio(struct bio *bp, int error)
1131{
1132
1196{
1197
1133 biofinish(bp, NULL, error);
1198 if (error) {
1199 bp->bio_resid = bp->bio_bcount;
1200 bp->bio_error = error;
1201 bp->bio_flags |= BIO_ERROR;
1202 }
1203
1204 biodone(bp);
1134}
1205}