Deleted Added
sdiff udiff text old ( 234349 ) new ( 238360 )
full compact
virtio_blk.c (234349) virtio_blk.c (238360)
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 13 unchanged lines hidden (view full) ---

22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for VirtIO block devices. */
28
29#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 13 unchanged lines hidden (view full) ---

22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for VirtIO block devices. */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: head/sys/dev/virtio/block/virtio_blk.c 234349 2012-04-16 18:29:12Z grehan $");
30__FBSDID("$FreeBSD: head/sys/dev/virtio/block/virtio_blk.c 238360 2012-07-11 02:57:19Z grehan $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/bio.h>
36#include <sys/malloc.h>
37#include <sys/module.h>
38#include <sys/sglist.h>
39#include <sys/lock.h>
40#include <sys/mutex.h>
41#include <sys/queue.h>
42#include <sys/taskqueue.h>
43
44#include <geom/geom_disk.h>
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/bio.h>
36#include <sys/malloc.h>
37#include <sys/module.h>
38#include <sys/sglist.h>
39#include <sys/lock.h>
40#include <sys/mutex.h>
41#include <sys/queue.h>
42#include <sys/taskqueue.h>
43
44#include <geom/geom_disk.h>
45#include <vm/uma.h>
46
47#include <machine/bus.h>
48#include <machine/resource.h>
49#include <sys/bus.h>
50#include <sys/rman.h>
51
52#include <dev/virtio/virtio.h>
53#include <dev/virtio/virtqueue.h>

--- 60 unchanged lines hidden (view full) ---

114static int vtblk_detach(device_t);
115static int vtblk_suspend(device_t);
116static int vtblk_resume(device_t);
117static int vtblk_shutdown(device_t);
118
119static int vtblk_open(struct disk *);
120static int vtblk_close(struct disk *);
121static int vtblk_ioctl(struct disk *, u_long, void *, int,
45
46#include <machine/bus.h>
47#include <machine/resource.h>
48#include <sys/bus.h>
49#include <sys/rman.h>
50
51#include <dev/virtio/virtio.h>
52#include <dev/virtio/virtqueue.h>

--- 60 unchanged lines hidden (view full) ---

113static int vtblk_detach(device_t);
114static int vtblk_suspend(device_t);
115static int vtblk_resume(device_t);
116static int vtblk_shutdown(device_t);
117
118static int vtblk_open(struct disk *);
119static int vtblk_close(struct disk *);
120static int vtblk_ioctl(struct disk *, u_long, void *, int,
122 struct thread *);
121 struct thread *);
123static int vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
124static void vtblk_strategy(struct bio *);
125
126static void vtblk_negotiate_features(struct vtblk_softc *);
127static int vtblk_maximum_segments(struct vtblk_softc *,
128 struct virtio_blk_config *);
129static int vtblk_alloc_virtqueue(struct vtblk_softc *);
130static void vtblk_alloc_disk(struct vtblk_softc *,

--- 57 unchanged lines hidden (view full) ---

188#define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc)))
189#define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc)))
190#define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc)))
191#define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED)
192#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \
193 mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED)
194
195#define VTBLK_DISK_NAME "vtbd"
122static int vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
123static void vtblk_strategy(struct bio *);
124
125static void vtblk_negotiate_features(struct vtblk_softc *);
126static int vtblk_maximum_segments(struct vtblk_softc *,
127 struct virtio_blk_config *);
128static int vtblk_alloc_virtqueue(struct vtblk_softc *);
129static void vtblk_alloc_disk(struct vtblk_softc *,

--- 57 unchanged lines hidden (view full) ---

187#define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc)))
188#define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc)))
189#define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc)))
190#define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED)
191#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \
192 mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED)
193
194#define VTBLK_DISK_NAME "vtbd"
196#define VTBLK_QUIESCE_TIMEOUT (30 * hz)
195#define VTBLK_QUIESCE_TIMEOUT (30 * hz)
197
198/*
199 * Each block request uses at least two segments - one for the header
200 * and one for the status.
201 */
202#define VTBLK_MIN_SEGMENTS 2
203
196
197/*
198 * Each block request uses at least two segments - one for the header
199 * and one for the status.
200 */
201#define VTBLK_MIN_SEGMENTS 2
202
204static uma_zone_t vtblk_req_zone;
205
206static device_method_t vtblk_methods[] = {
207 /* Device methods. */
208 DEVMETHOD(device_probe, vtblk_probe),
209 DEVMETHOD(device_attach, vtblk_attach),
210 DEVMETHOD(device_detach, vtblk_detach),
211 DEVMETHOD(device_suspend, vtblk_suspend),
212 DEVMETHOD(device_resume, vtblk_resume),
213 DEVMETHOD(device_shutdown, vtblk_shutdown),

--- 17 unchanged lines hidden (view full) ---

231vtblk_modevent(module_t mod, int type, void *unused)
232{
233 int error;
234
235 error = 0;
236
237 switch (type) {
238 case MOD_LOAD:
203static device_method_t vtblk_methods[] = {
204 /* Device methods. */
205 DEVMETHOD(device_probe, vtblk_probe),
206 DEVMETHOD(device_attach, vtblk_attach),
207 DEVMETHOD(device_detach, vtblk_detach),
208 DEVMETHOD(device_suspend, vtblk_suspend),
209 DEVMETHOD(device_resume, vtblk_resume),
210 DEVMETHOD(device_shutdown, vtblk_shutdown),

--- 17 unchanged lines hidden (view full) ---

228vtblk_modevent(module_t mod, int type, void *unused)
229{
230 int error;
231
232 error = 0;
233
234 switch (type) {
235 case MOD_LOAD:
239 vtblk_req_zone = uma_zcreate("vtblk_request",
240 sizeof(struct vtblk_request),
241 NULL, NULL, NULL, NULL, 0, 0);
242 break;
243 case MOD_QUIESCE:
244 case MOD_UNLOAD:
236 case MOD_QUIESCE:
237 case MOD_UNLOAD:
245 if (uma_zone_get_cur(vtblk_req_zone) > 0)
246 error = EBUSY;
247 else if (type == MOD_UNLOAD) {
248 uma_zdestroy(vtblk_req_zone);
249 vtblk_req_zone = NULL;
250 }
251 break;
252 case MOD_SHUTDOWN:
253 break;
254 default:
255 error = EOPNOTSUPP;
256 break;
257 }
258
259 return (error);

--- 51 unchanged lines hidden (view full) ---

311 error = ENOTSUP;
312 device_printf(dev, "host requires unsupported "
313 "maximum segment size feature\n");
314 goto fail;
315 }
316 }
317
318 sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
238 case MOD_SHUTDOWN:
239 break;
240 default:
241 error = EOPNOTSUPP;
242 break;
243 }
244
245 return (error);

--- 51 unchanged lines hidden (view full) ---

297 error = ENOTSUP;
298 device_printf(dev, "host requires unsupported "
299 "maximum segment size feature\n");
300 goto fail;
301 }
302 }
303
304 sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
319 if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
305 if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
320 error = EINVAL;
321 device_printf(dev, "fewer than minimum number of segments "
322 "allowed: %d\n", sc->vtblk_max_nsegs);
323 goto fail;
324 }
325
326 sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
327 if (sc->vtblk_sglist == NULL) {

--- 160 unchanged lines hidden (view full) ---

488vtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset,
489 size_t length)
490{
491 struct disk *dp;
492 struct vtblk_softc *sc;
493 int error;
494
495 dp = arg;
306 error = EINVAL;
307 device_printf(dev, "fewer than minimum number of segments "
308 "allowed: %d\n", sc->vtblk_max_nsegs);
309 goto fail;
310 }
311
312 sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
313 if (sc->vtblk_sglist == NULL) {

--- 160 unchanged lines hidden (view full) ---

474vtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset,
475 size_t length)
476{
477 struct disk *dp;
478 struct vtblk_softc *sc;
479 int error;
480
481 dp = arg;
496 error = 0;
497
498 if ((sc = dp->d_drv1) == NULL)
499 return (ENXIO);
500
501 VTBLK_LOCK(sc);
502
503 if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
504 vtblk_prepare_dump(sc);

--- 29 unchanged lines hidden (view full) ---

534 * be a better way to report our readonly'ness to GEOM above.
535 */
536 if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
537 (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
538 vtblk_finish_bio(bp, EROFS);
539 return;
540 }
541
482
483 if ((sc = dp->d_drv1) == NULL)
484 return (ENXIO);
485
486 VTBLK_LOCK(sc);
487
488 if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
489 vtblk_prepare_dump(sc);

--- 29 unchanged lines hidden (view full) ---

519 * be a better way to report our readonly'ness to GEOM above.
520 */
521 if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
522 (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
523 vtblk_finish_bio(bp, EROFS);
524 return;
525 }
526
542#ifdef INVARIANTS
527#ifdef INVARIANTS
543 /*
544 * Prevent read/write buffers spanning too many segments from
545 * getting into the queue. This should only trip if d_maxsize
546 * was incorrectly set.
547 */
548 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
549 int nsegs, max_nsegs;
528 /*
529 * Prevent read/write buffers spanning too many segments from
530 * getting into the queue. This should only trip if d_maxsize
531 * was incorrectly set.
532 */
533 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
534 int nsegs, max_nsegs;
550
535
551 nsegs = sglist_count(bp->bio_data, bp->bio_bcount);
552 max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS;
553
554 KASSERT(nsegs <= max_nsegs,
536 nsegs = sglist_count(bp->bio_data, bp->bio_bcount);
537 max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS;
538
539 KASSERT(nsegs <= max_nsegs,
555 ("bio spanned too many segments: %d, max: %d",
556 nsegs, max_nsegs));
540 ("bio %p spanned too many segments: %d, max: %d",
541 bp, nsegs, max_nsegs));
557 }
558#endif
559
560 VTBLK_LOCK(sc);
561 if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
562 vtblk_finish_bio(bp, ENXIO);
563 else {
564 bioq_disksort(&sc->vtblk_bioq, bp);

--- 230 unchanged lines hidden (view full) ---

795
796 sg = sc->vtblk_sglist;
797 bp = req->vbr_bp;
798 writable = 0;
799
800 VTBLK_LOCK_ASSERT(sc);
801
802 sglist_reset(sg);
542 }
543#endif
544
545 VTBLK_LOCK(sc);
546 if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
547 vtblk_finish_bio(bp, ENXIO);
548 else {
549 bioq_disksort(&sc->vtblk_bioq, bp);

--- 230 unchanged lines hidden (view full) ---

780
781 sg = sc->vtblk_sglist;
782 bp = req->vbr_bp;
783 writable = 0;
784
785 VTBLK_LOCK_ASSERT(sc);
786
787 sglist_reset(sg);
803 error = sglist_append(sg, &req->vbr_hdr,
804 sizeof(struct virtio_blk_outhdr));
805 KASSERT(error == 0, ("error adding header to sglist"));
806 KASSERT(sg->sg_nseg == 1,
807 ("header spanned multiple segments: %d", sg->sg_nseg));
808
788
789 sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr));
790
809 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
810 error = sglist_append(sg, bp->bio_data, bp->bio_bcount);
791 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
792 error = sglist_append(sg, bp->bio_data, bp->bio_bcount);
811 KASSERT(error == 0, ("error adding buffer to sglist"));
793 if (error || sg->sg_nseg == sg->sg_maxseg)
794 panic("%s: data buffer too big bio:%p error:%d",
795 __FUNCTION__, bp, error);
812
813 /* BIO_READ means the host writes into our buffer. */
814 if (bp->bio_cmd == BIO_READ)
796
797 /* BIO_READ means the host writes into our buffer. */
798 if (bp->bio_cmd == BIO_READ)
815 writable += sg->sg_nseg - 1;
799 writable = sg->sg_nseg - 1;
816 }
817
800 }
801
818 error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
819 KASSERT(error == 0, ("error adding ack to sglist"));
820 writable++;
802 writable++;
803 sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
821
804
822 KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
823 ("fewer than min segments: %d", sg->sg_nseg));
824
825 readable = sg->sg_nseg - writable;
826
827 return (virtqueue_enqueue(sc->vtblk_vq, req, sg, readable, writable));
828}
829
830static int
831vtblk_vq_intr(void *xsc)
832{

--- 157 unchanged lines hidden (view full) ---

990 buf.bio_cmd = BIO_FLUSH;
991
992 return (vtblk_poll_request(sc, req));
993}
994
995static int
996vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
997{
805 readable = sg->sg_nseg - writable;
806
807 return (virtqueue_enqueue(sc->vtblk_vq, req, sg, readable, writable));
808}
809
810static int
811vtblk_vq_intr(void *xsc)
812{

--- 157 unchanged lines hidden (view full) ---

970 buf.bio_cmd = BIO_FLUSH;
971
972 return (vtblk_poll_request(sc, req));
973}
974
975static int
976vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
977{
998 device_t dev;
999 struct virtqueue *vq;
1000 struct vtblk_request *r;
1001 int error;
1002
978 struct virtqueue *vq;
979 struct vtblk_request *r;
980 int error;
981
1003 dev = sc->vtblk_dev;
1004 vq = sc->vtblk_vq;
1005
1006 if (!virtqueue_empty(vq))
1007 return (EBUSY);
1008
1009 error = vtblk_execute_request(sc, req);
1010 if (error)
1011 return (error);
1012
1013 virtqueue_notify(vq);
1014
1015 r = virtqueue_poll(vq, NULL);
982 vq = sc->vtblk_vq;
983
984 if (!virtqueue_empty(vq))
985 return (EBUSY);
986
987 error = vtblk_execute_request(sc, req);
988 if (error)
989 return (error);
990
991 virtqueue_notify(vq);
992
993 r = virtqueue_poll(vq, NULL);
1016 KASSERT(r == req, ("unexpected request response"));
994 KASSERT(r == req, ("unexpected request response: %p/%p", r, req));
1017
1018 error = vtblk_request_error(req);
1019 if (error && bootverbose) {
995
996 error = vtblk_request_error(req);
997 if (error && bootverbose) {
1020 device_printf(dev, "vtblk_poll_request: IO error: %d\n",
1021 error);
998 device_printf(sc->vtblk_dev,
999 "%s: IO error: %d\n", __FUNCTION__, error);
1022 }
1023
1024 return (error);
1025}
1026
1027static void
1028vtblk_finish_completed(struct vtblk_softc *sc)
1029{

--- 55 unchanged lines hidden (view full) ---

1085 while (bioq_first(bioq) != NULL) {
1086 bp = bioq_takefirst(bioq);
1087 vtblk_finish_bio(bp, ENXIO);
1088 }
1089
1090 vtblk_free_requests(sc);
1091}
1092
1000 }
1001
1002 return (error);
1003}
1004
1005static void
1006vtblk_finish_completed(struct vtblk_softc *sc)
1007{

--- 55 unchanged lines hidden (view full) ---

1063 while (bioq_first(bioq) != NULL) {
1064 bp = bioq_takefirst(bioq);
1065 vtblk_finish_bio(bp, ENXIO);
1066 }
1067
1068 vtblk_free_requests(sc);
1069}
1070
1071#ifdef INVARIANTS
1072static void
1073vtblk_request_invariants(struct vtblk_request *req)
1074{
1075 int hdr_nsegs, ack_nsegs;
1076
1077 hdr_nsegs = sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr));
1078 ack_nsegs = sglist_count(&req->vbr_ack, sizeof(req->vbr_ack));
1079
1080 KASSERT(hdr_nsegs == 1, ("request header crossed page boundary"));
1081 KASSERT(ack_nsegs == 1, ("request ack crossed page boundary"));
1082}
1083#endif
1084
1093static int
1094vtblk_alloc_requests(struct vtblk_softc *sc)
1095{
1096 struct vtblk_request *req;
1097 int i, nreqs;
1098
1099 nreqs = virtqueue_size(sc->vtblk_vq);
1100
1101 /*
1102 * Preallocate sufficient requests to keep the virtqueue full. Each
1103 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
1104 * the number allocated when indirect descriptors are not available.
1105 */
1106 if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
1107 nreqs /= VTBLK_MIN_SEGMENTS;
1108
1109 for (i = 0; i < nreqs; i++) {
1085static int
1086vtblk_alloc_requests(struct vtblk_softc *sc)
1087{
1088 struct vtblk_request *req;
1089 int i, nreqs;
1090
1091 nreqs = virtqueue_size(sc->vtblk_vq);
1092
1093 /*
1094 * Preallocate sufficient requests to keep the virtqueue full. Each
1095 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
1096 * the number allocated when indirect descriptors are not available.
1097 */
1098 if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
1099 nreqs /= VTBLK_MIN_SEGMENTS;
1100
1101 for (i = 0; i < nreqs; i++) {
1110 req = uma_zalloc(vtblk_req_zone, M_NOWAIT);
1102 req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT);
1111 if (req == NULL)
1112 return (ENOMEM);
1113
1103 if (req == NULL)
1104 return (ENOMEM);
1105
1106#ifdef INVARIANTS
1107 vtblk_request_invariants(req);
1108#endif
1109
1114 sc->vtblk_request_count++;
1115 vtblk_enqueue_request(sc, req);
1116 }
1117
1118 return (0);
1119}
1120
1121static void
1122vtblk_free_requests(struct vtblk_softc *sc)
1123{
1124 struct vtblk_request *req;
1125
1126 KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready),
1127 ("ready requests left on queue"));
1128
1129 while ((req = vtblk_dequeue_request(sc)) != NULL) {
1130 sc->vtblk_request_count--;
1110 sc->vtblk_request_count++;
1111 vtblk_enqueue_request(sc, req);
1112 }
1113
1114 return (0);
1115}
1116
1117static void
1118vtblk_free_requests(struct vtblk_softc *sc)
1119{
1120 struct vtblk_request *req;
1121
1122 KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready),
1123 ("ready requests left on queue"));
1124
1125 while ((req = vtblk_dequeue_request(sc)) != NULL) {
1126 sc->vtblk_request_count--;
1131 uma_zfree(vtblk_req_zone, req);
1127 free(req, M_DEVBUF);
1132 }
1133
1128 }
1129
1134 KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
1130 KASSERT(sc->vtblk_request_count == 0,
1131 ("leaked requests: %d", sc->vtblk_request_count));
1135}
1136
1137static struct vtblk_request *
1138vtblk_dequeue_request(struct vtblk_softc *sc)
1139{
1140 struct vtblk_request *req;
1141
1142 req = TAILQ_FIRST(&sc->vtblk_req_free);

--- 65 unchanged lines hidden --- 		1132}
1133
1134static struct vtblk_request *
1135vtblk_dequeue_request(struct vtblk_softc *sc)
1136{
1137 struct vtblk_request *req;
1138
1139 req = TAILQ_FIRST(&sc->vtblk_req_free);

--- 65 unchanged lines hidden ---