virtio_blk.c revision 247829
1227652Sgrehan/*- 2227652Sgrehan * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3227652Sgrehan * All rights reserved. 4227652Sgrehan * 5227652Sgrehan * Redistribution and use in source and binary forms, with or without 6227652Sgrehan * modification, are permitted provided that the following conditions 7227652Sgrehan * are met: 8227652Sgrehan * 1. Redistributions of source code must retain the above copyright 9227652Sgrehan * notice unmodified, this list of conditions, and the following 10227652Sgrehan * disclaimer. 11227652Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 12227652Sgrehan * notice, this list of conditions and the following disclaimer in the 13227652Sgrehan * documentation and/or other materials provided with the distribution. 14227652Sgrehan * 15227652Sgrehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16227652Sgrehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17227652Sgrehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18227652Sgrehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19227652Sgrehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20227652Sgrehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21227652Sgrehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22227652Sgrehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23227652Sgrehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24227652Sgrehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25227652Sgrehan */ 26227652Sgrehan 27227652Sgrehan/* Driver for VirtIO block devices. */ 28227652Sgrehan 29227652Sgrehan#include <sys/cdefs.h> 30227652Sgrehan__FBSDID("$FreeBSD: head/sys/dev/virtio/block/virtio_blk.c 247829 2013-03-05 07:00:05Z bryanv $"); 31227652Sgrehan 32227652Sgrehan#include <sys/param.h> 33227652Sgrehan#include <sys/systm.h> 34227652Sgrehan#include <sys/kernel.h> 35227652Sgrehan#include <sys/bio.h> 36227652Sgrehan#include <sys/malloc.h> 37227652Sgrehan#include <sys/module.h> 38227652Sgrehan#include <sys/sglist.h> 39227652Sgrehan#include <sys/lock.h> 40227652Sgrehan#include <sys/mutex.h> 41227652Sgrehan#include <sys/queue.h> 42227652Sgrehan#include <sys/taskqueue.h> 43227652Sgrehan 44227652Sgrehan#include <geom/geom_disk.h> 45227652Sgrehan 46227652Sgrehan#include <machine/bus.h> 47227652Sgrehan#include <machine/resource.h> 48227652Sgrehan#include <sys/bus.h> 49227652Sgrehan#include <sys/rman.h> 50227652Sgrehan 51227652Sgrehan#include <dev/virtio/virtio.h> 52227652Sgrehan#include <dev/virtio/virtqueue.h> 53227652Sgrehan#include <dev/virtio/block/virtio_blk.h> 54227652Sgrehan 55227652Sgrehan#include "virtio_if.h" 56227652Sgrehan 57227652Sgrehanstruct vtblk_request { 58227652Sgrehan struct virtio_blk_outhdr vbr_hdr; 59227652Sgrehan struct bio *vbr_bp; 60227652Sgrehan uint8_t vbr_ack; 61227652Sgrehan 62227652Sgrehan TAILQ_ENTRY(vtblk_request) vbr_link; 63227652Sgrehan}; 64227652Sgrehan 65227652Sgrehanstruct vtblk_softc { 66227652Sgrehan device_t vtblk_dev; 67227652Sgrehan struct mtx vtblk_mtx; 68227652Sgrehan uint64_t vtblk_features; 69227652Sgrehan uint32_t vtblk_flags; 70227652Sgrehan#define VTBLK_FLAG_INDIRECT 0x0001 71227652Sgrehan#define VTBLK_FLAG_READONLY 0x0002 72234270Sgrehan#define VTBLK_FLAG_DETACH 0x0004 73234270Sgrehan#define VTBLK_FLAG_SUSPEND 0x0008 74227652Sgrehan#define VTBLK_FLAG_DUMPING 0x0010 75247829Sbryanv#define VTBLK_FLAG_BARRIER 0x0020 76227652Sgrehan 77227652Sgrehan struct virtqueue *vtblk_vq; 78227652Sgrehan struct sglist *vtblk_sglist; 79227652Sgrehan struct disk *vtblk_disk; 80227652Sgrehan 81227652Sgrehan struct bio_queue_head vtblk_bioq; 82227652Sgrehan TAILQ_HEAD(, vtblk_request) 83227652Sgrehan vtblk_req_free; 84227652Sgrehan TAILQ_HEAD(, vtblk_request) 85247829Sbryanv vtblk_req_ready; 86247829Sbryanv struct vtblk_request *vtblk_req_ordered; 87227652Sgrehan 88227652Sgrehan struct taskqueue *vtblk_tq; 89227652Sgrehan struct task vtblk_intr_task; 90227652Sgrehan 91227652Sgrehan int vtblk_max_nsegs; 92227652Sgrehan int vtblk_request_count; 93227652Sgrehan 94227652Sgrehan struct vtblk_request vtblk_dump_request; 95227652Sgrehan}; 96227652Sgrehan 97227652Sgrehanstatic struct virtio_feature_desc vtblk_feature_desc[] = { 98227652Sgrehan { VIRTIO_BLK_F_BARRIER, "HostBarrier" }, 99227652Sgrehan { VIRTIO_BLK_F_SIZE_MAX, "MaxSegSize" }, 100227652Sgrehan { VIRTIO_BLK_F_SEG_MAX, "MaxNumSegs" }, 101227652Sgrehan { VIRTIO_BLK_F_GEOMETRY, "DiskGeometry" }, 102227652Sgrehan { VIRTIO_BLK_F_RO, "ReadOnly" }, 103227652Sgrehan { VIRTIO_BLK_F_BLK_SIZE, "BlockSize" }, 104227652Sgrehan { VIRTIO_BLK_F_SCSI, "SCSICmds" }, 105227652Sgrehan { VIRTIO_BLK_F_FLUSH, "FlushCmd" }, 106227652Sgrehan { VIRTIO_BLK_F_TOPOLOGY, "Topology" }, 107227652Sgrehan 108227652Sgrehan { 0, NULL } 109227652Sgrehan}; 110227652Sgrehan 111227652Sgrehanstatic int vtblk_modevent(module_t, int, void *); 112227652Sgrehan 113227652Sgrehanstatic int vtblk_probe(device_t); 114227652Sgrehanstatic int vtblk_attach(device_t); 115227652Sgrehanstatic int vtblk_detach(device_t); 116227652Sgrehanstatic int vtblk_suspend(device_t); 117227652Sgrehanstatic int vtblk_resume(device_t); 118227652Sgrehanstatic int vtblk_shutdown(device_t); 119227652Sgrehan 120234270Sgrehanstatic int vtblk_open(struct disk *); 121234270Sgrehanstatic int vtblk_close(struct disk *); 122234270Sgrehanstatic int vtblk_ioctl(struct disk *, u_long, void *, int, 123238360Sgrehan struct thread *); 124234270Sgrehanstatic int vtblk_dump(void *, void *, vm_offset_t, off_t, size_t); 125234270Sgrehanstatic void vtblk_strategy(struct bio *); 126234270Sgrehan 127227652Sgrehanstatic void vtblk_negotiate_features(struct vtblk_softc *); 128227652Sgrehanstatic int vtblk_maximum_segments(struct vtblk_softc *, 129227652Sgrehan struct virtio_blk_config *); 130227652Sgrehanstatic int vtblk_alloc_virtqueue(struct vtblk_softc *); 131227652Sgrehanstatic void vtblk_alloc_disk(struct vtblk_softc *, 132227652Sgrehan struct virtio_blk_config *); 133227652Sgrehanstatic void vtblk_create_disk(struct vtblk_softc *); 134227652Sgrehan 135234270Sgrehanstatic int vtblk_quiesce(struct vtblk_softc *); 136227652Sgrehanstatic void vtblk_startio(struct vtblk_softc *); 137227652Sgrehanstatic struct vtblk_request * vtblk_bio_request(struct vtblk_softc *); 138227652Sgrehanstatic int vtblk_execute_request(struct vtblk_softc *, 139227652Sgrehan struct vtblk_request *); 140227652Sgrehan 141227652Sgrehanstatic int vtblk_vq_intr(void *); 142227652Sgrehanstatic void vtblk_intr_task(void *, int); 143227652Sgrehan 144227652Sgrehanstatic void vtblk_stop(struct vtblk_softc *); 145227652Sgrehan 146227652Sgrehanstatic void vtblk_get_ident(struct vtblk_softc *); 147227652Sgrehanstatic void vtblk_prepare_dump(struct vtblk_softc *); 148227652Sgrehanstatic int vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t); 149227652Sgrehanstatic int vtblk_flush_dump(struct vtblk_softc *); 150227652Sgrehanstatic int vtblk_poll_request(struct vtblk_softc *, 151227652Sgrehan struct vtblk_request *); 152227652Sgrehan 153234270Sgrehanstatic void vtblk_finish_completed(struct vtblk_softc *); 154227652Sgrehanstatic void vtblk_drain_vq(struct vtblk_softc *, int); 155227652Sgrehanstatic void vtblk_drain(struct vtblk_softc *); 156227652Sgrehan 157227652Sgrehanstatic int vtblk_alloc_requests(struct vtblk_softc *); 158227652Sgrehanstatic void vtblk_free_requests(struct vtblk_softc *); 159227652Sgrehanstatic struct vtblk_request * vtblk_dequeue_request(struct vtblk_softc *); 160227652Sgrehanstatic void vtblk_enqueue_request(struct vtblk_softc *, 161227652Sgrehan struct vtblk_request *); 162227652Sgrehan 163227652Sgrehanstatic struct vtblk_request * vtblk_dequeue_ready(struct vtblk_softc *); 164227652Sgrehanstatic void vtblk_enqueue_ready(struct vtblk_softc *, 165227652Sgrehan struct vtblk_request *); 166227652Sgrehan 167234270Sgrehanstatic int vtblk_request_error(struct vtblk_request *); 168234270Sgrehanstatic void vtblk_finish_bio(struct bio *, int); 169227652Sgrehan 170227652Sgrehan/* Tunables. */ 171227652Sgrehanstatic int vtblk_no_ident = 0; 172227652SgrehanTUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident); 173227652Sgrehan 174227652Sgrehan/* Features desired/implemented by this driver. */ 175227652Sgrehan#define VTBLK_FEATURES \ 176227652Sgrehan (VIRTIO_BLK_F_BARRIER | \ 177227652Sgrehan VIRTIO_BLK_F_SIZE_MAX | \ 178227652Sgrehan VIRTIO_BLK_F_SEG_MAX | \ 179227652Sgrehan VIRTIO_BLK_F_GEOMETRY | \ 180227652Sgrehan VIRTIO_BLK_F_RO | \ 181227652Sgrehan VIRTIO_BLK_F_BLK_SIZE | \ 182227652Sgrehan VIRTIO_BLK_F_FLUSH | \ 183227652Sgrehan VIRTIO_RING_F_INDIRECT_DESC) 184227652Sgrehan 185227652Sgrehan#define VTBLK_MTX(_sc) &(_sc)->vtblk_mtx 186227652Sgrehan#define VTBLK_LOCK_INIT(_sc, _name) \ 187227652Sgrehan mtx_init(VTBLK_MTX((_sc)), (_name), \ 188227652Sgrehan "VTBLK Lock", MTX_DEF) 189227652Sgrehan#define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc))) 190227652Sgrehan#define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc))) 191227652Sgrehan#define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc))) 192227652Sgrehan#define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED) 193227652Sgrehan#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \ 194227652Sgrehan mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED) 195227652Sgrehan 196227652Sgrehan#define VTBLK_DISK_NAME "vtbd" 197238360Sgrehan#define VTBLK_QUIESCE_TIMEOUT (30 * hz) 198227652Sgrehan 199227652Sgrehan/* 200227652Sgrehan * Each block request uses at least two segments - one for the header 201227652Sgrehan * and one for the status. 202227652Sgrehan */ 203227652Sgrehan#define VTBLK_MIN_SEGMENTS 2 204227652Sgrehan 205227652Sgrehanstatic device_method_t vtblk_methods[] = { 206227652Sgrehan /* Device methods. */ 207227652Sgrehan DEVMETHOD(device_probe, vtblk_probe), 208227652Sgrehan DEVMETHOD(device_attach, vtblk_attach), 209227652Sgrehan DEVMETHOD(device_detach, vtblk_detach), 210227652Sgrehan DEVMETHOD(device_suspend, vtblk_suspend), 211227652Sgrehan DEVMETHOD(device_resume, vtblk_resume), 212227652Sgrehan DEVMETHOD(device_shutdown, vtblk_shutdown), 213227652Sgrehan 214234270Sgrehan DEVMETHOD_END 215227652Sgrehan}; 216227652Sgrehan 217227652Sgrehanstatic driver_t vtblk_driver = { 218227652Sgrehan "vtblk", 219227652Sgrehan vtblk_methods, 220227652Sgrehan sizeof(struct vtblk_softc) 221227652Sgrehan}; 222227652Sgrehanstatic devclass_t vtblk_devclass; 223227652Sgrehan 224227652SgrehanDRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass, 225227652Sgrehan vtblk_modevent, 0); 226227652SgrehanMODULE_VERSION(virtio_blk, 1); 227227652SgrehanMODULE_DEPEND(virtio_blk, virtio, 1, 1, 1); 228227652Sgrehan 229227652Sgrehanstatic int 230227652Sgrehanvtblk_modevent(module_t mod, int type, void *unused) 231227652Sgrehan{ 232227652Sgrehan int error; 233227652Sgrehan 234227652Sgrehan error = 0; 235227652Sgrehan 236227652Sgrehan switch (type) { 237227652Sgrehan case MOD_LOAD: 238227652Sgrehan case MOD_QUIESCE: 239227652Sgrehan case MOD_UNLOAD: 240227652Sgrehan case MOD_SHUTDOWN: 241227652Sgrehan break; 242227652Sgrehan default: 243227652Sgrehan error = EOPNOTSUPP; 244227652Sgrehan break; 245227652Sgrehan } 246227652Sgrehan 247227652Sgrehan return (error); 248227652Sgrehan} 249227652Sgrehan 250227652Sgrehanstatic int 251227652Sgrehanvtblk_probe(device_t dev) 252227652Sgrehan{ 253227652Sgrehan 254227652Sgrehan if (virtio_get_device_type(dev) != VIRTIO_ID_BLOCK) 255227652Sgrehan return (ENXIO); 256227652Sgrehan 257227652Sgrehan device_set_desc(dev, "VirtIO Block Adapter"); 258227652Sgrehan 259227652Sgrehan return (BUS_PROBE_DEFAULT); 260227652Sgrehan} 261227652Sgrehan 262227652Sgrehanstatic int 263227652Sgrehanvtblk_attach(device_t dev) 264227652Sgrehan{ 265227652Sgrehan struct vtblk_softc *sc; 266227652Sgrehan struct virtio_blk_config blkcfg; 267227652Sgrehan int error; 268227652Sgrehan 269227652Sgrehan sc = device_get_softc(dev); 270227652Sgrehan sc->vtblk_dev = dev; 271227652Sgrehan 272227652Sgrehan VTBLK_LOCK_INIT(sc, device_get_nameunit(dev)); 273227652Sgrehan 274227652Sgrehan bioq_init(&sc->vtblk_bioq); 275227652Sgrehan TAILQ_INIT(&sc->vtblk_req_free); 276227652Sgrehan TAILQ_INIT(&sc->vtblk_req_ready); 277227652Sgrehan 278227652Sgrehan virtio_set_feature_desc(dev, vtblk_feature_desc); 279227652Sgrehan vtblk_negotiate_features(sc); 280227652Sgrehan 281227652Sgrehan if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 282227652Sgrehan sc->vtblk_flags |= VTBLK_FLAG_INDIRECT; 283227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) 284227652Sgrehan sc->vtblk_flags |= VTBLK_FLAG_READONLY; 285247829Sbryanv if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER)) 286247829Sbryanv sc->vtblk_flags |= VTBLK_FLAG_BARRIER; 287227652Sgrehan 288227652Sgrehan /* Get local copy of config. */ 289228301Sgrehan virtio_read_device_config(dev, 0, &blkcfg, 290228301Sgrehan sizeof(struct virtio_blk_config)); 291227652Sgrehan 292227652Sgrehan /* 293227652Sgrehan * With the current sglist(9) implementation, it is not easy 294227652Sgrehan * for us to support a maximum segment size as adjacent 295227652Sgrehan * segments are coalesced. For now, just make sure it's larger 296227652Sgrehan * than the maximum supported transfer size. 297227652Sgrehan */ 298227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) { 299227652Sgrehan if (blkcfg.size_max < MAXPHYS) { 300227652Sgrehan error = ENOTSUP; 301227652Sgrehan device_printf(dev, "host requires unsupported " 302227652Sgrehan "maximum segment size feature\n"); 303227652Sgrehan goto fail; 304227652Sgrehan } 305227652Sgrehan } 306227652Sgrehan 307227652Sgrehan sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg); 308238360Sgrehan if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) { 309234270Sgrehan error = EINVAL; 310234270Sgrehan device_printf(dev, "fewer than minimum number of segments " 311234270Sgrehan "allowed: %d\n", sc->vtblk_max_nsegs); 312234270Sgrehan goto fail; 313234270Sgrehan } 314227652Sgrehan 315227652Sgrehan sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT); 316227652Sgrehan if (sc->vtblk_sglist == NULL) { 317227652Sgrehan error = ENOMEM; 318227652Sgrehan device_printf(dev, "cannot allocate sglist\n"); 319227652Sgrehan goto fail; 320227652Sgrehan } 321227652Sgrehan 322227652Sgrehan error = vtblk_alloc_virtqueue(sc); 323227652Sgrehan if (error) { 324227652Sgrehan device_printf(dev, "cannot allocate virtqueue\n"); 325227652Sgrehan goto fail; 326227652Sgrehan } 327227652Sgrehan 328227652Sgrehan error = vtblk_alloc_requests(sc); 329227652Sgrehan if (error) { 330227652Sgrehan device_printf(dev, "cannot preallocate requests\n"); 331227652Sgrehan goto fail; 332227652Sgrehan } 333227652Sgrehan 334227652Sgrehan vtblk_alloc_disk(sc, &blkcfg); 335227652Sgrehan 336227652Sgrehan TASK_INIT(&sc->vtblk_intr_task, 0, vtblk_intr_task, sc); 337227652Sgrehan sc->vtblk_tq = taskqueue_create_fast("vtblk_taskq", M_NOWAIT, 338227652Sgrehan taskqueue_thread_enqueue, &sc->vtblk_tq); 339227652Sgrehan if (sc->vtblk_tq == NULL) { 340227652Sgrehan error = ENOMEM; 341227652Sgrehan device_printf(dev, "cannot allocate taskqueue\n"); 342227652Sgrehan goto fail; 343227652Sgrehan } 344227652Sgrehan 345227652Sgrehan error = virtio_setup_intr(dev, INTR_TYPE_BIO | INTR_ENTROPY); 346227652Sgrehan if (error) { 347227652Sgrehan device_printf(dev, "cannot setup virtqueue interrupt\n"); 348227652Sgrehan goto fail; 349227652Sgrehan } 350227652Sgrehan 351244200Sbryanv taskqueue_start_threads(&sc->vtblk_tq, 1, PI_DISK, "%s taskq", 352244200Sbryanv device_get_nameunit(dev)); 353244200Sbryanv 354227652Sgrehan vtblk_create_disk(sc); 355227652Sgrehan 356227652Sgrehan virtqueue_enable_intr(sc->vtblk_vq); 357227652Sgrehan 358227652Sgrehanfail: 359227652Sgrehan if (error) 360227652Sgrehan vtblk_detach(dev); 361227652Sgrehan 362227652Sgrehan return (error); 363227652Sgrehan} 364227652Sgrehan 365227652Sgrehanstatic int 366227652Sgrehanvtblk_detach(device_t dev) 367227652Sgrehan{ 368227652Sgrehan struct vtblk_softc *sc; 369227652Sgrehan 370227652Sgrehan sc = device_get_softc(dev); 371227652Sgrehan 372227652Sgrehan VTBLK_LOCK(sc); 373234270Sgrehan sc->vtblk_flags |= VTBLK_FLAG_DETACH; 374227652Sgrehan if (device_is_attached(dev)) 375227652Sgrehan vtblk_stop(sc); 376227652Sgrehan VTBLK_UNLOCK(sc); 377227652Sgrehan 378227652Sgrehan if (sc->vtblk_tq != NULL) { 379227652Sgrehan taskqueue_drain(sc->vtblk_tq, &sc->vtblk_intr_task); 380227652Sgrehan taskqueue_free(sc->vtblk_tq); 381227652Sgrehan sc->vtblk_tq = NULL; 382227652Sgrehan } 383227652Sgrehan 384227652Sgrehan vtblk_drain(sc); 385227652Sgrehan 386227652Sgrehan if (sc->vtblk_disk != NULL) { 387227652Sgrehan disk_destroy(sc->vtblk_disk); 388227652Sgrehan sc->vtblk_disk = NULL; 389227652Sgrehan } 390227652Sgrehan 391227652Sgrehan if (sc->vtblk_sglist != NULL) { 392227652Sgrehan sglist_free(sc->vtblk_sglist); 393227652Sgrehan sc->vtblk_sglist = NULL; 394227652Sgrehan } 395227652Sgrehan 396227652Sgrehan VTBLK_LOCK_DESTROY(sc); 397227652Sgrehan 398227652Sgrehan return (0); 399227652Sgrehan} 400227652Sgrehan 401227652Sgrehanstatic int 402227652Sgrehanvtblk_suspend(device_t dev) 403227652Sgrehan{ 404227652Sgrehan struct vtblk_softc *sc; 405234270Sgrehan int error; 406227652Sgrehan 407227652Sgrehan sc = device_get_softc(dev); 408227652Sgrehan 409227652Sgrehan VTBLK_LOCK(sc); 410234270Sgrehan sc->vtblk_flags |= VTBLK_FLAG_SUSPEND; 411234270Sgrehan /* XXX BMV: virtio_stop(), etc needed here? */ 412234270Sgrehan error = vtblk_quiesce(sc); 413234270Sgrehan if (error) 414234270Sgrehan sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND; 415227652Sgrehan VTBLK_UNLOCK(sc); 416227652Sgrehan 417234270Sgrehan return (error); 418227652Sgrehan} 419227652Sgrehan 420227652Sgrehanstatic int 421227652Sgrehanvtblk_resume(device_t dev) 422227652Sgrehan{ 423227652Sgrehan struct vtblk_softc *sc; 424227652Sgrehan 425227652Sgrehan sc = device_get_softc(dev); 426227652Sgrehan 427227652Sgrehan VTBLK_LOCK(sc); 428234270Sgrehan /* XXX BMV: virtio_reinit(), etc needed here? */ 429234270Sgrehan sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND; 430234270Sgrehan vtblk_startio(sc); 431227652Sgrehan VTBLK_UNLOCK(sc); 432227652Sgrehan 433227652Sgrehan return (0); 434227652Sgrehan} 435227652Sgrehan 436227652Sgrehanstatic int 437227652Sgrehanvtblk_shutdown(device_t dev) 438227652Sgrehan{ 439227652Sgrehan 440227652Sgrehan return (0); 441227652Sgrehan} 442227652Sgrehan 443227652Sgrehanstatic int 444227652Sgrehanvtblk_open(struct disk *dp) 445227652Sgrehan{ 446227652Sgrehan struct vtblk_softc *sc; 447227652Sgrehan 448227652Sgrehan if ((sc = dp->d_drv1) == NULL) 449227652Sgrehan return (ENXIO); 450227652Sgrehan 451234270Sgrehan return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0); 452227652Sgrehan} 453227652Sgrehan 454227652Sgrehanstatic int 455227652Sgrehanvtblk_close(struct disk *dp) 456227652Sgrehan{ 457227652Sgrehan struct vtblk_softc *sc; 458227652Sgrehan 459227652Sgrehan if ((sc = dp->d_drv1) == NULL) 460227652Sgrehan return (ENXIO); 461227652Sgrehan 462227652Sgrehan return (0); 463227652Sgrehan} 464227652Sgrehan 465227652Sgrehanstatic int 466227652Sgrehanvtblk_ioctl(struct disk *dp, u_long cmd, void *addr, int flag, 467227652Sgrehan struct thread *td) 468227652Sgrehan{ 469227652Sgrehan struct vtblk_softc *sc; 470227652Sgrehan 471227652Sgrehan if ((sc = dp->d_drv1) == NULL) 472227652Sgrehan return (ENXIO); 473227652Sgrehan 474227652Sgrehan return (ENOTTY); 475227652Sgrehan} 476227652Sgrehan 477227652Sgrehanstatic int 478227652Sgrehanvtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, 479227652Sgrehan size_t length) 480227652Sgrehan{ 481227652Sgrehan struct disk *dp; 482227652Sgrehan struct vtblk_softc *sc; 483227652Sgrehan int error; 484227652Sgrehan 485227652Sgrehan dp = arg; 486227652Sgrehan 487227652Sgrehan if ((sc = dp->d_drv1) == NULL) 488227652Sgrehan return (ENXIO); 489227652Sgrehan 490234270Sgrehan VTBLK_LOCK(sc); 491234270Sgrehan 492227652Sgrehan if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) { 493227652Sgrehan vtblk_prepare_dump(sc); 494227652Sgrehan sc->vtblk_flags |= VTBLK_FLAG_DUMPING; 495227652Sgrehan } 496227652Sgrehan 497227652Sgrehan if (length > 0) 498227652Sgrehan error = vtblk_write_dump(sc, virtual, offset, length); 499227652Sgrehan else if (virtual == NULL && offset == 0) 500227652Sgrehan error = vtblk_flush_dump(sc); 501234270Sgrehan else { 502234270Sgrehan error = EINVAL; 503234270Sgrehan sc->vtblk_flags &= ~VTBLK_FLAG_DUMPING; 504234270Sgrehan } 505227652Sgrehan 506227652Sgrehan VTBLK_UNLOCK(sc); 507227652Sgrehan 508227652Sgrehan return (error); 509227652Sgrehan} 510227652Sgrehan 511227652Sgrehanstatic void 512227652Sgrehanvtblk_strategy(struct bio *bp) 513227652Sgrehan{ 514227652Sgrehan struct vtblk_softc *sc; 515227652Sgrehan 516227652Sgrehan if ((sc = bp->bio_disk->d_drv1) == NULL) { 517234270Sgrehan vtblk_finish_bio(bp, EINVAL); 518227652Sgrehan return; 519227652Sgrehan } 520227652Sgrehan 521227652Sgrehan /* 522227652Sgrehan * Fail any write if RO. Unfortunately, there does not seem to 523227652Sgrehan * be a better way to report our readonly'ness to GEOM above. 524227652Sgrehan */ 525227652Sgrehan if (sc->vtblk_flags & VTBLK_FLAG_READONLY && 526227652Sgrehan (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) { 527234270Sgrehan vtblk_finish_bio(bp, EROFS); 528227652Sgrehan return; 529227652Sgrehan } 530227652Sgrehan 531238360Sgrehan#ifdef INVARIANTS 532227652Sgrehan /* 533227652Sgrehan * Prevent read/write buffers spanning too many segments from 534227652Sgrehan * getting into the queue. This should only trip if d_maxsize 535227652Sgrehan * was incorrectly set. 536227652Sgrehan */ 537227652Sgrehan if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { 538234270Sgrehan int nsegs, max_nsegs; 539238360Sgrehan 540234270Sgrehan nsegs = sglist_count(bp->bio_data, bp->bio_bcount); 541234270Sgrehan max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS; 542234270Sgrehan 543234270Sgrehan KASSERT(nsegs <= max_nsegs, 544238360Sgrehan ("bio %p spanned too many segments: %d, max: %d", 545238360Sgrehan bp, nsegs, max_nsegs)); 546227652Sgrehan } 547234270Sgrehan#endif 548227652Sgrehan 549227652Sgrehan VTBLK_LOCK(sc); 550234270Sgrehan if (sc->vtblk_flags & VTBLK_FLAG_DETACH) 551234270Sgrehan vtblk_finish_bio(bp, ENXIO); 552234270Sgrehan else { 553227652Sgrehan bioq_disksort(&sc->vtblk_bioq, bp); 554234270Sgrehan 555234270Sgrehan if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0) 556234270Sgrehan vtblk_startio(sc); 557234270Sgrehan } 558227652Sgrehan VTBLK_UNLOCK(sc); 559227652Sgrehan} 560227652Sgrehan 561227652Sgrehanstatic void 562227652Sgrehanvtblk_negotiate_features(struct vtblk_softc *sc) 563227652Sgrehan{ 564227652Sgrehan device_t dev; 565227652Sgrehan uint64_t features; 566227652Sgrehan 567227652Sgrehan dev = sc->vtblk_dev; 568227652Sgrehan features = VTBLK_FEATURES; 569227652Sgrehan 570227652Sgrehan sc->vtblk_features = virtio_negotiate_features(dev, features); 571227652Sgrehan} 572227652Sgrehan 573227652Sgrehanstatic int 574227652Sgrehanvtblk_maximum_segments(struct vtblk_softc *sc, 575227652Sgrehan struct virtio_blk_config *blkcfg) 576227652Sgrehan{ 577227652Sgrehan device_t dev; 578227652Sgrehan int nsegs; 579227652Sgrehan 580227652Sgrehan dev = sc->vtblk_dev; 581227652Sgrehan nsegs = VTBLK_MIN_SEGMENTS; 582227652Sgrehan 583227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) { 584227652Sgrehan nsegs += MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1); 585227652Sgrehan if (sc->vtblk_flags & VTBLK_FLAG_INDIRECT) 586227652Sgrehan nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); 587227652Sgrehan } else 588227652Sgrehan nsegs += 1; 589227652Sgrehan 590227652Sgrehan return (nsegs); 591227652Sgrehan} 592227652Sgrehan 593227652Sgrehanstatic int 594227652Sgrehanvtblk_alloc_virtqueue(struct vtblk_softc *sc) 595227652Sgrehan{ 596227652Sgrehan device_t dev; 597227652Sgrehan struct vq_alloc_info vq_info; 598227652Sgrehan 599227652Sgrehan dev = sc->vtblk_dev; 600227652Sgrehan 601227652Sgrehan VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs, 602227652Sgrehan vtblk_vq_intr, sc, &sc->vtblk_vq, 603227652Sgrehan "%s request", device_get_nameunit(dev)); 604227652Sgrehan 605227652Sgrehan return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info)); 606227652Sgrehan} 607227652Sgrehan 608227652Sgrehanstatic void 609227652Sgrehanvtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) 610227652Sgrehan{ 611227652Sgrehan device_t dev; 612227652Sgrehan struct disk *dp; 613227652Sgrehan 614227652Sgrehan dev = sc->vtblk_dev; 615227652Sgrehan 616227652Sgrehan sc->vtblk_disk = dp = disk_alloc(); 617227652Sgrehan dp->d_open = vtblk_open; 618227652Sgrehan dp->d_close = vtblk_close; 619227652Sgrehan dp->d_ioctl = vtblk_ioctl; 620227652Sgrehan dp->d_strategy = vtblk_strategy; 621227652Sgrehan dp->d_name = VTBLK_DISK_NAME; 622228301Sgrehan dp->d_unit = device_get_unit(dev); 623227652Sgrehan dp->d_drv1 = sc; 624227652Sgrehan 625227652Sgrehan if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0) 626227652Sgrehan dp->d_dump = vtblk_dump; 627227652Sgrehan 628227652Sgrehan /* Capacity is always in 512-byte units. */ 629227652Sgrehan dp->d_mediasize = blkcfg->capacity * 512; 630227652Sgrehan 631227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_BLK_SIZE)) 632228301Sgrehan dp->d_sectorsize = blkcfg->blk_size; 633227652Sgrehan else 634228301Sgrehan dp->d_sectorsize = 512; 635227652Sgrehan 636227652Sgrehan /* 637227652Sgrehan * The VirtIO maximum I/O size is given in terms of segments. 638227652Sgrehan * However, FreeBSD limits I/O size by logical buffer size, not 639227652Sgrehan * by physically contiguous pages. Therefore, we have to assume 640227652Sgrehan * no pages are contiguous. This may impose an artificially low 641227652Sgrehan * maximum I/O size. But in practice, since QEMU advertises 128 642227652Sgrehan * segments, this gives us a maximum IO size of 125 * PAGE_SIZE, 643227652Sgrehan * which is typically greater than MAXPHYS. Eventually we should 644227652Sgrehan * just advertise MAXPHYS and split buffers that are too big. 645227652Sgrehan * 646227652Sgrehan * Note we must subtract one additional segment in case of non 647227652Sgrehan * page aligned buffers. 648227652Sgrehan */ 649227652Sgrehan dp->d_maxsize = (sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS - 1) * 650227652Sgrehan PAGE_SIZE; 651227652Sgrehan if (dp->d_maxsize < PAGE_SIZE) 652227652Sgrehan dp->d_maxsize = PAGE_SIZE; /* XXX */ 653227652Sgrehan 654227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_GEOMETRY)) { 655227652Sgrehan dp->d_fwsectors = blkcfg->geometry.sectors; 656227652Sgrehan dp->d_fwheads = blkcfg->geometry.heads; 657227652Sgrehan } 658227652Sgrehan 659227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH)) 660227652Sgrehan dp->d_flags |= DISKFLAG_CANFLUSHCACHE; 661227652Sgrehan} 662227652Sgrehan 663227652Sgrehanstatic void 664227652Sgrehanvtblk_create_disk(struct vtblk_softc *sc) 665227652Sgrehan{ 666227652Sgrehan struct disk *dp; 667227652Sgrehan 668227652Sgrehan dp = sc->vtblk_disk; 669227652Sgrehan 670227652Sgrehan /* 671227652Sgrehan * Retrieving the identification string must be done after 672227652Sgrehan * the virtqueue interrupt is setup otherwise it will hang. 673227652Sgrehan */ 674227652Sgrehan vtblk_get_ident(sc); 675227652Sgrehan 676227652Sgrehan device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n", 677227652Sgrehan (uintmax_t) dp->d_mediasize >> 20, 678227652Sgrehan (uintmax_t) dp->d_mediasize / dp->d_sectorsize, 679227652Sgrehan dp->d_sectorsize); 680227652Sgrehan 681227652Sgrehan disk_create(dp, DISK_VERSION); 682227652Sgrehan} 683227652Sgrehan 684234270Sgrehanstatic int 685234270Sgrehanvtblk_quiesce(struct vtblk_softc *sc) 686234270Sgrehan{ 687234270Sgrehan int error; 688234270Sgrehan 689234270Sgrehan error = 0; 690234270Sgrehan 691234270Sgrehan VTBLK_LOCK_ASSERT(sc); 692234270Sgrehan 693234270Sgrehan while (!virtqueue_empty(sc->vtblk_vq)) { 694234270Sgrehan if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq", 695234270Sgrehan VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) { 696234270Sgrehan error = EBUSY; 697234270Sgrehan break; 698234270Sgrehan } 699234270Sgrehan } 700234270Sgrehan 701234270Sgrehan return (error); 702234270Sgrehan} 703234270Sgrehan 704227652Sgrehanstatic void 705227652Sgrehanvtblk_startio(struct vtblk_softc *sc) 706227652Sgrehan{ 707227652Sgrehan struct virtqueue *vq; 708227652Sgrehan struct vtblk_request *req; 709227652Sgrehan int enq; 710227652Sgrehan 711227652Sgrehan vq = sc->vtblk_vq; 712227652Sgrehan enq = 0; 713227652Sgrehan 714227652Sgrehan VTBLK_LOCK_ASSERT(sc); 715227652Sgrehan 716227652Sgrehan while (!virtqueue_full(vq)) { 717227652Sgrehan if ((req = vtblk_dequeue_ready(sc)) == NULL) 718227652Sgrehan req = vtblk_bio_request(sc); 719227652Sgrehan if (req == NULL) 720227652Sgrehan break; 721227652Sgrehan 722227652Sgrehan if (vtblk_execute_request(sc, req) != 0) { 723227652Sgrehan vtblk_enqueue_ready(sc, req); 724227652Sgrehan break; 725227652Sgrehan } 726227652Sgrehan 727227652Sgrehan enq++; 728227652Sgrehan } 729227652Sgrehan 730227652Sgrehan if (enq > 0) 731227652Sgrehan virtqueue_notify(vq); 732227652Sgrehan} 733227652Sgrehan 734227652Sgrehanstatic struct vtblk_request * 735227652Sgrehanvtblk_bio_request(struct vtblk_softc *sc) 736227652Sgrehan{ 737227652Sgrehan struct bio_queue_head *bioq; 738227652Sgrehan struct vtblk_request *req; 739227652Sgrehan struct bio *bp; 740227652Sgrehan 741227652Sgrehan bioq = &sc->vtblk_bioq; 742227652Sgrehan 743227652Sgrehan if (bioq_first(bioq) == NULL) 744227652Sgrehan return (NULL); 745227652Sgrehan 746227652Sgrehan req = vtblk_dequeue_request(sc); 747227652Sgrehan if (req == NULL) 748227652Sgrehan return (NULL); 749227652Sgrehan 750227652Sgrehan bp = bioq_takefirst(bioq); 751227652Sgrehan req->vbr_bp = bp; 752227652Sgrehan req->vbr_ack = -1; 753227652Sgrehan req->vbr_hdr.ioprio = 1; 754227652Sgrehan 755227652Sgrehan switch (bp->bio_cmd) { 756227652Sgrehan case BIO_FLUSH: 757227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; 758227652Sgrehan break; 759227652Sgrehan case BIO_READ: 760227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_IN; 761227652Sgrehan req->vbr_hdr.sector = bp->bio_offset / 512; 762227652Sgrehan break; 763227652Sgrehan case BIO_WRITE: 764227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_OUT; 765227652Sgrehan req->vbr_hdr.sector = bp->bio_offset / 512; 766227652Sgrehan break; 767227652Sgrehan default: 768234270Sgrehan panic("%s: bio with unhandled cmd: %d", __FUNCTION__, 769234270Sgrehan bp->bio_cmd); 770227652Sgrehan } 771227652Sgrehan 772227652Sgrehan return (req); 773227652Sgrehan} 774227652Sgrehan 775227652Sgrehanstatic int 776227652Sgrehanvtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req) 777227652Sgrehan{ 778247829Sbryanv struct virtqueue *vq; 779227652Sgrehan struct sglist *sg; 780227652Sgrehan struct bio *bp; 781247829Sbryanv int ordered, readable, writable, error; 782227652Sgrehan 783247829Sbryanv vq = sc->vtblk_vq; 784227652Sgrehan sg = sc->vtblk_sglist; 785227652Sgrehan bp = req->vbr_bp; 786247829Sbryanv ordered = 0; 787227652Sgrehan writable = 0; 788227652Sgrehan 789227652Sgrehan VTBLK_LOCK_ASSERT(sc); 790227652Sgrehan 791247829Sbryanv /* 792247829Sbryanv * Wait until the ordered request completes before 793247829Sbryanv * executing subsequent requests. 794247829Sbryanv */ 795247829Sbryanv if (sc->vtblk_req_ordered != NULL) 796247829Sbryanv return (EBUSY); 797247829Sbryanv 798247829Sbryanv if (bp->bio_flags & BIO_ORDERED) { 799247829Sbryanv if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) { 800247829Sbryanv /* 801247829Sbryanv * This request will be executed once all 802247829Sbryanv * the in-flight requests are completed. 803247829Sbryanv */ 804247829Sbryanv if (!virtqueue_empty(vq)) 805247829Sbryanv return (EBUSY); 806247829Sbryanv ordered = 1; 807247829Sbryanv } else 808247829Sbryanv req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER; 809247829Sbryanv } 810247829Sbryanv 811227652Sgrehan sglist_reset(sg); 812227652Sgrehan 813238360Sgrehan sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr)); 814238360Sgrehan 815227652Sgrehan if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { 816227652Sgrehan error = sglist_append(sg, bp->bio_data, bp->bio_bcount); 817238360Sgrehan if (error || sg->sg_nseg == sg->sg_maxseg) 818238360Sgrehan panic("%s: data buffer too big bio:%p error:%d", 819238360Sgrehan __FUNCTION__, bp, error); 820227652Sgrehan 821227652Sgrehan /* BIO_READ means the host writes into our buffer. */ 822227652Sgrehan if (bp->bio_cmd == BIO_READ) 823238360Sgrehan writable = sg->sg_nseg - 1; 824227652Sgrehan } 825227652Sgrehan 826227652Sgrehan writable++; 827238360Sgrehan sglist_append(sg, &req->vbr_ack, sizeof(uint8_t)); 828234270Sgrehan readable = sg->sg_nseg - writable; 829227652Sgrehan 830247829Sbryanv error = virtqueue_enqueue(vq, req, sg, readable, writable); 831247829Sbryanv if (error == 0 && ordered) 832247829Sbryanv sc->vtblk_req_ordered = req; 833247829Sbryanv 834247829Sbryanv return (error); 835227652Sgrehan} 836227652Sgrehan 837227652Sgrehanstatic int 838227652Sgrehanvtblk_vq_intr(void *xsc) 839227652Sgrehan{ 840227652Sgrehan struct vtblk_softc *sc; 841227652Sgrehan 842227652Sgrehan sc = xsc; 843227652Sgrehan 844227652Sgrehan virtqueue_disable_intr(sc->vtblk_vq); 845227652Sgrehan taskqueue_enqueue_fast(sc->vtblk_tq, &sc->vtblk_intr_task); 846227652Sgrehan 847227652Sgrehan return (1); 848227652Sgrehan} 849227652Sgrehan 850227652Sgrehanstatic void 851227652Sgrehanvtblk_intr_task(void *arg, int pending) 852227652Sgrehan{ 853227652Sgrehan struct vtblk_softc *sc; 854227652Sgrehan struct virtqueue *vq; 855227652Sgrehan 856227652Sgrehan sc = arg; 857227652Sgrehan vq = sc->vtblk_vq; 858227652Sgrehan 859227652Sgrehan VTBLK_LOCK(sc); 860234270Sgrehan if (sc->vtblk_flags & VTBLK_FLAG_DETACH) { 861227652Sgrehan VTBLK_UNLOCK(sc); 862227652Sgrehan return; 863227652Sgrehan } 864227652Sgrehan 865234270Sgrehan vtblk_finish_completed(sc); 866227652Sgrehan 867234270Sgrehan if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0) 868234270Sgrehan vtblk_startio(sc); 869234270Sgrehan else 870234270Sgrehan wakeup(&sc->vtblk_vq); 871227652Sgrehan 872227652Sgrehan if (virtqueue_enable_intr(vq) != 0) { 873227652Sgrehan virtqueue_disable_intr(vq); 874227652Sgrehan VTBLK_UNLOCK(sc); 875227652Sgrehan taskqueue_enqueue_fast(sc->vtblk_tq, 876227652Sgrehan &sc->vtblk_intr_task); 877227652Sgrehan return; 878227652Sgrehan } 879227652Sgrehan 880227652Sgrehan VTBLK_UNLOCK(sc); 881227652Sgrehan} 882227652Sgrehan 883227652Sgrehanstatic void 884227652Sgrehanvtblk_stop(struct vtblk_softc *sc) 885227652Sgrehan{ 886227652Sgrehan 887227652Sgrehan virtqueue_disable_intr(sc->vtblk_vq); 888227652Sgrehan virtio_stop(sc->vtblk_dev); 889227652Sgrehan} 890227652Sgrehan 891227652Sgrehanstatic void 892227652Sgrehanvtblk_get_ident(struct vtblk_softc *sc) 893227652Sgrehan{ 894227652Sgrehan struct bio buf; 895227652Sgrehan struct disk *dp; 896227652Sgrehan struct vtblk_request *req; 897227652Sgrehan int len, error; 898227652Sgrehan 899227652Sgrehan dp = sc->vtblk_disk; 900227652Sgrehan len = MIN(VIRTIO_BLK_ID_BYTES, DISK_IDENT_SIZE); 901227652Sgrehan 902227652Sgrehan if (vtblk_no_ident != 0) 903227652Sgrehan return; 904227652Sgrehan 905227652Sgrehan req = vtblk_dequeue_request(sc); 906227652Sgrehan if (req == NULL) 907227652Sgrehan return; 908227652Sgrehan 909227652Sgrehan req->vbr_ack = -1; 910227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID; 911227652Sgrehan req->vbr_hdr.ioprio = 1; 912227652Sgrehan req->vbr_hdr.sector = 0; 913227652Sgrehan 914227652Sgrehan req->vbr_bp = &buf; 915227652Sgrehan bzero(&buf, sizeof(struct bio)); 916227652Sgrehan 917227652Sgrehan buf.bio_cmd = BIO_READ; 918227652Sgrehan buf.bio_data = dp->d_ident; 919227652Sgrehan buf.bio_bcount = len; 920227652Sgrehan 921227652Sgrehan VTBLK_LOCK(sc); 922227652Sgrehan error = vtblk_poll_request(sc, req); 923227652Sgrehan VTBLK_UNLOCK(sc); 924227652Sgrehan 925228301Sgrehan vtblk_enqueue_request(sc, req); 926228301Sgrehan 927227652Sgrehan if (error) { 928227652Sgrehan device_printf(sc->vtblk_dev, 929227652Sgrehan "error getting device identifier: %d\n", error); 930227652Sgrehan } 931227652Sgrehan} 932227652Sgrehan 933227652Sgrehanstatic void 934227652Sgrehanvtblk_prepare_dump(struct vtblk_softc *sc) 935227652Sgrehan{ 936227652Sgrehan device_t dev; 937227652Sgrehan struct virtqueue *vq; 938227652Sgrehan 939227652Sgrehan dev = sc->vtblk_dev; 940227652Sgrehan vq = sc->vtblk_vq; 941227652Sgrehan 942227652Sgrehan vtblk_stop(sc); 943227652Sgrehan 944227652Sgrehan /* 945227652Sgrehan * Drain all requests caught in-flight in the virtqueue, 946227652Sgrehan * skipping biodone(). When dumping, only one request is 947227652Sgrehan * outstanding at a time, and we just poll the virtqueue 948227652Sgrehan * for the response. 949227652Sgrehan */ 950227652Sgrehan vtblk_drain_vq(sc, 1); 951227652Sgrehan 952227652Sgrehan if (virtio_reinit(dev, sc->vtblk_features) != 0) 953227652Sgrehan panic("cannot reinit VirtIO block device during dump"); 954227652Sgrehan 955227652Sgrehan virtqueue_disable_intr(vq); 956227652Sgrehan virtio_reinit_complete(dev); 957227652Sgrehan} 958227652Sgrehan 959227652Sgrehanstatic int 960227652Sgrehanvtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset, 961227652Sgrehan size_t length) 962227652Sgrehan{ 963227652Sgrehan struct bio buf; 964227652Sgrehan struct vtblk_request *req; 965227652Sgrehan 966227652Sgrehan req = &sc->vtblk_dump_request; 967227652Sgrehan req->vbr_ack = -1; 968227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_OUT; 969227652Sgrehan req->vbr_hdr.ioprio = 1; 970227652Sgrehan req->vbr_hdr.sector = offset / 512; 971227652Sgrehan 972227652Sgrehan req->vbr_bp = &buf; 973227652Sgrehan bzero(&buf, sizeof(struct bio)); 974227652Sgrehan 975227652Sgrehan buf.bio_cmd = BIO_WRITE; 976227652Sgrehan buf.bio_data = virtual; 977227652Sgrehan buf.bio_bcount = length; 978227652Sgrehan 979227652Sgrehan return (vtblk_poll_request(sc, req)); 980227652Sgrehan} 981227652Sgrehan 982227652Sgrehanstatic int 983227652Sgrehanvtblk_flush_dump(struct vtblk_softc *sc) 984227652Sgrehan{ 985227652Sgrehan struct bio buf; 986227652Sgrehan struct vtblk_request *req; 987227652Sgrehan 988227652Sgrehan req = &sc->vtblk_dump_request; 989227652Sgrehan req->vbr_ack = -1; 990227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; 991227652Sgrehan req->vbr_hdr.ioprio = 1; 992227652Sgrehan req->vbr_hdr.sector = 0; 993227652Sgrehan 994227652Sgrehan req->vbr_bp = &buf; 995227652Sgrehan bzero(&buf, sizeof(struct bio)); 996227652Sgrehan 997227652Sgrehan buf.bio_cmd = BIO_FLUSH; 998227652Sgrehan 999227652Sgrehan return (vtblk_poll_request(sc, req)); 1000227652Sgrehan} 1001227652Sgrehan 1002227652Sgrehanstatic int 1003227652Sgrehanvtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req) 1004227652Sgrehan{ 1005227652Sgrehan struct virtqueue *vq; 1006234349Sgrehan struct vtblk_request *r; 1007227652Sgrehan int error; 1008227652Sgrehan 1009227652Sgrehan vq = sc->vtblk_vq; 1010227652Sgrehan 1011227652Sgrehan if (!virtqueue_empty(vq)) 1012227652Sgrehan return (EBUSY); 1013227652Sgrehan 1014227652Sgrehan error = vtblk_execute_request(sc, req); 1015227652Sgrehan if (error) 1016227652Sgrehan return (error); 1017227652Sgrehan 1018227652Sgrehan virtqueue_notify(vq); 1019227652Sgrehan 1020234349Sgrehan r = virtqueue_poll(vq, NULL); 1021238360Sgrehan KASSERT(r == req, ("unexpected request response: %p/%p", r, req)); 1022227652Sgrehan 1023234270Sgrehan error = vtblk_request_error(req); 1024234270Sgrehan if (error && bootverbose) { 1025238360Sgrehan device_printf(sc->vtblk_dev, 1026238360Sgrehan "%s: IO error: %d\n", __FUNCTION__, error); 1027227652Sgrehan } 1028227652Sgrehan 1029227652Sgrehan return (error); 1030227652Sgrehan} 1031227652Sgrehan 1032227652Sgrehanstatic void 1033234270Sgrehanvtblk_finish_completed(struct vtblk_softc *sc) 1034234270Sgrehan{ 1035234270Sgrehan struct vtblk_request *req; 1036234270Sgrehan struct bio *bp; 1037234270Sgrehan int error; 1038234270Sgrehan 1039234270Sgrehan while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) { 1040234270Sgrehan bp = req->vbr_bp; 1041234270Sgrehan 1042247829Sbryanv if (sc->vtblk_req_ordered != NULL) { 1043247829Sbryanv /* This should be the only outstanding request. */ 1044247829Sbryanv MPASS(sc->vtblk_req_ordered == req); 1045247829Sbryanv sc->vtblk_req_ordered = NULL; 1046247829Sbryanv } 1047247829Sbryanv 1048234270Sgrehan error = vtblk_request_error(req); 1049234270Sgrehan if (error) 1050234270Sgrehan disk_err(bp, "hard error", -1, 1); 1051234270Sgrehan 1052234270Sgrehan vtblk_finish_bio(bp, error); 1053234270Sgrehan vtblk_enqueue_request(sc, req); 1054234270Sgrehan } 1055234270Sgrehan} 1056234270Sgrehan 1057234270Sgrehanstatic void 1058227652Sgrehanvtblk_drain_vq(struct vtblk_softc *sc, int skip_done) 1059227652Sgrehan{ 1060227652Sgrehan struct virtqueue *vq; 1061227652Sgrehan struct vtblk_request *req; 1062227652Sgrehan int last; 1063227652Sgrehan 1064227652Sgrehan vq = sc->vtblk_vq; 1065227652Sgrehan last = 0; 1066227652Sgrehan 1067227652Sgrehan while ((req = virtqueue_drain(vq, &last)) != NULL) { 1068227652Sgrehan if (!skip_done) 1069234270Sgrehan vtblk_finish_bio(req->vbr_bp, ENXIO); 1070227652Sgrehan 1071227652Sgrehan vtblk_enqueue_request(sc, req); 1072227652Sgrehan } 1073227652Sgrehan 1074247829Sbryanv sc->vtblk_req_ordered = NULL; 1075227652Sgrehan KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); 1076227652Sgrehan} 1077227652Sgrehan 1078227652Sgrehanstatic void 1079227652Sgrehanvtblk_drain(struct vtblk_softc *sc) 1080227652Sgrehan{ 1081227652Sgrehan struct bio_queue_head *bioq; 1082227652Sgrehan struct vtblk_request *req; 1083227652Sgrehan struct bio *bp; 1084227652Sgrehan 1085227652Sgrehan bioq = &sc->vtblk_bioq; 1086227652Sgrehan 1087234270Sgrehan if (sc->vtblk_vq != NULL) { 1088234270Sgrehan vtblk_finish_completed(sc); 1089227652Sgrehan vtblk_drain_vq(sc, 0); 1090234270Sgrehan } 1091227652Sgrehan 1092227652Sgrehan while ((req = vtblk_dequeue_ready(sc)) != NULL) { 1093234270Sgrehan vtblk_finish_bio(req->vbr_bp, ENXIO); 1094227652Sgrehan vtblk_enqueue_request(sc, req); 1095227652Sgrehan } 1096227652Sgrehan 1097227652Sgrehan while (bioq_first(bioq) != NULL) { 1098227652Sgrehan bp = bioq_takefirst(bioq); 1099234270Sgrehan vtblk_finish_bio(bp, ENXIO); 1100227652Sgrehan } 1101227652Sgrehan 1102227652Sgrehan vtblk_free_requests(sc); 1103227652Sgrehan} 1104227652Sgrehan 1105238360Sgrehan#ifdef INVARIANTS 1106238360Sgrehanstatic void 1107238360Sgrehanvtblk_request_invariants(struct vtblk_request *req) 1108238360Sgrehan{ 1109238360Sgrehan int hdr_nsegs, ack_nsegs; 1110238360Sgrehan 1111238360Sgrehan hdr_nsegs = sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)); 1112238360Sgrehan ack_nsegs = sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)); 1113238360Sgrehan 1114238360Sgrehan KASSERT(hdr_nsegs == 1, ("request header crossed page boundary")); 1115238360Sgrehan KASSERT(ack_nsegs == 1, ("request ack crossed page boundary")); 1116238360Sgrehan} 1117238360Sgrehan#endif 1118238360Sgrehan 1119227652Sgrehanstatic int 1120227652Sgrehanvtblk_alloc_requests(struct vtblk_softc *sc) 1121227652Sgrehan{ 1122227652Sgrehan struct vtblk_request *req; 1123234270Sgrehan int i, nreqs; 1124227652Sgrehan 1125234270Sgrehan nreqs = virtqueue_size(sc->vtblk_vq); 1126227652Sgrehan 1127227652Sgrehan /* 1128227652Sgrehan * Preallocate sufficient requests to keep the virtqueue full. Each 1129227652Sgrehan * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce 1130227652Sgrehan * the number allocated when indirect descriptors are not available. 1131227652Sgrehan */ 1132227652Sgrehan if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0) 1133234270Sgrehan nreqs /= VTBLK_MIN_SEGMENTS; 1134227652Sgrehan 1135234270Sgrehan for (i = 0; i < nreqs; i++) { 1136238360Sgrehan req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT); 1137227652Sgrehan if (req == NULL) 1138227652Sgrehan return (ENOMEM); 1139227652Sgrehan 1140238360Sgrehan#ifdef INVARIANTS 1141238360Sgrehan vtblk_request_invariants(req); 1142238360Sgrehan#endif 1143238360Sgrehan 1144227652Sgrehan sc->vtblk_request_count++; 1145227652Sgrehan vtblk_enqueue_request(sc, req); 1146227652Sgrehan } 1147227652Sgrehan 1148227652Sgrehan return (0); 1149227652Sgrehan} 1150227652Sgrehan 1151227652Sgrehanstatic void 1152227652Sgrehanvtblk_free_requests(struct vtblk_softc *sc) 1153227652Sgrehan{ 1154227652Sgrehan struct vtblk_request *req; 1155227652Sgrehan 1156234270Sgrehan KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready), 1157234270Sgrehan ("ready requests left on queue")); 1158234270Sgrehan 1159227652Sgrehan while ((req = vtblk_dequeue_request(sc)) != NULL) { 1160227652Sgrehan sc->vtblk_request_count--; 1161238360Sgrehan free(req, M_DEVBUF); 1162227652Sgrehan } 1163227652Sgrehan 1164238360Sgrehan KASSERT(sc->vtblk_request_count == 0, 1165238360Sgrehan ("leaked requests: %d", sc->vtblk_request_count)); 1166227652Sgrehan} 1167227652Sgrehan 1168227652Sgrehanstatic struct vtblk_request * 1169227652Sgrehanvtblk_dequeue_request(struct vtblk_softc *sc) 1170227652Sgrehan{ 1171227652Sgrehan struct vtblk_request *req; 1172227652Sgrehan 1173227652Sgrehan req = TAILQ_FIRST(&sc->vtblk_req_free); 1174227652Sgrehan if (req != NULL) 1175227652Sgrehan TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link); 1176227652Sgrehan 1177227652Sgrehan return (req); 1178227652Sgrehan} 1179227652Sgrehan 1180227652Sgrehanstatic void 1181227652Sgrehanvtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req) 1182227652Sgrehan{ 1183227652Sgrehan 1184227652Sgrehan bzero(req, sizeof(struct vtblk_request)); 1185227652Sgrehan TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link); 1186227652Sgrehan} 1187227652Sgrehan 1188227652Sgrehanstatic struct vtblk_request * 1189227652Sgrehanvtblk_dequeue_ready(struct vtblk_softc *sc) 1190227652Sgrehan{ 1191227652Sgrehan struct vtblk_request *req; 1192227652Sgrehan 1193227652Sgrehan req = TAILQ_FIRST(&sc->vtblk_req_ready); 1194227652Sgrehan if (req != NULL) 1195227652Sgrehan TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link); 1196227652Sgrehan 1197227652Sgrehan return (req); 1198227652Sgrehan} 1199227652Sgrehan 1200227652Sgrehanstatic void 1201227652Sgrehanvtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req) 1202227652Sgrehan{ 1203227652Sgrehan 1204227652Sgrehan TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link); 1205227652Sgrehan} 1206227652Sgrehan 1207234270Sgrehanstatic int 1208234270Sgrehanvtblk_request_error(struct vtblk_request *req) 1209234270Sgrehan{ 1210234270Sgrehan int error; 1211234270Sgrehan 1212234270Sgrehan switch (req->vbr_ack) { 1213234270Sgrehan case VIRTIO_BLK_S_OK: 1214234270Sgrehan error = 0; 1215234270Sgrehan break; 1216234270Sgrehan case VIRTIO_BLK_S_UNSUPP: 1217234270Sgrehan error = ENOTSUP; 1218234270Sgrehan break; 1219234270Sgrehan default: 1220234270Sgrehan error = EIO; 1221234270Sgrehan break; 1222234270Sgrehan } 1223234270Sgrehan 1224234270Sgrehan return (error); 1225234270Sgrehan} 1226234270Sgrehan 1227227652Sgrehanstatic void 1228234270Sgrehanvtblk_finish_bio(struct bio *bp, int error) 1229227652Sgrehan{ 1230227652Sgrehan 1231234270Sgrehan if (error) { 1232234270Sgrehan bp->bio_resid = bp->bio_bcount; 1233234270Sgrehan bp->bio_error = error; 1234234270Sgrehan bp->bio_flags |= BIO_ERROR; 1235234270Sgrehan } 1236234270Sgrehan 1237234270Sgrehan biodone(bp); 1238227652Sgrehan} 1239