1227652Sgrehan/*- 2252707Sbryanv * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 3227652Sgrehan * All rights reserved. 4227652Sgrehan * 5227652Sgrehan * Redistribution and use in source and binary forms, with or without 6227652Sgrehan * modification, are permitted provided that the following conditions 7227652Sgrehan * are met: 8227652Sgrehan * 1. Redistributions of source code must retain the above copyright 9227652Sgrehan * notice unmodified, this list of conditions, and the following 10227652Sgrehan * disclaimer. 11227652Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 12227652Sgrehan * notice, this list of conditions and the following disclaimer in the 13227652Sgrehan * documentation and/or other materials provided with the distribution. 14227652Sgrehan * 15227652Sgrehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16227652Sgrehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17227652Sgrehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18227652Sgrehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19227652Sgrehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20227652Sgrehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21227652Sgrehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22227652Sgrehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23227652Sgrehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24227652Sgrehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25227652Sgrehan */ 26227652Sgrehan 27227652Sgrehan/* Driver for VirtIO block devices. */ 28227652Sgrehan 29227652Sgrehan#include <sys/cdefs.h> 30227652Sgrehan__FBSDID("$FreeBSD: releng/10.3/sys/dev/virtio/block/virtio_blk.c 284344 2015-06-13 17:40:33Z bryanv $"); 31227652Sgrehan 32227652Sgrehan#include <sys/param.h> 33227652Sgrehan#include <sys/systm.h> 34227652Sgrehan#include <sys/kernel.h> 35227652Sgrehan#include <sys/bio.h> 36227652Sgrehan#include <sys/malloc.h> 37227652Sgrehan#include <sys/module.h> 38227652Sgrehan#include <sys/sglist.h> 39252703Sbryanv#include <sys/sysctl.h> 40227652Sgrehan#include <sys/lock.h> 41227652Sgrehan#include <sys/mutex.h> 42227652Sgrehan#include <sys/queue.h> 43227652Sgrehan 44227652Sgrehan#include <geom/geom_disk.h> 45227652Sgrehan 46227652Sgrehan#include <machine/bus.h> 47227652Sgrehan#include <machine/resource.h> 48227652Sgrehan#include <sys/bus.h> 49227652Sgrehan#include <sys/rman.h> 50227652Sgrehan 51227652Sgrehan#include <dev/virtio/virtio.h> 52227652Sgrehan#include <dev/virtio/virtqueue.h> 53227652Sgrehan#include <dev/virtio/block/virtio_blk.h> 54227652Sgrehan 55227652Sgrehan#include "virtio_if.h" 56227652Sgrehan 57227652Sgrehanstruct vtblk_request { 58227652Sgrehan struct virtio_blk_outhdr vbr_hdr; 59227652Sgrehan struct bio *vbr_bp; 60227652Sgrehan uint8_t vbr_ack; 61227652Sgrehan TAILQ_ENTRY(vtblk_request) vbr_link; 62227652Sgrehan}; 63227652Sgrehan 64252703Sbryanvenum vtblk_cache_mode { 65252703Sbryanv VTBLK_CACHE_WRITETHROUGH, 66252703Sbryanv VTBLK_CACHE_WRITEBACK, 67252703Sbryanv VTBLK_CACHE_MAX 68252703Sbryanv}; 69252703Sbryanv 70227652Sgrehanstruct vtblk_softc { 71227652Sgrehan device_t vtblk_dev; 72227652Sgrehan struct mtx vtblk_mtx; 73227652Sgrehan uint64_t vtblk_features; 74227652Sgrehan uint32_t vtblk_flags; 75227652Sgrehan#define VTBLK_FLAG_INDIRECT 0x0001 76227652Sgrehan#define VTBLK_FLAG_READONLY 0x0002 77234270Sgrehan#define VTBLK_FLAG_DETACH 0x0004 78234270Sgrehan#define VTBLK_FLAG_SUSPEND 0x0008 79284344Sbryanv#define VTBLK_FLAG_BARRIER 0x0010 80284344Sbryanv#define VTBLK_FLAG_WC_CONFIG 0x0020 81227652Sgrehan 82227652Sgrehan struct virtqueue *vtblk_vq; 83227652Sgrehan struct sglist *vtblk_sglist; 84227652Sgrehan struct disk *vtblk_disk; 85227652Sgrehan 86227652Sgrehan struct bio_queue_head vtblk_bioq; 87227652Sgrehan TAILQ_HEAD(, vtblk_request) 88227652Sgrehan vtblk_req_free; 89227652Sgrehan TAILQ_HEAD(, vtblk_request) 90247829Sbryanv vtblk_req_ready; 91247829Sbryanv struct vtblk_request *vtblk_req_ordered; 92227652Sgrehan 93227652Sgrehan int vtblk_max_nsegs; 94227652Sgrehan int vtblk_request_count; 95252703Sbryanv enum vtblk_cache_mode vtblk_write_cache; 96227652Sgrehan 97284344Sbryanv struct bio_queue vtblk_dump_queue; 98227652Sgrehan struct vtblk_request vtblk_dump_request; 99227652Sgrehan}; 100227652Sgrehan 101227652Sgrehanstatic struct virtio_feature_desc vtblk_feature_desc[] = { 102227652Sgrehan { VIRTIO_BLK_F_BARRIER, "HostBarrier" }, 103227652Sgrehan { VIRTIO_BLK_F_SIZE_MAX, "MaxSegSize" }, 104227652Sgrehan { VIRTIO_BLK_F_SEG_MAX, "MaxNumSegs" }, 105227652Sgrehan { VIRTIO_BLK_F_GEOMETRY, "DiskGeometry" }, 106227652Sgrehan { VIRTIO_BLK_F_RO, "ReadOnly" }, 107227652Sgrehan { VIRTIO_BLK_F_BLK_SIZE, "BlockSize" }, 108227652Sgrehan { VIRTIO_BLK_F_SCSI, "SCSICmds" }, 109252703Sbryanv { VIRTIO_BLK_F_WCE, "WriteCache" }, 110227652Sgrehan { VIRTIO_BLK_F_TOPOLOGY, "Topology" }, 111252703Sbryanv { VIRTIO_BLK_F_CONFIG_WCE, "ConfigWCE" }, 112227652Sgrehan 113227652Sgrehan { 0, NULL } 114227652Sgrehan}; 115227652Sgrehan 116227652Sgrehanstatic int vtblk_modevent(module_t, int, void *); 117227652Sgrehan 118227652Sgrehanstatic int vtblk_probe(device_t); 119227652Sgrehanstatic int vtblk_attach(device_t); 120227652Sgrehanstatic int vtblk_detach(device_t); 121227652Sgrehanstatic int vtblk_suspend(device_t); 122227652Sgrehanstatic int vtblk_resume(device_t); 123227652Sgrehanstatic int vtblk_shutdown(device_t); 124252703Sbryanvstatic int vtblk_config_change(device_t); 125227652Sgrehan 126234270Sgrehanstatic int vtblk_open(struct disk *); 127234270Sgrehanstatic int vtblk_close(struct disk *); 128234270Sgrehanstatic int vtblk_ioctl(struct disk *, u_long, void *, int, 129238360Sgrehan struct thread *); 130234270Sgrehanstatic int vtblk_dump(void *, void *, vm_offset_t, off_t, size_t); 131234270Sgrehanstatic void vtblk_strategy(struct bio *); 132234270Sgrehan 133227652Sgrehanstatic void vtblk_negotiate_features(struct vtblk_softc *); 134276487Sbryanvstatic void vtblk_setup_features(struct vtblk_softc *); 135227652Sgrehanstatic int vtblk_maximum_segments(struct vtblk_softc *, 136227652Sgrehan struct virtio_blk_config *); 137227652Sgrehanstatic int vtblk_alloc_virtqueue(struct vtblk_softc *); 138252703Sbryanvstatic void vtblk_resize_disk(struct vtblk_softc *, uint64_t); 139227652Sgrehanstatic void vtblk_alloc_disk(struct vtblk_softc *, 140227652Sgrehan struct virtio_blk_config *); 141227652Sgrehanstatic void vtblk_create_disk(struct vtblk_softc *); 142227652Sgrehan 143276487Sbryanvstatic int vtblk_request_prealloc(struct vtblk_softc *); 144276487Sbryanvstatic void vtblk_request_free(struct vtblk_softc *); 145276487Sbryanvstatic struct vtblk_request * 146276487Sbryanv vtblk_request_dequeue(struct vtblk_softc *); 147276487Sbryanvstatic void vtblk_request_enqueue(struct vtblk_softc *, 148227652Sgrehan struct vtblk_request *); 149276487Sbryanvstatic struct vtblk_request * 150276487Sbryanv vtblk_request_next_ready(struct vtblk_softc *); 151276487Sbryanvstatic void vtblk_request_requeue_ready(struct vtblk_softc *, 152276487Sbryanv struct vtblk_request *); 153276487Sbryanvstatic struct vtblk_request * 154276487Sbryanv vtblk_request_next(struct vtblk_softc *); 155276487Sbryanvstatic struct vtblk_request * 156276487Sbryanv vtblk_request_bio(struct vtblk_softc *); 157276487Sbryanvstatic int vtblk_request_execute(struct vtblk_softc *, 158276487Sbryanv struct vtblk_request *); 159276487Sbryanvstatic int vtblk_request_error(struct vtblk_request *); 160227652Sgrehan 161276487Sbryanvstatic void vtblk_queue_completed(struct vtblk_softc *, 162276487Sbryanv struct bio_queue *); 163276487Sbryanvstatic void vtblk_done_completed(struct vtblk_softc *, 164276487Sbryanv struct bio_queue *); 165284344Sbryanvstatic void vtblk_drain_vq(struct vtblk_softc *); 166276487Sbryanvstatic void vtblk_drain(struct vtblk_softc *); 167227652Sgrehan 168276487Sbryanvstatic void vtblk_startio(struct vtblk_softc *); 169276487Sbryanvstatic void vtblk_bio_done(struct vtblk_softc *, struct bio *, int); 170227652Sgrehan 171252703Sbryanvstatic void vtblk_read_config(struct vtblk_softc *, 172252703Sbryanv struct virtio_blk_config *); 173276487Sbryanvstatic void vtblk_ident(struct vtblk_softc *); 174227652Sgrehanstatic int vtblk_poll_request(struct vtblk_softc *, 175227652Sgrehan struct vtblk_request *); 176276487Sbryanvstatic int vtblk_quiesce(struct vtblk_softc *); 177276487Sbryanvstatic void vtblk_vq_intr(void *); 178276487Sbryanvstatic void vtblk_stop(struct vtblk_softc *); 179227652Sgrehan 180284344Sbryanvstatic void vtblk_dump_quiesce(struct vtblk_softc *); 181276487Sbryanvstatic int vtblk_dump_write(struct vtblk_softc *, void *, off_t, size_t); 182276487Sbryanvstatic int vtblk_dump_flush(struct vtblk_softc *); 183284344Sbryanvstatic void vtblk_dump_complete(struct vtblk_softc *); 184227652Sgrehan 185276487Sbryanvstatic void vtblk_set_write_cache(struct vtblk_softc *, int); 186276487Sbryanvstatic int vtblk_write_cache_enabled(struct vtblk_softc *sc, 187276487Sbryanv struct virtio_blk_config *); 188276487Sbryanvstatic int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS); 189227652Sgrehan 190252703Sbryanvstatic void vtblk_setup_sysctl(struct vtblk_softc *); 191252703Sbryanvstatic int vtblk_tunable_int(struct vtblk_softc *, const char *, int); 192252703Sbryanv 193227652Sgrehan/* Tunables. */ 194227652Sgrehanstatic int vtblk_no_ident = 0; 195227652SgrehanTUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident); 196252703Sbryanvstatic int vtblk_writecache_mode = -1; 197252703SbryanvTUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode); 198227652Sgrehan 199227652Sgrehan/* Features desired/implemented by this driver. */ 200227652Sgrehan#define VTBLK_FEATURES \ 201227652Sgrehan (VIRTIO_BLK_F_BARRIER | \ 202227652Sgrehan VIRTIO_BLK_F_SIZE_MAX | \ 203227652Sgrehan VIRTIO_BLK_F_SEG_MAX | \ 204227652Sgrehan VIRTIO_BLK_F_GEOMETRY | \ 205227652Sgrehan VIRTIO_BLK_F_RO | \ 206227652Sgrehan VIRTIO_BLK_F_BLK_SIZE | \ 207252703Sbryanv VIRTIO_BLK_F_WCE | \ 208280241Smav VIRTIO_BLK_F_TOPOLOGY | \ 209252703Sbryanv VIRTIO_BLK_F_CONFIG_WCE | \ 210227652Sgrehan VIRTIO_RING_F_INDIRECT_DESC) 211227652Sgrehan 212227652Sgrehan#define VTBLK_MTX(_sc) &(_sc)->vtblk_mtx 213227652Sgrehan#define VTBLK_LOCK_INIT(_sc, _name) \ 214227652Sgrehan mtx_init(VTBLK_MTX((_sc)), (_name), \ 215252703Sbryanv "VirtIO Block Lock", MTX_DEF) 216227652Sgrehan#define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc))) 217227652Sgrehan#define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc))) 218227652Sgrehan#define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc))) 219227652Sgrehan#define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED) 220227652Sgrehan#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \ 221227652Sgrehan mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED) 222227652Sgrehan 223227652Sgrehan#define VTBLK_DISK_NAME "vtbd" 224238360Sgrehan#define VTBLK_QUIESCE_TIMEOUT (30 * hz) 225227652Sgrehan 226227652Sgrehan/* 227227652Sgrehan * Each block request uses at least two segments - one for the header 228227652Sgrehan * and one for the status. 229227652Sgrehan */ 230227652Sgrehan#define VTBLK_MIN_SEGMENTS 2 231227652Sgrehan 232227652Sgrehanstatic device_method_t vtblk_methods[] = { 233227652Sgrehan /* Device methods. */ 234227652Sgrehan DEVMETHOD(device_probe, vtblk_probe), 235227652Sgrehan DEVMETHOD(device_attach, vtblk_attach), 236227652Sgrehan DEVMETHOD(device_detach, vtblk_detach), 237227652Sgrehan DEVMETHOD(device_suspend, vtblk_suspend), 238227652Sgrehan DEVMETHOD(device_resume, vtblk_resume), 239227652Sgrehan DEVMETHOD(device_shutdown, vtblk_shutdown), 240227652Sgrehan 241252703Sbryanv /* VirtIO methods. */ 242252703Sbryanv DEVMETHOD(virtio_config_change, vtblk_config_change), 243252703Sbryanv 244234270Sgrehan DEVMETHOD_END 245227652Sgrehan}; 246227652Sgrehan 247227652Sgrehanstatic driver_t vtblk_driver = { 248227652Sgrehan "vtblk", 249227652Sgrehan vtblk_methods, 250227652Sgrehan sizeof(struct vtblk_softc) 251227652Sgrehan}; 252227652Sgrehanstatic devclass_t vtblk_devclass; 253227652Sgrehan 254227652SgrehanDRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass, 255227652Sgrehan vtblk_modevent, 0); 256227652SgrehanMODULE_VERSION(virtio_blk, 1); 257227652SgrehanMODULE_DEPEND(virtio_blk, virtio, 1, 1, 1); 258227652Sgrehan 259227652Sgrehanstatic int 260227652Sgrehanvtblk_modevent(module_t mod, int type, void *unused) 261227652Sgrehan{ 262227652Sgrehan int error; 263227652Sgrehan 264227652Sgrehan error = 0; 265227652Sgrehan 266227652Sgrehan switch (type) { 267227652Sgrehan case MOD_LOAD: 268227652Sgrehan case MOD_QUIESCE: 269227652Sgrehan case MOD_UNLOAD: 270227652Sgrehan case MOD_SHUTDOWN: 271227652Sgrehan break; 272227652Sgrehan default: 273227652Sgrehan error = EOPNOTSUPP; 274227652Sgrehan break; 275227652Sgrehan } 276227652Sgrehan 277227652Sgrehan return (error); 278227652Sgrehan} 279227652Sgrehan 280227652Sgrehanstatic int 281227652Sgrehanvtblk_probe(device_t dev) 282227652Sgrehan{ 283227652Sgrehan 284227652Sgrehan if (virtio_get_device_type(dev) != VIRTIO_ID_BLOCK) 285227652Sgrehan return (ENXIO); 286227652Sgrehan 287227652Sgrehan device_set_desc(dev, "VirtIO Block Adapter"); 288227652Sgrehan 289227652Sgrehan return (BUS_PROBE_DEFAULT); 290227652Sgrehan} 291227652Sgrehan 292227652Sgrehanstatic int 293227652Sgrehanvtblk_attach(device_t dev) 294227652Sgrehan{ 295227652Sgrehan struct vtblk_softc *sc; 296227652Sgrehan struct virtio_blk_config blkcfg; 297227652Sgrehan int error; 298227652Sgrehan 299276487Sbryanv virtio_set_feature_desc(dev, vtblk_feature_desc); 300276487Sbryanv 301227652Sgrehan sc = device_get_softc(dev); 302227652Sgrehan sc->vtblk_dev = dev; 303227652Sgrehan VTBLK_LOCK_INIT(sc, device_get_nameunit(dev)); 304227652Sgrehan bioq_init(&sc->vtblk_bioq); 305284344Sbryanv TAILQ_INIT(&sc->vtblk_dump_queue); 306227652Sgrehan TAILQ_INIT(&sc->vtblk_req_free); 307227652Sgrehan TAILQ_INIT(&sc->vtblk_req_ready); 308227652Sgrehan 309252703Sbryanv vtblk_setup_sysctl(sc); 310276487Sbryanv vtblk_setup_features(sc); 311252703Sbryanv 312252703Sbryanv vtblk_read_config(sc, &blkcfg); 313227652Sgrehan 314227652Sgrehan /* 315227652Sgrehan * With the current sglist(9) implementation, it is not easy 316227652Sgrehan * for us to support a maximum segment size as adjacent 317227652Sgrehan * segments are coalesced. For now, just make sure it's larger 318227652Sgrehan * than the maximum supported transfer size. 319227652Sgrehan */ 320227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) { 321227652Sgrehan if (blkcfg.size_max < MAXPHYS) { 322227652Sgrehan error = ENOTSUP; 323227652Sgrehan device_printf(dev, "host requires unsupported " 324227652Sgrehan "maximum segment size feature\n"); 325227652Sgrehan goto fail; 326227652Sgrehan } 327227652Sgrehan } 328227652Sgrehan 329227652Sgrehan sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg); 330238360Sgrehan if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) { 331234270Sgrehan error = EINVAL; 332234270Sgrehan device_printf(dev, "fewer than minimum number of segments " 333234270Sgrehan "allowed: %d\n", sc->vtblk_max_nsegs); 334234270Sgrehan goto fail; 335234270Sgrehan } 336227652Sgrehan 337227652Sgrehan sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT); 338227652Sgrehan if (sc->vtblk_sglist == NULL) { 339227652Sgrehan error = ENOMEM; 340227652Sgrehan device_printf(dev, "cannot allocate sglist\n"); 341227652Sgrehan goto fail; 342227652Sgrehan } 343227652Sgrehan 344227652Sgrehan error = vtblk_alloc_virtqueue(sc); 345227652Sgrehan if (error) { 346227652Sgrehan device_printf(dev, "cannot allocate virtqueue\n"); 347227652Sgrehan goto fail; 348227652Sgrehan } 349227652Sgrehan 350276487Sbryanv error = vtblk_request_prealloc(sc); 351227652Sgrehan if (error) { 352227652Sgrehan device_printf(dev, "cannot preallocate requests\n"); 353227652Sgrehan goto fail; 354227652Sgrehan } 355227652Sgrehan 356227652Sgrehan vtblk_alloc_disk(sc, &blkcfg); 357227652Sgrehan 358227652Sgrehan error = virtio_setup_intr(dev, INTR_TYPE_BIO | INTR_ENTROPY); 359227652Sgrehan if (error) { 360227652Sgrehan device_printf(dev, "cannot setup virtqueue interrupt\n"); 361227652Sgrehan goto fail; 362227652Sgrehan } 363227652Sgrehan 364227652Sgrehan vtblk_create_disk(sc); 365227652Sgrehan 366227652Sgrehan virtqueue_enable_intr(sc->vtblk_vq); 367227652Sgrehan 368227652Sgrehanfail: 369227652Sgrehan if (error) 370227652Sgrehan vtblk_detach(dev); 371227652Sgrehan 372227652Sgrehan return (error); 373227652Sgrehan} 374227652Sgrehan 375227652Sgrehanstatic int 376227652Sgrehanvtblk_detach(device_t dev) 377227652Sgrehan{ 378227652Sgrehan struct vtblk_softc *sc; 379227652Sgrehan 380227652Sgrehan sc = device_get_softc(dev); 381227652Sgrehan 382227652Sgrehan VTBLK_LOCK(sc); 383234270Sgrehan sc->vtblk_flags |= VTBLK_FLAG_DETACH; 384227652Sgrehan if (device_is_attached(dev)) 385227652Sgrehan vtblk_stop(sc); 386227652Sgrehan VTBLK_UNLOCK(sc); 387227652Sgrehan 388227652Sgrehan vtblk_drain(sc); 389227652Sgrehan 390227652Sgrehan if (sc->vtblk_disk != NULL) { 391227652Sgrehan disk_destroy(sc->vtblk_disk); 392227652Sgrehan sc->vtblk_disk = NULL; 393227652Sgrehan } 394227652Sgrehan 395227652Sgrehan if (sc->vtblk_sglist != NULL) { 396227652Sgrehan sglist_free(sc->vtblk_sglist); 397227652Sgrehan sc->vtblk_sglist = NULL; 398227652Sgrehan } 399227652Sgrehan 400227652Sgrehan VTBLK_LOCK_DESTROY(sc); 401227652Sgrehan 402227652Sgrehan return (0); 403227652Sgrehan} 404227652Sgrehan 405227652Sgrehanstatic int 406227652Sgrehanvtblk_suspend(device_t dev) 407227652Sgrehan{ 408227652Sgrehan struct vtblk_softc *sc; 409234270Sgrehan int error; 410227652Sgrehan 411227652Sgrehan sc = device_get_softc(dev); 412227652Sgrehan 413227652Sgrehan VTBLK_LOCK(sc); 414234270Sgrehan sc->vtblk_flags |= VTBLK_FLAG_SUSPEND; 415234270Sgrehan /* XXX BMV: virtio_stop(), etc needed here? */ 416234270Sgrehan error = vtblk_quiesce(sc); 417234270Sgrehan if (error) 418234270Sgrehan sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND; 419227652Sgrehan VTBLK_UNLOCK(sc); 420227652Sgrehan 421234270Sgrehan return (error); 422227652Sgrehan} 423227652Sgrehan 424227652Sgrehanstatic int 425227652Sgrehanvtblk_resume(device_t dev) 426227652Sgrehan{ 427227652Sgrehan struct vtblk_softc *sc; 428227652Sgrehan 429227652Sgrehan sc = device_get_softc(dev); 430227652Sgrehan 431227652Sgrehan VTBLK_LOCK(sc); 432234270Sgrehan /* XXX BMV: virtio_reinit(), etc needed here? */ 433234270Sgrehan sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND; 434234270Sgrehan vtblk_startio(sc); 435227652Sgrehan VTBLK_UNLOCK(sc); 436227652Sgrehan 437227652Sgrehan return (0); 438227652Sgrehan} 439227652Sgrehan 440227652Sgrehanstatic int 441227652Sgrehanvtblk_shutdown(device_t dev) 442227652Sgrehan{ 443227652Sgrehan 444227652Sgrehan return (0); 445227652Sgrehan} 446227652Sgrehan 447227652Sgrehanstatic int 448252703Sbryanvvtblk_config_change(device_t dev) 449252703Sbryanv{ 450252703Sbryanv struct vtblk_softc *sc; 451252703Sbryanv struct virtio_blk_config blkcfg; 452252703Sbryanv uint64_t capacity; 453252703Sbryanv 454252703Sbryanv sc = device_get_softc(dev); 455252703Sbryanv 456252703Sbryanv vtblk_read_config(sc, &blkcfg); 457252703Sbryanv 458252703Sbryanv /* Capacity is always in 512-byte units. */ 459252703Sbryanv capacity = blkcfg.capacity * 512; 460252703Sbryanv 461252703Sbryanv if (sc->vtblk_disk->d_mediasize != capacity) 462252703Sbryanv vtblk_resize_disk(sc, capacity); 463252703Sbryanv 464252703Sbryanv return (0); 465252703Sbryanv} 466252703Sbryanv 467252703Sbryanvstatic int 468227652Sgrehanvtblk_open(struct disk *dp) 469227652Sgrehan{ 470227652Sgrehan struct vtblk_softc *sc; 471227652Sgrehan 472227652Sgrehan if ((sc = dp->d_drv1) == NULL) 473227652Sgrehan return (ENXIO); 474227652Sgrehan 475234270Sgrehan return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0); 476227652Sgrehan} 477227652Sgrehan 478227652Sgrehanstatic int 479227652Sgrehanvtblk_close(struct disk *dp) 480227652Sgrehan{ 481227652Sgrehan struct vtblk_softc *sc; 482227652Sgrehan 483227652Sgrehan if ((sc = dp->d_drv1) == NULL) 484227652Sgrehan return (ENXIO); 485227652Sgrehan 486227652Sgrehan return (0); 487227652Sgrehan} 488227652Sgrehan 489227652Sgrehanstatic int 490227652Sgrehanvtblk_ioctl(struct disk *dp, u_long cmd, void *addr, int flag, 491227652Sgrehan struct thread *td) 492227652Sgrehan{ 493227652Sgrehan struct vtblk_softc *sc; 494227652Sgrehan 495227652Sgrehan if ((sc = dp->d_drv1) == NULL) 496227652Sgrehan return (ENXIO); 497227652Sgrehan 498227652Sgrehan return (ENOTTY); 499227652Sgrehan} 500227652Sgrehan 501227652Sgrehanstatic int 502227652Sgrehanvtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, 503227652Sgrehan size_t length) 504227652Sgrehan{ 505227652Sgrehan struct disk *dp; 506227652Sgrehan struct vtblk_softc *sc; 507227652Sgrehan int error; 508227652Sgrehan 509227652Sgrehan dp = arg; 510284344Sbryanv error = 0; 511227652Sgrehan 512227652Sgrehan if ((sc = dp->d_drv1) == NULL) 513227652Sgrehan return (ENXIO); 514227652Sgrehan 515234270Sgrehan VTBLK_LOCK(sc); 516234270Sgrehan 517284344Sbryanv vtblk_dump_quiesce(sc); 518227652Sgrehan 519227652Sgrehan if (length > 0) 520276487Sbryanv error = vtblk_dump_write(sc, virtual, offset, length); 521284344Sbryanv if (error || (virtual == NULL && offset == 0)) 522284344Sbryanv vtblk_dump_complete(sc); 523227652Sgrehan 524227652Sgrehan VTBLK_UNLOCK(sc); 525227652Sgrehan 526227652Sgrehan return (error); 527227652Sgrehan} 528227652Sgrehan 529227652Sgrehanstatic void 530227652Sgrehanvtblk_strategy(struct bio *bp) 531227652Sgrehan{ 532227652Sgrehan struct vtblk_softc *sc; 533227652Sgrehan 534227652Sgrehan if ((sc = bp->bio_disk->d_drv1) == NULL) { 535276487Sbryanv vtblk_bio_done(NULL, bp, EINVAL); 536227652Sgrehan return; 537227652Sgrehan } 538227652Sgrehan 539227652Sgrehan /* 540227652Sgrehan * Fail any write if RO. Unfortunately, there does not seem to 541227652Sgrehan * be a better way to report our readonly'ness to GEOM above. 542227652Sgrehan */ 543227652Sgrehan if (sc->vtblk_flags & VTBLK_FLAG_READONLY && 544227652Sgrehan (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) { 545276487Sbryanv vtblk_bio_done(sc, bp, EROFS); 546227652Sgrehan return; 547227652Sgrehan } 548227652Sgrehan 549276487Sbryanv VTBLK_LOCK(sc); 550238360Sgrehan 551276487Sbryanv if (sc->vtblk_flags & VTBLK_FLAG_DETACH) { 552276487Sbryanv VTBLK_UNLOCK(sc); 553276487Sbryanv vtblk_bio_done(sc, bp, ENXIO); 554276487Sbryanv return; 555227652Sgrehan } 556227652Sgrehan 557276487Sbryanv bioq_insert_tail(&sc->vtblk_bioq, bp); 558276487Sbryanv vtblk_startio(sc); 559234270Sgrehan 560227652Sgrehan VTBLK_UNLOCK(sc); 561227652Sgrehan} 562227652Sgrehan 563227652Sgrehanstatic void 564227652Sgrehanvtblk_negotiate_features(struct vtblk_softc *sc) 565227652Sgrehan{ 566227652Sgrehan device_t dev; 567227652Sgrehan uint64_t features; 568227652Sgrehan 569227652Sgrehan dev = sc->vtblk_dev; 570227652Sgrehan features = VTBLK_FEATURES; 571227652Sgrehan 572227652Sgrehan sc->vtblk_features = virtio_negotiate_features(dev, features); 573227652Sgrehan} 574227652Sgrehan 575276487Sbryanvstatic void 576276487Sbryanvvtblk_setup_features(struct vtblk_softc *sc) 577276487Sbryanv{ 578276487Sbryanv device_t dev; 579276487Sbryanv 580276487Sbryanv dev = sc->vtblk_dev; 581276487Sbryanv 582276487Sbryanv vtblk_negotiate_features(sc); 583276487Sbryanv 584276487Sbryanv if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 585276487Sbryanv sc->vtblk_flags |= VTBLK_FLAG_INDIRECT; 586276487Sbryanv if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) 587276487Sbryanv sc->vtblk_flags |= VTBLK_FLAG_READONLY; 588276487Sbryanv if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER)) 589276487Sbryanv sc->vtblk_flags |= VTBLK_FLAG_BARRIER; 590276487Sbryanv if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE)) 591276487Sbryanv sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG; 592276487Sbryanv} 593276487Sbryanv 594227652Sgrehanstatic int 595227652Sgrehanvtblk_maximum_segments(struct vtblk_softc *sc, 596227652Sgrehan struct virtio_blk_config *blkcfg) 597227652Sgrehan{ 598227652Sgrehan device_t dev; 599227652Sgrehan int nsegs; 600227652Sgrehan 601227652Sgrehan dev = sc->vtblk_dev; 602227652Sgrehan nsegs = VTBLK_MIN_SEGMENTS; 603227652Sgrehan 604227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) { 605227652Sgrehan nsegs += MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1); 606227652Sgrehan if (sc->vtblk_flags & VTBLK_FLAG_INDIRECT) 607227652Sgrehan nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); 608227652Sgrehan } else 609227652Sgrehan nsegs += 1; 610227652Sgrehan 611227652Sgrehan return (nsegs); 612227652Sgrehan} 613227652Sgrehan 614227652Sgrehanstatic int 615227652Sgrehanvtblk_alloc_virtqueue(struct vtblk_softc *sc) 616227652Sgrehan{ 617227652Sgrehan device_t dev; 618227652Sgrehan struct vq_alloc_info vq_info; 619227652Sgrehan 620227652Sgrehan dev = sc->vtblk_dev; 621227652Sgrehan 622227652Sgrehan VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs, 623227652Sgrehan vtblk_vq_intr, sc, &sc->vtblk_vq, 624227652Sgrehan "%s request", device_get_nameunit(dev)); 625227652Sgrehan 626227652Sgrehan return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info)); 627227652Sgrehan} 628227652Sgrehan 629227652Sgrehanstatic void 630252703Sbryanvvtblk_resize_disk(struct vtblk_softc *sc, uint64_t new_capacity) 631252703Sbryanv{ 632252703Sbryanv device_t dev; 633252703Sbryanv struct disk *dp; 634252703Sbryanv int error; 635252703Sbryanv 636252703Sbryanv dev = sc->vtblk_dev; 637252703Sbryanv dp = sc->vtblk_disk; 638252703Sbryanv 639252703Sbryanv dp->d_mediasize = new_capacity; 640252703Sbryanv if (bootverbose) { 641252703Sbryanv device_printf(dev, "resized to %juMB (%ju %u byte sectors)\n", 642252703Sbryanv (uintmax_t) dp->d_mediasize >> 20, 643252703Sbryanv (uintmax_t) dp->d_mediasize / dp->d_sectorsize, 644252703Sbryanv dp->d_sectorsize); 645252703Sbryanv } 646252703Sbryanv 647252703Sbryanv error = disk_resize(dp, M_NOWAIT); 648252703Sbryanv if (error) { 649252703Sbryanv device_printf(dev, 650252703Sbryanv "disk_resize(9) failed, error: %d\n", error); 651252703Sbryanv } 652252703Sbryanv} 653252703Sbryanv 654252703Sbryanvstatic void 655227652Sgrehanvtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) 656227652Sgrehan{ 657227652Sgrehan device_t dev; 658227652Sgrehan struct disk *dp; 659227652Sgrehan 660227652Sgrehan dev = sc->vtblk_dev; 661227652Sgrehan 662227652Sgrehan sc->vtblk_disk = dp = disk_alloc(); 663227652Sgrehan dp->d_open = vtblk_open; 664227652Sgrehan dp->d_close = vtblk_close; 665227652Sgrehan dp->d_ioctl = vtblk_ioctl; 666227652Sgrehan dp->d_strategy = vtblk_strategy; 667227652Sgrehan dp->d_name = VTBLK_DISK_NAME; 668228301Sgrehan dp->d_unit = device_get_unit(dev); 669227652Sgrehan dp->d_drv1 = sc; 670276487Sbryanv dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO | 671276487Sbryanv DISKFLAG_DIRECT_COMPLETION; 672252703Sbryanv dp->d_hba_vendor = virtio_get_vendor(dev); 673252703Sbryanv dp->d_hba_device = virtio_get_device(dev); 674252703Sbryanv dp->d_hba_subvendor = virtio_get_subvendor(dev); 675252703Sbryanv dp->d_hba_subdevice = virtio_get_subdevice(dev); 676227652Sgrehan 677227652Sgrehan if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0) 678227652Sgrehan dp->d_dump = vtblk_dump; 679227652Sgrehan 680227652Sgrehan /* Capacity is always in 512-byte units. */ 681227652Sgrehan dp->d_mediasize = blkcfg->capacity * 512; 682227652Sgrehan 683227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_BLK_SIZE)) 684228301Sgrehan dp->d_sectorsize = blkcfg->blk_size; 685227652Sgrehan else 686228301Sgrehan dp->d_sectorsize = 512; 687227652Sgrehan 688227652Sgrehan /* 689227652Sgrehan * The VirtIO maximum I/O size is given in terms of segments. 690227652Sgrehan * However, FreeBSD limits I/O size by logical buffer size, not 691227652Sgrehan * by physically contiguous pages. Therefore, we have to assume 692227652Sgrehan * no pages are contiguous. This may impose an artificially low 693227652Sgrehan * maximum I/O size. But in practice, since QEMU advertises 128 694227652Sgrehan * segments, this gives us a maximum IO size of 125 * PAGE_SIZE, 695227652Sgrehan * which is typically greater than MAXPHYS. Eventually we should 696227652Sgrehan * just advertise MAXPHYS and split buffers that are too big. 697227652Sgrehan * 698227652Sgrehan * Note we must subtract one additional segment in case of non 699227652Sgrehan * page aligned buffers. 700227652Sgrehan */ 701227652Sgrehan dp->d_maxsize = (sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS - 1) * 702227652Sgrehan PAGE_SIZE; 703227652Sgrehan if (dp->d_maxsize < PAGE_SIZE) 704227652Sgrehan dp->d_maxsize = PAGE_SIZE; /* XXX */ 705227652Sgrehan 706227652Sgrehan if (virtio_with_feature(dev, VIRTIO_BLK_F_GEOMETRY)) { 707227652Sgrehan dp->d_fwsectors = blkcfg->geometry.sectors; 708227652Sgrehan dp->d_fwheads = blkcfg->geometry.heads; 709227652Sgrehan } 710227652Sgrehan 711281976Smav if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY) && 712281976Smav blkcfg->topology.physical_block_exp > 0) { 713252703Sbryanv dp->d_stripesize = dp->d_sectorsize * 714252703Sbryanv (1 << blkcfg->topology.physical_block_exp); 715252703Sbryanv dp->d_stripeoffset = (dp->d_stripesize - 716252703Sbryanv blkcfg->topology.alignment_offset * dp->d_sectorsize) % 717252703Sbryanv dp->d_stripesize; 718252703Sbryanv } 719252703Sbryanv 720252703Sbryanv if (vtblk_write_cache_enabled(sc, blkcfg) != 0) 721252703Sbryanv sc->vtblk_write_cache = VTBLK_CACHE_WRITEBACK; 722252703Sbryanv else 723252703Sbryanv sc->vtblk_write_cache = VTBLK_CACHE_WRITETHROUGH; 724227652Sgrehan} 725227652Sgrehan 726227652Sgrehanstatic void 727227652Sgrehanvtblk_create_disk(struct vtblk_softc *sc) 728227652Sgrehan{ 729227652Sgrehan struct disk *dp; 730227652Sgrehan 731227652Sgrehan dp = sc->vtblk_disk; 732227652Sgrehan 733276487Sbryanv vtblk_ident(sc); 734227652Sgrehan 735227652Sgrehan device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n", 736227652Sgrehan (uintmax_t) dp->d_mediasize >> 20, 737227652Sgrehan (uintmax_t) dp->d_mediasize / dp->d_sectorsize, 738227652Sgrehan dp->d_sectorsize); 739227652Sgrehan 740227652Sgrehan disk_create(dp, DISK_VERSION); 741227652Sgrehan} 742227652Sgrehan 743234270Sgrehanstatic int 744276487Sbryanvvtblk_request_prealloc(struct vtblk_softc *sc) 745234270Sgrehan{ 746276487Sbryanv struct vtblk_request *req; 747276487Sbryanv int i, nreqs; 748234270Sgrehan 749276487Sbryanv nreqs = virtqueue_size(sc->vtblk_vq); 750234270Sgrehan 751276487Sbryanv /* 752276487Sbryanv * Preallocate sufficient requests to keep the virtqueue full. Each 753276487Sbryanv * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce 754276487Sbryanv * the number allocated when indirect descriptors are not available. 755276487Sbryanv */ 756276487Sbryanv if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0) 757276487Sbryanv nreqs /= VTBLK_MIN_SEGMENTS; 758234270Sgrehan 759276487Sbryanv for (i = 0; i < nreqs; i++) { 760276487Sbryanv req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT); 761276487Sbryanv if (req == NULL) 762276487Sbryanv return (ENOMEM); 763276487Sbryanv 764276487Sbryanv MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1); 765276487Sbryanv MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1); 766276487Sbryanv 767276487Sbryanv sc->vtblk_request_count++; 768276487Sbryanv vtblk_request_enqueue(sc, req); 769234270Sgrehan } 770234270Sgrehan 771276487Sbryanv return (0); 772234270Sgrehan} 773234270Sgrehan 774227652Sgrehanstatic void 775276487Sbryanvvtblk_request_free(struct vtblk_softc *sc) 776227652Sgrehan{ 777227652Sgrehan struct vtblk_request *req; 778227652Sgrehan 779276487Sbryanv MPASS(TAILQ_EMPTY(&sc->vtblk_req_ready)); 780227652Sgrehan 781276487Sbryanv while ((req = vtblk_request_dequeue(sc)) != NULL) { 782276487Sbryanv sc->vtblk_request_count--; 783276487Sbryanv free(req, M_DEVBUF); 784276487Sbryanv } 785227652Sgrehan 786276487Sbryanv KASSERT(sc->vtblk_request_count == 0, 787276487Sbryanv ("%s: leaked %d requests", __func__, sc->vtblk_request_count)); 788276487Sbryanv} 789227652Sgrehan 790276487Sbryanvstatic struct vtblk_request * 791276487Sbryanvvtblk_request_dequeue(struct vtblk_softc *sc) 792276487Sbryanv{ 793276487Sbryanv struct vtblk_request *req; 794227652Sgrehan 795276487Sbryanv req = TAILQ_FIRST(&sc->vtblk_req_free); 796276487Sbryanv if (req != NULL) { 797276487Sbryanv TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link); 798276487Sbryanv bzero(req, sizeof(struct vtblk_request)); 799227652Sgrehan } 800227652Sgrehan 801276487Sbryanv return (req); 802227652Sgrehan} 803227652Sgrehan 804276487Sbryanvstatic void 805276487Sbryanvvtblk_request_enqueue(struct vtblk_softc *sc, struct vtblk_request *req) 806276487Sbryanv{ 807276487Sbryanv 808276487Sbryanv TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link); 809276487Sbryanv} 810276487Sbryanv 811227652Sgrehanstatic struct vtblk_request * 812276487Sbryanvvtblk_request_next_ready(struct vtblk_softc *sc) 813227652Sgrehan{ 814276487Sbryanv struct vtblk_request *req; 815276487Sbryanv 816276487Sbryanv req = TAILQ_FIRST(&sc->vtblk_req_ready); 817276487Sbryanv if (req != NULL) 818276487Sbryanv TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link); 819276487Sbryanv 820276487Sbryanv return (req); 821276487Sbryanv} 822276487Sbryanv 823276487Sbryanvstatic void 824276487Sbryanvvtblk_request_requeue_ready(struct vtblk_softc *sc, struct vtblk_request *req) 825276487Sbryanv{ 826276487Sbryanv 827276487Sbryanv /* NOTE: Currently, there will be at most one request in the queue. */ 828276487Sbryanv TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link); 829276487Sbryanv} 830276487Sbryanv 831276487Sbryanvstatic struct vtblk_request * 832276487Sbryanvvtblk_request_next(struct vtblk_softc *sc) 833276487Sbryanv{ 834276487Sbryanv struct vtblk_request *req; 835276487Sbryanv 836276487Sbryanv req = vtblk_request_next_ready(sc); 837276487Sbryanv if (req != NULL) 838276487Sbryanv return (req); 839276487Sbryanv 840276487Sbryanv return (vtblk_request_bio(sc)); 841276487Sbryanv} 842276487Sbryanv 843276487Sbryanvstatic struct vtblk_request * 844276487Sbryanvvtblk_request_bio(struct vtblk_softc *sc) 845276487Sbryanv{ 846227652Sgrehan struct bio_queue_head *bioq; 847227652Sgrehan struct vtblk_request *req; 848227652Sgrehan struct bio *bp; 849227652Sgrehan 850227652Sgrehan bioq = &sc->vtblk_bioq; 851227652Sgrehan 852227652Sgrehan if (bioq_first(bioq) == NULL) 853227652Sgrehan return (NULL); 854227652Sgrehan 855276487Sbryanv req = vtblk_request_dequeue(sc); 856227652Sgrehan if (req == NULL) 857227652Sgrehan return (NULL); 858227652Sgrehan 859227652Sgrehan bp = bioq_takefirst(bioq); 860227652Sgrehan req->vbr_bp = bp; 861227652Sgrehan req->vbr_ack = -1; 862227652Sgrehan req->vbr_hdr.ioprio = 1; 863227652Sgrehan 864227652Sgrehan switch (bp->bio_cmd) { 865227652Sgrehan case BIO_FLUSH: 866227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; 867227652Sgrehan break; 868227652Sgrehan case BIO_READ: 869227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_IN; 870227652Sgrehan req->vbr_hdr.sector = bp->bio_offset / 512; 871227652Sgrehan break; 872227652Sgrehan case BIO_WRITE: 873227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_OUT; 874227652Sgrehan req->vbr_hdr.sector = bp->bio_offset / 512; 875227652Sgrehan break; 876227652Sgrehan default: 877252703Sbryanv panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd); 878227652Sgrehan } 879227652Sgrehan 880276487Sbryanv if (bp->bio_flags & BIO_ORDERED) 881276487Sbryanv req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER; 882276487Sbryanv 883227652Sgrehan return (req); 884227652Sgrehan} 885227652Sgrehan 886227652Sgrehanstatic int 887276487Sbryanvvtblk_request_execute(struct vtblk_softc *sc, struct vtblk_request *req) 888227652Sgrehan{ 889247829Sbryanv struct virtqueue *vq; 890227652Sgrehan struct sglist *sg; 891227652Sgrehan struct bio *bp; 892247829Sbryanv int ordered, readable, writable, error; 893227652Sgrehan 894247829Sbryanv vq = sc->vtblk_vq; 895227652Sgrehan sg = sc->vtblk_sglist; 896227652Sgrehan bp = req->vbr_bp; 897247829Sbryanv ordered = 0; 898227652Sgrehan writable = 0; 899227652Sgrehan 900247829Sbryanv /* 901276487Sbryanv * Some hosts (such as bhyve) do not implement the barrier feature, 902276487Sbryanv * so we emulate it in the driver by allowing the barrier request 903276487Sbryanv * to be the only one in flight. 904247829Sbryanv */ 905276487Sbryanv if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) { 906276487Sbryanv if (sc->vtblk_req_ordered != NULL) 907276487Sbryanv return (EBUSY); 908276487Sbryanv if (bp->bio_flags & BIO_ORDERED) { 909247829Sbryanv if (!virtqueue_empty(vq)) 910247829Sbryanv return (EBUSY); 911247829Sbryanv ordered = 1; 912276487Sbryanv req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER; 913276487Sbryanv } 914247829Sbryanv } 915247829Sbryanv 916227652Sgrehan sglist_reset(sg); 917238360Sgrehan sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr)); 918238360Sgrehan 919227652Sgrehan if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { 920260857Sbryanv error = sglist_append_bio(sg, bp); 921260857Sbryanv if (error || sg->sg_nseg == sg->sg_maxseg) { 922276487Sbryanv panic("%s: bio %p data buffer too big %d", 923252703Sbryanv __func__, bp, error); 924260857Sbryanv } 925227652Sgrehan 926227652Sgrehan /* BIO_READ means the host writes into our buffer. */ 927227652Sgrehan if (bp->bio_cmd == BIO_READ) 928238360Sgrehan writable = sg->sg_nseg - 1; 929227652Sgrehan } 930227652Sgrehan 931227652Sgrehan writable++; 932238360Sgrehan sglist_append(sg, &req->vbr_ack, sizeof(uint8_t)); 933234270Sgrehan readable = sg->sg_nseg - writable; 934227652Sgrehan 935247829Sbryanv error = virtqueue_enqueue(vq, req, sg, readable, writable); 936247829Sbryanv if (error == 0 && ordered) 937247829Sbryanv sc->vtblk_req_ordered = req; 938247829Sbryanv 939247829Sbryanv return (error); 940227652Sgrehan} 941227652Sgrehan 942276487Sbryanvstatic int 943276487Sbryanvvtblk_request_error(struct vtblk_request *req) 944276487Sbryanv{ 945276487Sbryanv int error; 946276487Sbryanv 947276487Sbryanv switch (req->vbr_ack) { 948276487Sbryanv case VIRTIO_BLK_S_OK: 949276487Sbryanv error = 0; 950276487Sbryanv break; 951276487Sbryanv case VIRTIO_BLK_S_UNSUPP: 952276487Sbryanv error = ENOTSUP; 953276487Sbryanv break; 954276487Sbryanv default: 955276487Sbryanv error = EIO; 956276487Sbryanv break; 957276487Sbryanv } 958276487Sbryanv 959276487Sbryanv return (error); 960276487Sbryanv} 961276487Sbryanv 962252702Sbryanvstatic void 963276487Sbryanvvtblk_queue_completed(struct vtblk_softc *sc, struct bio_queue *queue) 964227652Sgrehan{ 965276487Sbryanv struct vtblk_request *req; 966276487Sbryanv struct bio *bp; 967276487Sbryanv 968276487Sbryanv while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) { 969276487Sbryanv if (sc->vtblk_req_ordered != NULL) { 970276487Sbryanv MPASS(sc->vtblk_req_ordered == req); 971276487Sbryanv sc->vtblk_req_ordered = NULL; 972276487Sbryanv } 973276487Sbryanv 974276487Sbryanv bp = req->vbr_bp; 975276487Sbryanv bp->bio_error = vtblk_request_error(req); 976276487Sbryanv TAILQ_INSERT_TAIL(queue, bp, bio_queue); 977276487Sbryanv 978276487Sbryanv vtblk_request_enqueue(sc, req); 979276487Sbryanv } 980276487Sbryanv} 981276487Sbryanv 982276487Sbryanvstatic void 983276487Sbryanvvtblk_done_completed(struct vtblk_softc *sc, struct bio_queue *queue) 984276487Sbryanv{ 985276487Sbryanv struct bio *bp, *tmp; 986276487Sbryanv 987276487Sbryanv TAILQ_FOREACH_SAFE(bp, queue, bio_queue, tmp) { 988276487Sbryanv if (bp->bio_error != 0) 989276487Sbryanv disk_err(bp, "hard error", -1, 1); 990276487Sbryanv vtblk_bio_done(sc, bp, bp->bio_error); 991276487Sbryanv } 992276487Sbryanv} 993276487Sbryanv 994276487Sbryanvstatic void 995284344Sbryanvvtblk_drain_vq(struct vtblk_softc *sc) 996276487Sbryanv{ 997252702Sbryanv struct virtqueue *vq; 998276487Sbryanv struct vtblk_request *req; 999276487Sbryanv int last; 1000227652Sgrehan 1001227652Sgrehan vq = sc->vtblk_vq; 1002276487Sbryanv last = 0; 1003227652Sgrehan 1004276487Sbryanv while ((req = virtqueue_drain(vq, &last)) != NULL) { 1005284344Sbryanv vtblk_bio_done(sc, req->vbr_bp, ENXIO); 1006276487Sbryanv vtblk_request_enqueue(sc, req); 1007227652Sgrehan } 1008227652Sgrehan 1009276487Sbryanv sc->vtblk_req_ordered = NULL; 1010276487Sbryanv KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); 1011276487Sbryanv} 1012227652Sgrehan 1013276487Sbryanvstatic void 1014276487Sbryanvvtblk_drain(struct vtblk_softc *sc) 1015276487Sbryanv{ 1016276487Sbryanv struct bio_queue queue; 1017276487Sbryanv struct bio_queue_head *bioq; 1018276487Sbryanv struct vtblk_request *req; 1019276487Sbryanv struct bio *bp; 1020227652Sgrehan 1021276487Sbryanv bioq = &sc->vtblk_bioq; 1022276487Sbryanv TAILQ_INIT(&queue); 1023276487Sbryanv 1024276487Sbryanv if (sc->vtblk_vq != NULL) { 1025276487Sbryanv vtblk_queue_completed(sc, &queue); 1026276487Sbryanv vtblk_done_completed(sc, &queue); 1027276487Sbryanv 1028284344Sbryanv vtblk_drain_vq(sc); 1029227652Sgrehan } 1030227652Sgrehan 1031276487Sbryanv while ((req = vtblk_request_next_ready(sc)) != NULL) { 1032276487Sbryanv vtblk_bio_done(sc, req->vbr_bp, ENXIO); 1033276487Sbryanv vtblk_request_enqueue(sc, req); 1034276487Sbryanv } 1035276487Sbryanv 1036276487Sbryanv while (bioq_first(bioq) != NULL) { 1037276487Sbryanv bp = bioq_takefirst(bioq); 1038276487Sbryanv vtblk_bio_done(sc, bp, ENXIO); 1039276487Sbryanv } 1040276487Sbryanv 1041276487Sbryanv vtblk_request_free(sc); 1042227652Sgrehan} 1043227652Sgrehan 1044227652Sgrehanstatic void 1045276487Sbryanvvtblk_startio(struct vtblk_softc *sc) 1046227652Sgrehan{ 1047276487Sbryanv struct virtqueue *vq; 1048276487Sbryanv struct vtblk_request *req; 1049276487Sbryanv int enq; 1050227652Sgrehan 1051276487Sbryanv VTBLK_LOCK_ASSERT(sc); 1052276487Sbryanv vq = sc->vtblk_vq; 1053276487Sbryanv enq = 0; 1054276487Sbryanv 1055276487Sbryanv if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND) 1056276487Sbryanv return; 1057276487Sbryanv 1058276487Sbryanv while (!virtqueue_full(vq)) { 1059276487Sbryanv req = vtblk_request_next(sc); 1060276487Sbryanv if (req == NULL) 1061276487Sbryanv break; 1062276487Sbryanv 1063276487Sbryanv if (vtblk_request_execute(sc, req) != 0) { 1064276487Sbryanv vtblk_request_requeue_ready(sc, req); 1065276487Sbryanv break; 1066276487Sbryanv } 1067276487Sbryanv 1068276487Sbryanv enq++; 1069276487Sbryanv } 1070276487Sbryanv 1071276487Sbryanv if (enq > 0) 1072276487Sbryanv virtqueue_notify(vq); 1073227652Sgrehan} 1074227652Sgrehan 1075276487Sbryanvstatic void 1076276487Sbryanvvtblk_bio_done(struct vtblk_softc *sc, struct bio *bp, int error) 1077276487Sbryanv{ 1078276487Sbryanv 1079276487Sbryanv /* Because of GEOM direct dispatch, we cannot hold any locks. */ 1080276487Sbryanv if (sc != NULL) 1081276487Sbryanv VTBLK_LOCK_ASSERT_NOTOWNED(sc); 1082276487Sbryanv 1083276487Sbryanv if (error) { 1084276487Sbryanv bp->bio_resid = bp->bio_bcount; 1085276487Sbryanv bp->bio_error = error; 1086276487Sbryanv bp->bio_flags |= BIO_ERROR; 1087276487Sbryanv } 1088276487Sbryanv 1089276487Sbryanv biodone(bp); 1090276487Sbryanv} 1091276487Sbryanv 1092252703Sbryanv#define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg) \ 1093252703Sbryanv if (virtio_with_feature(_dev, _feature)) { \ 1094252703Sbryanv virtio_read_device_config(_dev, \ 1095252703Sbryanv offsetof(struct virtio_blk_config, _field), \ 1096252703Sbryanv &(_cfg)->_field, sizeof((_cfg)->_field)); \ 1097252703Sbryanv } 1098252703Sbryanv 1099227652Sgrehanstatic void 1100252703Sbryanvvtblk_read_config(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) 1101252703Sbryanv{ 1102252703Sbryanv device_t dev; 1103252703Sbryanv 1104252703Sbryanv dev = sc->vtblk_dev; 1105252703Sbryanv 1106252703Sbryanv bzero(blkcfg, sizeof(struct virtio_blk_config)); 1107252703Sbryanv 1108252703Sbryanv /* The capacity is always available. */ 1109252703Sbryanv virtio_read_device_config(dev, offsetof(struct virtio_blk_config, 1110252703Sbryanv capacity), &blkcfg->capacity, sizeof(blkcfg->capacity)); 1111252703Sbryanv 1112252703Sbryanv /* Read the configuration if the feature was negotiated. */ 1113252703Sbryanv VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg); 1114252703Sbryanv VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg); 1115252703Sbryanv VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry, blkcfg); 1116252703Sbryanv VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg); 1117252703Sbryanv VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology, blkcfg); 1118252703Sbryanv VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, writeback, blkcfg); 1119252703Sbryanv} 1120252703Sbryanv 1121252703Sbryanv#undef VTBLK_GET_CONFIG 1122252703Sbryanv 1123252703Sbryanvstatic void 1124276487Sbryanvvtblk_ident(struct vtblk_softc *sc) 1125227652Sgrehan{ 1126227652Sgrehan struct bio buf; 1127227652Sgrehan struct disk *dp; 1128227652Sgrehan struct vtblk_request *req; 1129227652Sgrehan int len, error; 1130227652Sgrehan 1131227652Sgrehan dp = sc->vtblk_disk; 1132227652Sgrehan len = MIN(VIRTIO_BLK_ID_BYTES, DISK_IDENT_SIZE); 1133227652Sgrehan 1134252703Sbryanv if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0) 1135227652Sgrehan return; 1136227652Sgrehan 1137276487Sbryanv req = vtblk_request_dequeue(sc); 1138227652Sgrehan if (req == NULL) 1139227652Sgrehan return; 1140227652Sgrehan 1141227652Sgrehan req->vbr_ack = -1; 1142227652Sgrehan req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID; 1143227652Sgrehan req->vbr_hdr.ioprio = 1; 1144227652Sgrehan req->vbr_hdr.sector = 0; 1145227652Sgrehan 1146227652Sgrehan req->vbr_bp = &buf; 1147227652Sgrehan bzero(&buf, sizeof(struct bio)); 1148227652Sgrehan 1149227652Sgrehan buf.bio_cmd = BIO_READ; 1150227652Sgrehan buf.bio_data = dp->d_ident; 1151227652Sgrehan buf.bio_bcount = len; 1152227652Sgrehan 1153227652Sgrehan VTBLK_LOCK(sc); 1154227652Sgrehan error = vtblk_poll_request(sc, req); 1155227652Sgrehan VTBLK_UNLOCK(sc); 1156227652Sgrehan 1157276487Sbryanv vtblk_request_enqueue(sc, req); 1158228301Sgrehan 1159227652Sgrehan if (error) { 1160227652Sgrehan device_printf(sc->vtblk_dev, 1161227652Sgrehan "error getting device identifier: %d\n", error); 1162227652Sgrehan } 1163227652Sgrehan} 1164227652Sgrehan 1165227652Sgrehanstatic int 1166227652Sgrehanvtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req) 1167227652Sgrehan{ 1168227652Sgrehan struct virtqueue *vq; 1169227652Sgrehan int error; 1170227652Sgrehan 1171227652Sgrehan vq = sc->vtblk_vq; 1172227652Sgrehan 1173227652Sgrehan if (!virtqueue_empty(vq)) 1174227652Sgrehan return (EBUSY); 1175227652Sgrehan 1176276487Sbryanv error = vtblk_request_execute(sc, req); 1177227652Sgrehan if (error) 1178227652Sgrehan return (error); 1179227652Sgrehan 1180227652Sgrehan virtqueue_notify(vq); 1181252703Sbryanv virtqueue_poll(vq, NULL); 1182227652Sgrehan 1183234270Sgrehan error = vtblk_request_error(req); 1184234270Sgrehan if (error && bootverbose) { 1185238360Sgrehan device_printf(sc->vtblk_dev, 1186252703Sbryanv "%s: IO error: %d\n", __func__, error); 1187227652Sgrehan } 1188227652Sgrehan 1189227652Sgrehan return (error); 1190227652Sgrehan} 1191227652Sgrehan 1192276487Sbryanvstatic int 1193276487Sbryanvvtblk_quiesce(struct vtblk_softc *sc) 1194234270Sgrehan{ 1195234270Sgrehan int error; 1196234270Sgrehan 1197276487Sbryanv VTBLK_LOCK_ASSERT(sc); 1198276487Sbryanv error = 0; 1199234270Sgrehan 1200276487Sbryanv while (!virtqueue_empty(sc->vtblk_vq)) { 1201276487Sbryanv if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq", 1202276487Sbryanv VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) { 1203276487Sbryanv error = EBUSY; 1204276487Sbryanv break; 1205247829Sbryanv } 1206276487Sbryanv } 1207247829Sbryanv 1208276487Sbryanv return (error); 1209234270Sgrehan} 1210234270Sgrehan 1211234270Sgrehanstatic void 1212276487Sbryanvvtblk_vq_intr(void *xsc) 1213227652Sgrehan{ 1214276487Sbryanv struct vtblk_softc *sc; 1215227652Sgrehan struct virtqueue *vq; 1216276487Sbryanv struct bio_queue queue; 1217227652Sgrehan 1218276487Sbryanv sc = xsc; 1219227652Sgrehan vq = sc->vtblk_vq; 1220276487Sbryanv TAILQ_INIT(&queue); 1221227652Sgrehan 1222276487Sbryanv VTBLK_LOCK(sc); 1223227652Sgrehan 1224276487Sbryanvagain: 1225276487Sbryanv if (sc->vtblk_flags & VTBLK_FLAG_DETACH) 1226276487Sbryanv goto out; 1227227652Sgrehan 1228276487Sbryanv vtblk_queue_completed(sc, &queue); 1229276487Sbryanv vtblk_startio(sc); 1230227652Sgrehan 1231276487Sbryanv if (virtqueue_enable_intr(vq) != 0) { 1232276487Sbryanv virtqueue_disable_intr(vq); 1233276487Sbryanv goto again; 1234234270Sgrehan } 1235227652Sgrehan 1236276487Sbryanv if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND) 1237276487Sbryanv wakeup(&sc->vtblk_vq); 1238227652Sgrehan 1239276487Sbryanvout: 1240276487Sbryanv VTBLK_UNLOCK(sc); 1241276487Sbryanv vtblk_done_completed(sc, &queue); 1242227652Sgrehan} 1243227652Sgrehan 1244238360Sgrehanstatic void 1245276487Sbryanvvtblk_stop(struct vtblk_softc *sc) 1246238360Sgrehan{ 1247238360Sgrehan 1248276487Sbryanv virtqueue_disable_intr(sc->vtblk_vq); 1249276487Sbryanv virtio_stop(sc->vtblk_dev); 1250238360Sgrehan} 1251238360Sgrehan 1252276487Sbryanvstatic void 1253284344Sbryanvvtblk_dump_quiesce(struct vtblk_softc *sc) 1254227652Sgrehan{ 1255227652Sgrehan 1256227652Sgrehan /* 1257284344Sbryanv * Spin here until all the requests in-flight at the time of the 1258284344Sbryanv * dump are completed and queued. The queued requests will be 1259284344Sbryanv * biodone'd once the dump is finished. 1260227652Sgrehan */ 1261284344Sbryanv while (!virtqueue_empty(sc->vtblk_vq)) 1262284344Sbryanv vtblk_queue_completed(sc, &sc->vtblk_dump_queue); 1263227652Sgrehan} 1264227652Sgrehan 1265276487Sbryanvstatic int 1266276487Sbryanvvtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset, 1267276487Sbryanv size_t length) 1268227652Sgrehan{ 1269276487Sbryanv struct bio buf; 1270227652Sgrehan struct vtblk_request *req; 1271227652Sgrehan 1272276487Sbryanv req = &sc->vtblk_dump_request; 1273276487Sbryanv req->vbr_ack = -1; 1274276487Sbryanv req->vbr_hdr.type = VIRTIO_BLK_T_OUT; 1275276487Sbryanv req->vbr_hdr.ioprio = 1; 1276276487Sbryanv req->vbr_hdr.sector = offset / 512; 1277234270Sgrehan 1278276487Sbryanv req->vbr_bp = &buf; 1279276487Sbryanv bzero(&buf, sizeof(struct bio)); 1280227652Sgrehan 1281276487Sbryanv buf.bio_cmd = BIO_WRITE; 1282276487Sbryanv buf.bio_data = virtual; 1283276487Sbryanv buf.bio_bcount = length; 1284276487Sbryanv 1285276487Sbryanv return (vtblk_poll_request(sc, req)); 1286227652Sgrehan} 1287227652Sgrehan 1288276487Sbryanvstatic int 1289276487Sbryanvvtblk_dump_flush(struct vtblk_softc *sc) 1290227652Sgrehan{ 1291276487Sbryanv struct bio buf; 1292227652Sgrehan struct vtblk_request *req; 1293227652Sgrehan 1294276487Sbryanv req = &sc->vtblk_dump_request; 1295276487Sbryanv req->vbr_ack = -1; 1296276487Sbryanv req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; 1297276487Sbryanv req->vbr_hdr.ioprio = 1; 1298276487Sbryanv req->vbr_hdr.sector = 0; 1299227652Sgrehan 1300276487Sbryanv req->vbr_bp = &buf; 1301276487Sbryanv bzero(&buf, sizeof(struct bio)); 1302276487Sbryanv 1303276487Sbryanv buf.bio_cmd = BIO_FLUSH; 1304276487Sbryanv 1305276487Sbryanv return (vtblk_poll_request(sc, req)); 1306227652Sgrehan} 1307227652Sgrehan 1308227652Sgrehanstatic void 1309284344Sbryanvvtblk_dump_complete(struct vtblk_softc *sc) 1310284344Sbryanv{ 1311284344Sbryanv 1312284344Sbryanv vtblk_dump_flush(sc); 1313284344Sbryanv 1314284344Sbryanv VTBLK_UNLOCK(sc); 1315284344Sbryanv vtblk_done_completed(sc, &sc->vtblk_dump_queue); 1316284344Sbryanv VTBLK_LOCK(sc); 1317284344Sbryanv} 1318284344Sbryanv 1319284344Sbryanvstatic void 1320276487Sbryanvvtblk_set_write_cache(struct vtblk_softc *sc, int wc) 1321227652Sgrehan{ 1322227652Sgrehan 1323276487Sbryanv /* Set either writeback (1) or writethrough (0) mode. */ 1324276487Sbryanv virtio_write_dev_config_1(sc->vtblk_dev, 1325276487Sbryanv offsetof(struct virtio_blk_config, writeback), wc); 1326227652Sgrehan} 1327227652Sgrehan 1328276487Sbryanvstatic int 1329276487Sbryanvvtblk_write_cache_enabled(struct vtblk_softc *sc, 1330276487Sbryanv struct virtio_blk_config *blkcfg) 1331227652Sgrehan{ 1332276487Sbryanv int wc; 1333227652Sgrehan 1334276487Sbryanv if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) { 1335276487Sbryanv wc = vtblk_tunable_int(sc, "writecache_mode", 1336276487Sbryanv vtblk_writecache_mode); 1337276487Sbryanv if (wc >= 0 && wc < VTBLK_CACHE_MAX) 1338276487Sbryanv vtblk_set_write_cache(sc, wc); 1339276487Sbryanv else 1340276487Sbryanv wc = blkcfg->writeback; 1341276487Sbryanv } else 1342276487Sbryanv wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE); 1343227652Sgrehan 1344276487Sbryanv return (wc); 1345227652Sgrehan} 1346227652Sgrehan 1347234270Sgrehanstatic int 1348276487Sbryanvvtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS) 1349234270Sgrehan{ 1350276487Sbryanv struct vtblk_softc *sc; 1351276487Sbryanv int wc, error; 1352234270Sgrehan 1353276487Sbryanv sc = oidp->oid_arg1; 1354276487Sbryanv wc = sc->vtblk_write_cache; 1355234270Sgrehan 1356276487Sbryanv error = sysctl_handle_int(oidp, &wc, 0, req); 1357276487Sbryanv if (error || req->newptr == NULL) 1358276487Sbryanv return (error); 1359276487Sbryanv if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0) 1360276487Sbryanv return (EPERM); 1361276487Sbryanv if (wc < 0 || wc >= VTBLK_CACHE_MAX) 1362276487Sbryanv return (EINVAL); 1363234270Sgrehan 1364276487Sbryanv VTBLK_LOCK(sc); 1365276487Sbryanv sc->vtblk_write_cache = wc; 1366276487Sbryanv vtblk_set_write_cache(sc, sc->vtblk_write_cache); 1367276487Sbryanv VTBLK_UNLOCK(sc); 1368227652Sgrehan 1369276487Sbryanv return (0); 1370227652Sgrehan} 1371252703Sbryanv 1372252703Sbryanvstatic void 1373252703Sbryanvvtblk_setup_sysctl(struct vtblk_softc *sc) 1374252703Sbryanv{ 1375252703Sbryanv device_t dev; 1376252703Sbryanv struct sysctl_ctx_list *ctx; 1377252703Sbryanv struct sysctl_oid *tree; 1378252703Sbryanv struct sysctl_oid_list *child; 1379252703Sbryanv 1380252703Sbryanv dev = sc->vtblk_dev; 1381252703Sbryanv ctx = device_get_sysctl_ctx(dev); 1382252703Sbryanv tree = device_get_sysctl_tree(dev); 1383252703Sbryanv child = SYSCTL_CHILDREN(tree); 1384252703Sbryanv 1385252703Sbryanv SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "writecache_mode", 1386252703Sbryanv CTLTYPE_INT | CTLFLAG_RW, sc, 0, vtblk_write_cache_sysctl, 1387252703Sbryanv "I", "Write cache mode (writethrough (0) or writeback (1))"); 1388252703Sbryanv} 1389252703Sbryanv 1390252703Sbryanvstatic int 1391252703Sbryanvvtblk_tunable_int(struct vtblk_softc *sc, const char *knob, int def) 1392252703Sbryanv{ 1393252703Sbryanv char path[64]; 1394252703Sbryanv 1395252703Sbryanv snprintf(path, sizeof(path), 1396252703Sbryanv "hw.vtblk.%d.%s", device_get_unit(sc->vtblk_dev), knob); 1397252703Sbryanv TUNABLE_INT_FETCH(path, &def); 1398252703Sbryanv 1399252703Sbryanv return (def); 1400252703Sbryanv} 1401