1/*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31/* 32 * BERI virtio block backend driver 33 */ 34 35#include <sys/cdefs.h> 36__FBSDID("$FreeBSD$"); 37 38#include <sys/param.h> 39#include <sys/systm.h> 40#include <sys/bus.h> 41#include <sys/kernel.h> 42#include <sys/module.h> 43#include <sys/rman.h> 44#include <sys/conf.h> 45#include <sys/stat.h> 46#include <sys/endian.h> 47#include <sys/disk.h> 48#include <sys/vnode.h> 49#include <sys/fcntl.h> 50#include <sys/kthread.h> 51#include <sys/buf.h> 52#include <sys/mdioctl.h> 53#include <sys/namei.h> 54 55#include <machine/bus.h> 56#include <machine/fdt.h> 57#include <machine/cpu.h> 58#include <machine/intr.h> 59 60#include <dev/fdt/fdt_common.h> 61#include <dev/ofw/openfirm.h> 62#include <dev/ofw/ofw_bus.h> 63#include <dev/ofw/ofw_bus_subr.h> 64 65#include <dev/beri/virtio/virtio.h> 66#include <dev/beri/virtio/virtio_mmio_platform.h> 67#include <dev/altera/pio/pio.h> 68#include <dev/virtio/mmio/virtio_mmio.h> 69#include <dev/virtio/block/virtio_blk.h> 70#include <dev/virtio/virtio_ids.h> 71#include <dev/virtio/virtio_config.h> 72#include <dev/virtio/virtio_ring.h> 73 74#include "pio_if.h" 75 76#define DPRINTF(fmt, ...) 77 78/* We use indirect descriptors */ 79#define NUM_DESCS 1 80#define NUM_QUEUES 1 81 82#define VTBLK_BLK_ID_BYTES 20 83#define VTBLK_MAXSEGS 256 84 85struct beri_vtblk_softc { 86 struct resource *res[1]; 87 bus_space_tag_t bst; 88 bus_space_handle_t bsh; 89 struct cdev *cdev; 90 device_t dev; 91 int opened; 92 device_t pio_recv; 93 device_t pio_send; 94 struct vqueue_info vs_queues[NUM_QUEUES]; 95 char ident[VTBLK_BLK_ID_BYTES]; 96 struct ucred *cred; 97 struct vnode *vnode; 98 struct thread *vtblk_ktd; 99 struct sx sc_mtx; 100 int beri_mem_offset; 101 struct md_ioctl *mdio; 102 struct virtio_blk_config *cfg; 103}; 104 105static struct resource_spec beri_spec[] = { 106 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 107 { -1, 0 } 108}; 109 110static int 111vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov, 112 int cnt, int offset, int operation, int iolen) 113{ 114 struct vnode *vp; 115 struct mount *mp; 116 struct uio auio; 117 int error; 118 119 bzero(&auio, sizeof(auio)); 120 121 vp = sc->vnode; 122 123 KASSERT(vp != NULL, ("file not opened")); 124 125 auio.uio_iov = iov; 126 auio.uio_iovcnt = cnt; 127 auio.uio_offset = offset; 128 auio.uio_segflg = UIO_SYSSPACE; 129 auio.uio_rw = operation; 130 auio.uio_resid = iolen; 131 auio.uio_td = curthread; 132 133 if (operation == 0) { 134 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 135 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); 136 VOP_UNLOCK(vp); 137 } else { 138 (void) vn_start_write(vp, &mp, V_WAIT); 139 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 140 error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred); 141 VOP_UNLOCK(vp); 142 vn_finished_write(mp); 143 } 144 145 return (error); 146} 147 148static void 149vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq) 150{ 151 struct iovec iov[VTBLK_MAXSEGS + 2]; 152 uint16_t flags[VTBLK_MAXSEGS + 2]; 153 struct virtio_blk_outhdr *vbh; 154 struct iovec *tiov; 155 uint8_t *status; 156 off_t offset; 157 int iolen; 158 int type; 159 int i, n; 160 int err; 161 162 n = vq_getchain(sc->beri_mem_offset, vq, iov, 163 VTBLK_MAXSEGS + 2, flags); 164 KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2, 165 ("wrong n value %d", n)); 166 167 tiov = getcopy(iov, n); 168 vbh = iov[0].iov_base; 169 170 status = iov[n-1].iov_base; 171 KASSERT(iov[n-1].iov_len == 1, 172 ("iov_len == %d", iov[n-1].iov_len)); 173 174 type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER; 175 offset = be64toh(vbh->sector) * DEV_BSIZE; 176 177 iolen = 0; 178 for (i = 1; i < (n-1); i++) { 179 iolen += iov[i].iov_len; 180 } 181 182 switch (type) { 183 case VIRTIO_BLK_T_OUT: 184 case VIRTIO_BLK_T_IN: 185 err = vtblk_rdwr(sc, tiov + 1, i - 1, 186 offset, type, iolen); 187 break; 188 case VIRTIO_BLK_T_GET_ID: 189 /* Assume a single buffer */ 190 strncpy(iov[1].iov_base, sc->ident, 191 MIN(iov[1].iov_len, sizeof(sc->ident))); 192 err = 0; 193 break; 194 case VIRTIO_BLK_T_FLUSH: 195 /* Possible? */ 196 default: 197 err = -ENOSYS; 198 break; 199 } 200 201 if (err < 0) { 202 if (err == -ENOSYS) { 203 *status = VIRTIO_BLK_S_UNSUPP; 204 } else 205 *status = VIRTIO_BLK_S_IOERR; 206 } else 207 *status = VIRTIO_BLK_S_OK; 208 209 free(tiov, M_DEVBUF); 210 vq_relchain(vq, iov, n, 1); 211} 212 213static int 214close_file(struct beri_vtblk_softc *sc, struct thread *td) 215{ 216 int error; 217 218 if (sc->vnode != NULL) { 219 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 220 sc->vnode->v_vflag &= ~VV_MD; 221 VOP_UNLOCK(sc->vnode); 222 error = vn_close(sc->vnode, (FREAD|FWRITE), 223 sc->cred, td); 224 if (error != 0) 225 return (error); 226 sc->vnode = NULL; 227 } 228 229 if (sc->cred != NULL) 230 crfree(sc->cred); 231 232 return (0); 233} 234 235static int 236open_file(struct beri_vtblk_softc *sc, struct thread *td) 237{ 238 struct nameidata nd; 239 struct vattr vattr; 240 int error; 241 int flags; 242 243 flags = (FREAD | FWRITE); 244 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, 245 sc->mdio->md_file, td); 246 error = vn_open(&nd, &flags, 0, NULL); 247 if (error != 0) 248 return (error); 249 NDFREE(&nd, NDF_ONLY_PNBUF); 250 251 if (nd.ni_vp->v_type != VREG) { 252 return (EINVAL); 253 } 254 255 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); 256 if (error != 0) 257 return (error); 258 259 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { 260 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); 261 if (VN_IS_DOOMED(nd.ni_vp)) { 262 return (1); 263 } 264 } 265 nd.ni_vp->v_vflag |= VV_MD; 266 VOP_UNLOCK(nd.ni_vp); 267 268 sc->vnode = nd.ni_vp; 269 sc->cred = crhold(td->td_ucred); 270 271 return (0); 272} 273 274static int 275vtblk_notify(struct beri_vtblk_softc *sc) 276{ 277 struct vqueue_info *vq; 278 int queue; 279 int reg; 280 281 vq = &sc->vs_queues[0]; 282 if (!vq_ring_ready(vq)) 283 return (0); 284 285 if (!sc->opened) 286 return (0); 287 288 reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY); 289 queue = be16toh(reg); 290 291 KASSERT(queue == 0, ("we support single queue only")); 292 293 /* Process new descriptors */ 294 vq = &sc->vs_queues[queue]; 295 vq->vq_save_used = be16toh(vq->vq_used->idx); 296 while (vq_has_descs(vq)) 297 vtblk_proc(sc, vq); 298 299 /* Interrupt the other side */ 300 if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) { 301 reg = htobe32(VIRTIO_MMIO_INT_VRING); 302 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 303 PIO_SET(sc->pio_send, Q_INTR, 1); 304 } 305 306 return (0); 307} 308 309static int 310vq_init(struct beri_vtblk_softc *sc) 311{ 312 struct vqueue_info *vq; 313 uint8_t *base; 314 int size; 315 int reg; 316 int pfn; 317 318 vq = &sc->vs_queues[0]; 319 vq->vq_qsize = NUM_DESCS; 320 321 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 322 pfn = be32toh(reg); 323 vq->vq_pfn = pfn; 324 325 size = vring_size(vq->vq_qsize, VRING_ALIGN); 326 base = paddr_map(sc->beri_mem_offset, 327 (pfn << PAGE_SHIFT), size); 328 329 /* First pages are descriptors */ 330 vq->vq_desc = (struct vring_desc *)base; 331 base += vq->vq_qsize * sizeof(struct vring_desc); 332 333 /* Then avail ring */ 334 vq->vq_avail = (struct vring_avail *)base; 335 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 336 337 /* Then it's rounded up to the next page */ 338 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 339 340 /* And the last pages are the used ring */ 341 vq->vq_used = (struct vring_used *)base; 342 343 /* Mark queue as allocated, and start at 0 when we use it. */ 344 vq->vq_flags = VQ_ALLOC; 345 vq->vq_last_avail = 0; 346 347 return (0); 348} 349 350static void 351vtblk_thread(void *arg) 352{ 353 struct beri_vtblk_softc *sc; 354 int err; 355 356 sc = arg; 357 358 sx_xlock(&sc->sc_mtx); 359 for (;;) { 360 err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz); 361 vtblk_notify(sc); 362 } 363 sx_xunlock(&sc->sc_mtx); 364 365 kthread_exit(); 366} 367 368static int 369backend_info(struct beri_vtblk_softc *sc) 370{ 371 struct virtio_blk_config *cfg; 372 uint32_t *s; 373 int reg; 374 int i; 375 376 /* Specify that we provide block device */ 377 reg = htobe32(VIRTIO_ID_BLOCK); 378 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 379 380 /* Queue size */ 381 reg = htobe32(NUM_DESCS); 382 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 383 384 /* Our features */ 385 reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC 386 | VIRTIO_BLK_F_BLK_SIZE 387 | VIRTIO_BLK_F_SEG_MAX); 388 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 389 390 cfg = sc->cfg; 391 cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE); 392 cfg->size_max = 0; /* not negotiated */ 393 cfg->seg_max = htobe32(VTBLK_MAXSEGS); 394 cfg->blk_size = htobe32(DEV_BSIZE); 395 396 s = (uint32_t *)cfg; 397 398 for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) { 399 WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s); 400 s+=1; 401 } 402 403 strncpy(sc->ident, "Virtio block backend", sizeof(sc->ident)); 404 405 return (0); 406} 407 408static void 409vtblk_intr(void *arg) 410{ 411 struct beri_vtblk_softc *sc; 412 int pending; 413 int reg; 414 415 sc = arg; 416 417 reg = PIO_READ(sc->pio_recv); 418 419 /* Ack */ 420 PIO_SET(sc->pio_recv, reg, 0); 421 422 pending = htobe32(reg); 423 424 if (pending & Q_PFN) { 425 vq_init(sc); 426 } 427 428 if (pending & Q_NOTIFY) { 429 wakeup(sc); 430 } 431} 432 433static int 434beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 435 int flags, struct thread *td) 436{ 437 struct beri_vtblk_softc *sc; 438 int err; 439 440 sc = dev->si_drv1; 441 442 switch (cmd) { 443 case MDIOCATTACH: 444 /* take file as argument */ 445 if (sc->vnode != NULL) { 446 /* Already opened */ 447 return (1); 448 } 449 sc->mdio = (struct md_ioctl *)addr; 450 backend_info(sc); 451 DPRINTF("opening file, td 0x%08x\n", (int)td); 452 err = open_file(sc, td); 453 if (err) 454 return (err); 455 PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc); 456 sc->opened = 1; 457 break; 458 case MDIOCDETACH: 459 if (sc->vnode == NULL) { 460 /* File not opened */ 461 return (1); 462 } 463 sc->opened = 0; 464 DPRINTF("closing file, td 0x%08x\n", (int)td); 465 err = close_file(sc, td); 466 if (err) 467 return (err); 468 PIO_TEARDOWN_IRQ(sc->pio_recv); 469 break; 470 default: 471 break; 472 } 473 474 return (0); 475} 476 477static struct cdevsw beri_cdevsw = { 478 .d_version = D_VERSION, 479 .d_ioctl = beri_ioctl, 480 .d_name = "virtio block backend", 481}; 482 483static int 484beri_vtblk_probe(device_t dev) 485{ 486 487 if (!ofw_bus_status_okay(dev)) 488 return (ENXIO); 489 490 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk")) 491 return (ENXIO); 492 493 device_set_desc(dev, "SRI-Cambridge BERI block"); 494 return (BUS_PROBE_DEFAULT); 495} 496 497static int 498beri_vtblk_attach(device_t dev) 499{ 500 struct beri_vtblk_softc *sc; 501 int error; 502 503 sc = device_get_softc(dev); 504 sc->dev = dev; 505 506 if (bus_alloc_resources(dev, beri_spec, sc->res)) { 507 device_printf(dev, "could not allocate resources\n"); 508 return (ENXIO); 509 } 510 511 /* Memory interface */ 512 sc->bst = rman_get_bustag(sc->res[0]); 513 sc->bsh = rman_get_bushandle(sc->res[0]); 514 515 sc->cfg = malloc(sizeof(struct virtio_blk_config), 516 M_DEVBUF, M_NOWAIT|M_ZERO); 517 518 sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev)); 519 520 error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd, 521 0, 0, "beri_virtio_block"); 522 if (error) { 523 device_printf(dev, "cannot create kthread\n"); 524 return (ENXIO); 525 } 526 527 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 528 return (ENXIO); 529 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 530 return (ENXIO); 531 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 532 return (ENXIO); 533 534 sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL, 535 S_IRWXU, "beri_vtblk"); 536 if (sc->cdev == NULL) { 537 device_printf(dev, "Failed to create character device.\n"); 538 return (ENXIO); 539 } 540 541 sc->cdev->si_drv1 = sc; 542 return (0); 543} 544 545static device_method_t beri_vtblk_methods[] = { 546 DEVMETHOD(device_probe, beri_vtblk_probe), 547 DEVMETHOD(device_attach, beri_vtblk_attach), 548 { 0, 0 } 549}; 550 551static driver_t beri_vtblk_driver = { 552 "beri_vtblk", 553 beri_vtblk_methods, 554 sizeof(struct beri_vtblk_softc), 555}; 556 557static devclass_t beri_vtblk_devclass; 558 559DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver, 560 beri_vtblk_devclass, 0, 0); 561