1/*- 2 * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27/* Driver for VirtIO SCSI devices. */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/scsi/virtio_scsi.c 315813 2017-03-23 06:41:13Z mav $"); 31 32#include <sys/param.h> 33#include <sys/systm.h> 34#include <sys/kernel.h> 35#include <sys/kthread.h> 36#include <sys/malloc.h> 37#include <sys/module.h> 38#include <sys/sglist.h> 39#include <sys/sysctl.h> 40#include <sys/lock.h> 41#include <sys/mutex.h> 42#include <sys/callout.h> 43#include <sys/queue.h> 44#include <sys/sbuf.h> 45 46#include <machine/stdarg.h> 47 48#include <machine/bus.h> 49#include <machine/resource.h> 50#include <sys/bus.h> 51#include <sys/rman.h> 52 53#include <cam/cam.h> 54#include <cam/cam_ccb.h> 55#include <cam/cam_sim.h> 56#include <cam/cam_periph.h> 57#include <cam/cam_xpt_sim.h> 58#include <cam/cam_debug.h> 59#include <cam/scsi/scsi_all.h> 60#include <cam/scsi/scsi_message.h> 61 62#include <dev/virtio/virtio.h> 63#include <dev/virtio/virtqueue.h> 64#include <dev/virtio/scsi/virtio_scsi.h> 65#include <dev/virtio/scsi/virtio_scsivar.h> 66 67#include "virtio_if.h" 68 69static int vtscsi_modevent(module_t, int, void *); 70 71static int vtscsi_probe(device_t); 72static int vtscsi_attach(device_t); 73static int vtscsi_detach(device_t); 74static int vtscsi_suspend(device_t); 75static int vtscsi_resume(device_t); 76 77static void vtscsi_negotiate_features(struct vtscsi_softc *); 78static void vtscsi_read_config(struct vtscsi_softc *, 79 struct virtio_scsi_config *); 80static int vtscsi_maximum_segments(struct vtscsi_softc *, int); 81static int vtscsi_alloc_virtqueues(struct vtscsi_softc *); 82static void vtscsi_write_device_config(struct vtscsi_softc *); 83static int vtscsi_reinit(struct vtscsi_softc *); 84 85static int vtscsi_alloc_cam(struct vtscsi_softc *); 86static int vtscsi_register_cam(struct vtscsi_softc *); 87static void vtscsi_free_cam(struct vtscsi_softc *); 88static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *); 89static int vtscsi_register_async(struct vtscsi_softc *); 90static void vtscsi_deregister_async(struct vtscsi_softc *); 91static void vtscsi_cam_action(struct cam_sim *, union ccb *); 92static void vtscsi_cam_poll(struct cam_sim *); 93 94static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *, 95 union ccb *); 96static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *, 97 union ccb *); 98static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *); 99static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *); 100static void vtscsi_cam_abort(struct vtscsi_softc *, union ccb *); 101static void vtscsi_cam_path_inquiry(struct vtscsi_softc *, 102 struct cam_sim *, union ccb *); 103 104static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *, 105 struct sglist *, struct ccb_scsiio *); 106static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *, 107 struct vtscsi_request *, int *, int *); 108static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *, 109 struct vtscsi_request *); 110static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *); 111static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *, 112 struct vtscsi_request *); 113static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *, 114 struct vtscsi_request *); 115static void vtscsi_timedout_scsi_cmd(void *); 116static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *); 117static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *, 118 struct ccb_scsiio *, struct virtio_scsi_cmd_resp *); 119static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *, 120 struct vtscsi_request *); 121 122static void vtscsi_poll_ctrl_req(struct vtscsi_softc *, 123 struct vtscsi_request *); 124static int vtscsi_execute_ctrl_req(struct vtscsi_softc *, 125 struct vtscsi_request *, struct sglist *, int, int, int); 126static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c, 127 struct vtscsi_request *); 128static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *, 129 struct vtscsi_request *); 130static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *, 131 struct vtscsi_request *); 132 133static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *); 134static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []); 135static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *, 136 struct virtio_scsi_cmd_req *); 137static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t, 138 uintptr_t, struct virtio_scsi_ctrl_tmf_req *); 139 140static void vtscsi_freeze_simq(struct vtscsi_softc *, int); 141static int vtscsi_thaw_simq(struct vtscsi_softc *, int); 142 143static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t, 144 lun_id_t); 145static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t, 146 lun_id_t); 147static void vtscsi_execute_rescan_bus(struct vtscsi_softc *); 148 149static void vtscsi_handle_event(struct vtscsi_softc *, 150 struct virtio_scsi_event *); 151static int vtscsi_enqueue_event_buf(struct vtscsi_softc *, 152 struct virtio_scsi_event *); 153static int vtscsi_init_event_vq(struct vtscsi_softc *); 154static void vtscsi_reinit_event_vq(struct vtscsi_softc *); 155static void vtscsi_drain_event_vq(struct vtscsi_softc *); 156 157static void vtscsi_complete_vqs_locked(struct vtscsi_softc *); 158static void vtscsi_complete_vqs(struct vtscsi_softc *); 159static void vtscsi_drain_vqs(struct vtscsi_softc *); 160static void vtscsi_cancel_request(struct vtscsi_softc *, 161 struct vtscsi_request *); 162static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *); 163static void vtscsi_stop(struct vtscsi_softc *); 164static int vtscsi_reset_bus(struct vtscsi_softc *); 165 166static void vtscsi_init_request(struct vtscsi_softc *, 167 struct vtscsi_request *); 168static int vtscsi_alloc_requests(struct vtscsi_softc *); 169static void vtscsi_free_requests(struct vtscsi_softc *); 170static void vtscsi_enqueue_request(struct vtscsi_softc *, 171 struct vtscsi_request *); 172static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *); 173 174static void vtscsi_complete_request(struct vtscsi_request *); 175static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *); 176 177static void vtscsi_control_vq_intr(void *); 178static void vtscsi_event_vq_intr(void *); 179static void vtscsi_request_vq_intr(void *); 180static void vtscsi_disable_vqs_intr(struct vtscsi_softc *); 181static void vtscsi_enable_vqs_intr(struct vtscsi_softc *); 182 183static void vtscsi_get_tunables(struct vtscsi_softc *); 184static void vtscsi_add_sysctl(struct vtscsi_softc *); 185 186static void vtscsi_printf_req(struct vtscsi_request *, const char *, 187 const char *, ...); 188 189/* Global tunables. */ 190/* 191 * The current QEMU VirtIO SCSI implementation does not cancel in-flight 192 * IO during virtio_stop(). So in-flight requests still complete after the 193 * device reset. We would have to wait for all the in-flight IO to complete, 194 * which defeats the typical purpose of a bus reset. We could simulate the 195 * bus reset with either I_T_NEXUS_RESET of all the targets, or with 196 * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the 197 * control virtqueue). But this isn't very useful if things really go off 198 * the rails, so default to disabled for now. 199 */ 200static int vtscsi_bus_reset_disable = 1; 201TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable); 202 203static struct virtio_feature_desc vtscsi_feature_desc[] = { 204 { VIRTIO_SCSI_F_INOUT, "InOut" }, 205 { VIRTIO_SCSI_F_HOTPLUG, "Hotplug" }, 206 207 { 0, NULL } 208}; 209 210static device_method_t vtscsi_methods[] = { 211 /* Device methods. */ 212 DEVMETHOD(device_probe, vtscsi_probe), 213 DEVMETHOD(device_attach, vtscsi_attach), 214 DEVMETHOD(device_detach, vtscsi_detach), 215 DEVMETHOD(device_suspend, vtscsi_suspend), 216 DEVMETHOD(device_resume, vtscsi_resume), 217 218 DEVMETHOD_END 219}; 220 221static driver_t vtscsi_driver = { 222 "vtscsi", 223 vtscsi_methods, 224 sizeof(struct vtscsi_softc) 225}; 226static devclass_t vtscsi_devclass; 227 228DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass, 229 vtscsi_modevent, 0); 230MODULE_VERSION(virtio_scsi, 1); 231MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1); 232MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1); 233 234static int 235vtscsi_modevent(module_t mod, int type, void *unused) 236{ 237 int error; 238 239 switch (type) { 240 case MOD_LOAD: 241 case MOD_QUIESCE: 242 case MOD_UNLOAD: 243 case MOD_SHUTDOWN: 244 error = 0; 245 break; 246 default: 247 error = EOPNOTSUPP; 248 break; 249 } 250 251 return (error); 252} 253 254static int 255vtscsi_probe(device_t dev) 256{ 257 258 if (virtio_get_device_type(dev) != VIRTIO_ID_SCSI) 259 return (ENXIO); 260 261 device_set_desc(dev, "VirtIO SCSI Adapter"); 262 263 return (BUS_PROBE_DEFAULT); 264} 265 266static int 267vtscsi_attach(device_t dev) 268{ 269 struct vtscsi_softc *sc; 270 struct virtio_scsi_config scsicfg; 271 int error; 272 273 sc = device_get_softc(dev); 274 sc->vtscsi_dev = dev; 275 276 VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev)); 277 TAILQ_INIT(&sc->vtscsi_req_free); 278 279 vtscsi_get_tunables(sc); 280 vtscsi_add_sysctl(sc); 281 282 virtio_set_feature_desc(dev, vtscsi_feature_desc); 283 vtscsi_negotiate_features(sc); 284 285 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 286 sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; 287 if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) 288 sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; 289 if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) 290 sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; 291 292 vtscsi_read_config(sc, &scsicfg); 293 294 sc->vtscsi_max_channel = scsicfg.max_channel; 295 sc->vtscsi_max_target = scsicfg.max_target; 296 sc->vtscsi_max_lun = scsicfg.max_lun; 297 sc->vtscsi_event_buf_size = scsicfg.event_info_size; 298 299 vtscsi_write_device_config(sc); 300 301 sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max); 302 sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT); 303 if (sc->vtscsi_sglist == NULL) { 304 error = ENOMEM; 305 device_printf(dev, "cannot allocate sglist\n"); 306 goto fail; 307 } 308 309 error = vtscsi_alloc_virtqueues(sc); 310 if (error) { 311 device_printf(dev, "cannot allocate virtqueues\n"); 312 goto fail; 313 } 314 315 error = vtscsi_init_event_vq(sc); 316 if (error) { 317 device_printf(dev, "cannot populate the eventvq\n"); 318 goto fail; 319 } 320 321 error = vtscsi_alloc_requests(sc); 322 if (error) { 323 device_printf(dev, "cannot allocate requests\n"); 324 goto fail; 325 } 326 327 error = vtscsi_alloc_cam(sc); 328 if (error) { 329 device_printf(dev, "cannot allocate CAM structures\n"); 330 goto fail; 331 } 332 333 error = virtio_setup_intr(dev, INTR_TYPE_CAM); 334 if (error) { 335 device_printf(dev, "cannot setup virtqueue interrupts\n"); 336 goto fail; 337 } 338 339 vtscsi_enable_vqs_intr(sc); 340 341 /* 342 * Register with CAM after interrupts are enabled so we will get 343 * notified of the probe responses. 344 */ 345 error = vtscsi_register_cam(sc); 346 if (error) { 347 device_printf(dev, "cannot register with CAM\n"); 348 goto fail; 349 } 350 351fail: 352 if (error) 353 vtscsi_detach(dev); 354 355 return (error); 356} 357 358static int 359vtscsi_detach(device_t dev) 360{ 361 struct vtscsi_softc *sc; 362 363 sc = device_get_softc(dev); 364 365 VTSCSI_LOCK(sc); 366 sc->vtscsi_flags |= VTSCSI_FLAG_DETACH; 367 if (device_is_attached(dev)) 368 vtscsi_stop(sc); 369 VTSCSI_UNLOCK(sc); 370 371 vtscsi_complete_vqs(sc); 372 vtscsi_drain_vqs(sc); 373 374 vtscsi_free_cam(sc); 375 vtscsi_free_requests(sc); 376 377 if (sc->vtscsi_sglist != NULL) { 378 sglist_free(sc->vtscsi_sglist); 379 sc->vtscsi_sglist = NULL; 380 } 381 382 VTSCSI_LOCK_DESTROY(sc); 383 384 return (0); 385} 386 387static int 388vtscsi_suspend(device_t dev) 389{ 390 391 return (0); 392} 393 394static int 395vtscsi_resume(device_t dev) 396{ 397 398 return (0); 399} 400 401static void 402vtscsi_negotiate_features(struct vtscsi_softc *sc) 403{ 404 device_t dev; 405 uint64_t features; 406 407 dev = sc->vtscsi_dev; 408 features = virtio_negotiate_features(dev, VTSCSI_FEATURES); 409 sc->vtscsi_features = features; 410} 411 412#define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \ 413 virtio_read_device_config(_dev, \ 414 offsetof(struct virtio_scsi_config, _field), \ 415 &(_cfg)->_field, sizeof((_cfg)->_field)) \ 416 417static void 418vtscsi_read_config(struct vtscsi_softc *sc, 419 struct virtio_scsi_config *scsicfg) 420{ 421 device_t dev; 422 423 dev = sc->vtscsi_dev; 424 425 bzero(scsicfg, sizeof(struct virtio_scsi_config)); 426 427 VTSCSI_GET_CONFIG(dev, num_queues, scsicfg); 428 VTSCSI_GET_CONFIG(dev, seg_max, scsicfg); 429 VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg); 430 VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg); 431 VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg); 432 VTSCSI_GET_CONFIG(dev, sense_size, scsicfg); 433 VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg); 434 VTSCSI_GET_CONFIG(dev, max_channel, scsicfg); 435 VTSCSI_GET_CONFIG(dev, max_target, scsicfg); 436 VTSCSI_GET_CONFIG(dev, max_lun, scsicfg); 437} 438 439#undef VTSCSI_GET_CONFIG 440 441static int 442vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max) 443{ 444 int nsegs; 445 446 nsegs = VTSCSI_MIN_SEGMENTS; 447 448 if (seg_max > 0) { 449 nsegs += MIN(seg_max, MAXPHYS / PAGE_SIZE + 1); 450 if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) 451 nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); 452 } else 453 nsegs += 1; 454 455 return (nsegs); 456} 457 458static int 459vtscsi_alloc_virtqueues(struct vtscsi_softc *sc) 460{ 461 device_t dev; 462 struct vq_alloc_info vq_info[3]; 463 int nvqs; 464 465 dev = sc->vtscsi_dev; 466 nvqs = 3; 467 468 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc, 469 &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev)); 470 471 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc, 472 &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev)); 473 474 VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs, 475 vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq, 476 "%s request", device_get_nameunit(dev)); 477 478 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 479} 480 481static void 482vtscsi_write_device_config(struct vtscsi_softc *sc) 483{ 484 485 virtio_write_dev_config_4(sc->vtscsi_dev, 486 offsetof(struct virtio_scsi_config, sense_size), 487 VIRTIO_SCSI_SENSE_SIZE); 488 489 /* 490 * This is the size in the virtio_scsi_cmd_req structure. Note 491 * this value (32) is larger than the maximum CAM CDB size (16). 492 */ 493 virtio_write_dev_config_4(sc->vtscsi_dev, 494 offsetof(struct virtio_scsi_config, cdb_size), 495 VIRTIO_SCSI_CDB_SIZE); 496} 497 498static int 499vtscsi_reinit(struct vtscsi_softc *sc) 500{ 501 device_t dev; 502 int error; 503 504 dev = sc->vtscsi_dev; 505 506 error = virtio_reinit(dev, sc->vtscsi_features); 507 if (error == 0) { 508 vtscsi_write_device_config(sc); 509 vtscsi_reinit_event_vq(sc); 510 virtio_reinit_complete(dev); 511 512 vtscsi_enable_vqs_intr(sc); 513 } 514 515 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error); 516 517 return (error); 518} 519 520static int 521vtscsi_alloc_cam(struct vtscsi_softc *sc) 522{ 523 device_t dev; 524 struct cam_devq *devq; 525 int openings; 526 527 dev = sc->vtscsi_dev; 528 openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS; 529 530 devq = cam_simq_alloc(openings); 531 if (devq == NULL) { 532 device_printf(dev, "cannot allocate SIM queue\n"); 533 return (ENOMEM); 534 } 535 536 sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll, 537 "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1, 538 openings, devq); 539 if (sc->vtscsi_sim == NULL) { 540 cam_simq_free(devq); 541 device_printf(dev, "cannot allocate SIM\n"); 542 return (ENOMEM); 543 } 544 545 return (0); 546} 547 548static int 549vtscsi_register_cam(struct vtscsi_softc *sc) 550{ 551 device_t dev; 552 int registered, error; 553 554 dev = sc->vtscsi_dev; 555 registered = 0; 556 557 VTSCSI_LOCK(sc); 558 559 if (xpt_bus_register(sc->vtscsi_sim, dev, 0) != CAM_SUCCESS) { 560 error = ENOMEM; 561 device_printf(dev, "cannot register XPT bus\n"); 562 goto fail; 563 } 564 565 registered = 1; 566 567 if (xpt_create_path(&sc->vtscsi_path, NULL, 568 cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD, 569 CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 570 error = ENOMEM; 571 device_printf(dev, "cannot create bus path\n"); 572 goto fail; 573 } 574 575 if (vtscsi_register_async(sc) != CAM_REQ_CMP) { 576 error = EIO; 577 device_printf(dev, "cannot register async callback\n"); 578 goto fail; 579 } 580 581 VTSCSI_UNLOCK(sc); 582 583 return (0); 584 585fail: 586 if (sc->vtscsi_path != NULL) { 587 xpt_free_path(sc->vtscsi_path); 588 sc->vtscsi_path = NULL; 589 } 590 591 if (registered != 0) 592 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 593 594 VTSCSI_UNLOCK(sc); 595 596 return (error); 597} 598 599static void 600vtscsi_free_cam(struct vtscsi_softc *sc) 601{ 602 603 VTSCSI_LOCK(sc); 604 605 if (sc->vtscsi_path != NULL) { 606 vtscsi_deregister_async(sc); 607 608 xpt_free_path(sc->vtscsi_path); 609 sc->vtscsi_path = NULL; 610 611 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 612 } 613 614 if (sc->vtscsi_sim != NULL) { 615 cam_sim_free(sc->vtscsi_sim, 1); 616 sc->vtscsi_sim = NULL; 617 } 618 619 VTSCSI_UNLOCK(sc); 620} 621 622static void 623vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg) 624{ 625 struct cam_sim *sim; 626 struct vtscsi_softc *sc; 627 628 sim = cb_arg; 629 sc = cam_sim_softc(sim); 630 631 vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code); 632 633 /* 634 * TODO Once QEMU supports event reporting, we should 635 * (un)subscribe to events here. 636 */ 637 switch (code) { 638 case AC_FOUND_DEVICE: 639 break; 640 case AC_LOST_DEVICE: 641 break; 642 } 643} 644 645static int 646vtscsi_register_async(struct vtscsi_softc *sc) 647{ 648 struct ccb_setasync csa; 649 650 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 651 csa.ccb_h.func_code = XPT_SASYNC_CB; 652 csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE; 653 csa.callback = vtscsi_cam_async; 654 csa.callback_arg = sc->vtscsi_sim; 655 656 xpt_action((union ccb *) &csa); 657 658 return (csa.ccb_h.status); 659} 660 661static void 662vtscsi_deregister_async(struct vtscsi_softc *sc) 663{ 664 struct ccb_setasync csa; 665 666 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 667 csa.ccb_h.func_code = XPT_SASYNC_CB; 668 csa.event_enable = 0; 669 csa.callback = vtscsi_cam_async; 670 csa.callback_arg = sc->vtscsi_sim; 671 672 xpt_action((union ccb *) &csa); 673} 674 675static void 676vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb) 677{ 678 struct vtscsi_softc *sc; 679 struct ccb_hdr *ccbh; 680 681 sc = cam_sim_softc(sim); 682 ccbh = &ccb->ccb_h; 683 684 VTSCSI_LOCK_OWNED(sc); 685 686 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) { 687 /* 688 * The VTSCSI_MTX is briefly dropped between setting 689 * VTSCSI_FLAG_DETACH and deregistering with CAM, so 690 * drop any CCBs that come in during that window. 691 */ 692 ccbh->status = CAM_NO_HBA; 693 xpt_done(ccb); 694 return; 695 } 696 697 switch (ccbh->func_code) { 698 case XPT_SCSI_IO: 699 vtscsi_cam_scsi_io(sc, sim, ccb); 700 break; 701 702 case XPT_SET_TRAN_SETTINGS: 703 ccbh->status = CAM_FUNC_NOTAVAIL; 704 xpt_done(ccb); 705 break; 706 707 case XPT_GET_TRAN_SETTINGS: 708 vtscsi_cam_get_tran_settings(sc, ccb); 709 break; 710 711 case XPT_RESET_BUS: 712 vtscsi_cam_reset_bus(sc, ccb); 713 break; 714 715 case XPT_RESET_DEV: 716 vtscsi_cam_reset_dev(sc, ccb); 717 break; 718 719 case XPT_ABORT: 720 vtscsi_cam_abort(sc, ccb); 721 break; 722 723 case XPT_CALC_GEOMETRY: 724 cam_calc_geometry(&ccb->ccg, 1); 725 xpt_done(ccb); 726 break; 727 728 case XPT_PATH_INQ: 729 vtscsi_cam_path_inquiry(sc, sim, ccb); 730 break; 731 732 default: 733 vtscsi_dprintf(sc, VTSCSI_ERROR, 734 "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code); 735 736 ccbh->status = CAM_REQ_INVALID; 737 xpt_done(ccb); 738 break; 739 } 740} 741 742static void 743vtscsi_cam_poll(struct cam_sim *sim) 744{ 745 struct vtscsi_softc *sc; 746 747 sc = cam_sim_softc(sim); 748 749 vtscsi_complete_vqs_locked(sc); 750} 751 752static void 753vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim, 754 union ccb *ccb) 755{ 756 struct ccb_hdr *ccbh; 757 struct ccb_scsiio *csio; 758 int error; 759 760 ccbh = &ccb->ccb_h; 761 csio = &ccb->csio; 762 763 if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) { 764 error = EINVAL; 765 ccbh->status = CAM_REQ_INVALID; 766 goto done; 767 } 768 769 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH && 770 (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) { 771 error = EINVAL; 772 ccbh->status = CAM_REQ_INVALID; 773 goto done; 774 } 775 776 error = vtscsi_start_scsi_cmd(sc, ccb); 777 778done: 779 if (error) { 780 vtscsi_dprintf(sc, VTSCSI_ERROR, 781 "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status); 782 xpt_done(ccb); 783 } 784} 785 786static void 787vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb) 788{ 789 struct ccb_trans_settings *cts; 790 struct ccb_trans_settings_scsi *scsi; 791 792 cts = &ccb->cts; 793 scsi = &cts->proto_specific.scsi; 794 795 cts->protocol = PROTO_SCSI; 796 cts->protocol_version = SCSI_REV_SPC3; 797 cts->transport = XPORT_SAS; 798 cts->transport_version = 0; 799 800 scsi->valid = CTS_SCSI_VALID_TQ; 801 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; 802 803 ccb->ccb_h.status = CAM_REQ_CMP; 804 xpt_done(ccb); 805} 806 807static void 808vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb) 809{ 810 int error; 811 812 error = vtscsi_reset_bus(sc); 813 if (error == 0) 814 ccb->ccb_h.status = CAM_REQ_CMP; 815 else 816 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 817 818 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n", 819 error, ccb, ccb->ccb_h.status); 820 821 xpt_done(ccb); 822} 823 824static void 825vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb) 826{ 827 struct ccb_hdr *ccbh; 828 struct vtscsi_request *req; 829 int error; 830 831 ccbh = &ccb->ccb_h; 832 833 req = vtscsi_dequeue_request(sc); 834 if (req == NULL) { 835 error = EAGAIN; 836 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 837 goto fail; 838 } 839 840 req->vsr_ccb = ccb; 841 842 error = vtscsi_execute_reset_dev_cmd(sc, req); 843 if (error == 0) 844 return; 845 846 vtscsi_enqueue_request(sc, req); 847 848fail: 849 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 850 error, req, ccb); 851 852 if (error == EAGAIN) 853 ccbh->status = CAM_RESRC_UNAVAIL; 854 else 855 ccbh->status = CAM_REQ_CMP_ERR; 856 857 xpt_done(ccb); 858} 859 860static void 861vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb) 862{ 863 struct vtscsi_request *req; 864 struct ccb_hdr *ccbh; 865 int error; 866 867 ccbh = &ccb->ccb_h; 868 869 req = vtscsi_dequeue_request(sc); 870 if (req == NULL) { 871 error = EAGAIN; 872 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 873 goto fail; 874 } 875 876 req->vsr_ccb = ccb; 877 878 error = vtscsi_execute_abort_task_cmd(sc, req); 879 if (error == 0) 880 return; 881 882 vtscsi_enqueue_request(sc, req); 883 884fail: 885 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 886 error, req, ccb); 887 888 if (error == EAGAIN) 889 ccbh->status = CAM_RESRC_UNAVAIL; 890 else 891 ccbh->status = CAM_REQ_CMP_ERR; 892 893 xpt_done(ccb); 894} 895 896static void 897vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim, 898 union ccb *ccb) 899{ 900 device_t dev; 901 struct ccb_pathinq *cpi; 902 903 dev = sc->vtscsi_dev; 904 cpi = &ccb->cpi; 905 906 vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb); 907 908 cpi->version_num = 1; 909 cpi->hba_inquiry = PI_TAG_ABLE; 910 cpi->target_sprt = 0; 911 cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; 912 if (vtscsi_bus_reset_disable != 0) 913 cpi->hba_misc |= PIM_NOBUSRESET; 914 cpi->hba_eng_cnt = 0; 915 916 cpi->max_target = sc->vtscsi_max_target; 917 cpi->max_lun = sc->vtscsi_max_lun; 918 cpi->initiator_id = VTSCSI_INITIATOR_ID; 919 920 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 921 strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN); 922 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 923 924 cpi->unit_number = cam_sim_unit(sim); 925 cpi->bus_id = cam_sim_bus(sim); 926 927 cpi->base_transfer_speed = 300000; 928 929 cpi->protocol = PROTO_SCSI; 930 cpi->protocol_version = SCSI_REV_SPC3; 931 cpi->transport = XPORT_SAS; 932 cpi->transport_version = 0; 933 934 cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) * 935 PAGE_SIZE; 936 937 cpi->hba_vendor = virtio_get_vendor(dev); 938 cpi->hba_device = virtio_get_device(dev); 939 cpi->hba_subvendor = virtio_get_subvendor(dev); 940 cpi->hba_subdevice = virtio_get_subdevice(dev); 941 942 ccb->ccb_h.status = CAM_REQ_CMP; 943 xpt_done(ccb); 944} 945 946static int 947vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg, 948 struct ccb_scsiio *csio) 949{ 950 struct ccb_hdr *ccbh; 951 struct bus_dma_segment *dseg; 952 int i, error; 953 954 ccbh = &csio->ccb_h; 955 error = 0; 956 957 switch ((ccbh->flags & CAM_DATA_MASK)) { 958 case CAM_DATA_VADDR: 959 error = sglist_append(sg, csio->data_ptr, csio->dxfer_len); 960 break; 961 case CAM_DATA_PADDR: 962 error = sglist_append_phys(sg, 963 (vm_paddr_t)(vm_offset_t) csio->data_ptr, csio->dxfer_len); 964 break; 965 case CAM_DATA_SG: 966 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 967 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 968 error = sglist_append(sg, 969 (void *)(vm_offset_t) dseg->ds_addr, dseg->ds_len); 970 } 971 break; 972 case CAM_DATA_SG_PADDR: 973 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 974 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 975 error = sglist_append_phys(sg, 976 (vm_paddr_t) dseg->ds_addr, dseg->ds_len); 977 } 978 break; 979 case CAM_DATA_BIO: 980 error = sglist_append_bio(sg, (struct bio *) csio->data_ptr); 981 break; 982 default: 983 error = EINVAL; 984 break; 985 } 986 987 return (error); 988} 989 990static int 991vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req, 992 int *readable, int *writable) 993{ 994 struct sglist *sg; 995 struct ccb_hdr *ccbh; 996 struct ccb_scsiio *csio; 997 struct virtio_scsi_cmd_req *cmd_req; 998 struct virtio_scsi_cmd_resp *cmd_resp; 999 int error; 1000 1001 sg = sc->vtscsi_sglist; 1002 csio = &req->vsr_ccb->csio; 1003 ccbh = &csio->ccb_h; 1004 cmd_req = &req->vsr_cmd_req; 1005 cmd_resp = &req->vsr_cmd_resp; 1006 1007 sglist_reset(sg); 1008 1009 sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req)); 1010 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) { 1011 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1012 /* At least one segment must be left for the response. */ 1013 if (error || sg->sg_nseg == sg->sg_maxseg) 1014 goto fail; 1015 } 1016 1017 *readable = sg->sg_nseg; 1018 1019 sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp)); 1020 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) { 1021 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1022 if (error) 1023 goto fail; 1024 } 1025 1026 *writable = sg->sg_nseg - *readable; 1027 1028 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d " 1029 "writable=%d\n", req, ccbh, *readable, *writable); 1030 1031 return (0); 1032 1033fail: 1034 /* 1035 * This should never happen unless maxio was incorrectly set. 1036 */ 1037 vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0); 1038 1039 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p " 1040 "nseg=%d maxseg=%d\n", 1041 error, req, ccbh, sg->sg_nseg, sg->sg_maxseg); 1042 1043 return (EFBIG); 1044} 1045 1046static int 1047vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1048{ 1049 struct sglist *sg; 1050 struct virtqueue *vq; 1051 struct ccb_scsiio *csio; 1052 struct ccb_hdr *ccbh; 1053 struct virtio_scsi_cmd_req *cmd_req; 1054 struct virtio_scsi_cmd_resp *cmd_resp; 1055 int readable, writable, error; 1056 1057 sg = sc->vtscsi_sglist; 1058 vq = sc->vtscsi_request_vq; 1059 csio = &req->vsr_ccb->csio; 1060 ccbh = &csio->ccb_h; 1061 cmd_req = &req->vsr_cmd_req; 1062 cmd_resp = &req->vsr_cmd_resp; 1063 1064 vtscsi_init_scsi_cmd_req(csio, cmd_req); 1065 1066 error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable); 1067 if (error) 1068 return (error); 1069 1070 req->vsr_complete = vtscsi_complete_scsi_cmd; 1071 cmd_resp->response = -1; 1072 1073 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1074 if (error) { 1075 vtscsi_dprintf(sc, VTSCSI_ERROR, 1076 "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh); 1077 1078 ccbh->status = CAM_REQUEUE_REQ; 1079 vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ); 1080 return (error); 1081 } 1082 1083 ccbh->status |= CAM_SIM_QUEUED; 1084 ccbh->ccbh_vtscsi_req = req; 1085 1086 virtqueue_notify(vq); 1087 1088 if (ccbh->timeout != CAM_TIME_INFINITY) { 1089 req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET; 1090 callout_reset_sbt(&req->vsr_callout, SBT_1MS * ccbh->timeout, 1091 0, vtscsi_timedout_scsi_cmd, req, 0); 1092 } 1093 1094 vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n", 1095 req, ccbh); 1096 1097 return (0); 1098} 1099 1100static int 1101vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb) 1102{ 1103 struct vtscsi_request *req; 1104 int error; 1105 1106 req = vtscsi_dequeue_request(sc); 1107 if (req == NULL) { 1108 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1109 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 1110 return (ENOBUFS); 1111 } 1112 1113 req->vsr_ccb = ccb; 1114 1115 error = vtscsi_execute_scsi_cmd(sc, req); 1116 if (error) 1117 vtscsi_enqueue_request(sc, req); 1118 1119 return (error); 1120} 1121 1122static void 1123vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1124 struct vtscsi_request *req) 1125{ 1126 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1127 struct vtscsi_request *to_req; 1128 uint8_t response; 1129 1130 tmf_resp = &req->vsr_tmf_resp; 1131 response = tmf_resp->response; 1132 to_req = req->vsr_timedout_req; 1133 1134 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n", 1135 req, to_req, response); 1136 1137 vtscsi_enqueue_request(sc, req); 1138 1139 /* 1140 * The timedout request could have completed between when the 1141 * abort task was sent and when the host processed it. 1142 */ 1143 if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT) 1144 return; 1145 1146 /* The timedout request was successfully aborted. */ 1147 if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) 1148 return; 1149 1150 /* Don't bother if the device is going away. */ 1151 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1152 return; 1153 1154 /* The timedout request will be aborted by the reset. */ 1155 if (sc->vtscsi_flags & VTSCSI_FLAG_RESET) 1156 return; 1157 1158 vtscsi_reset_bus(sc); 1159} 1160 1161static int 1162vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1163 struct vtscsi_request *to_req) 1164{ 1165 struct sglist *sg; 1166 struct ccb_hdr *to_ccbh; 1167 struct vtscsi_request *req; 1168 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1169 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1170 int error; 1171 1172 sg = sc->vtscsi_sglist; 1173 to_ccbh = &to_req->vsr_ccb->ccb_h; 1174 1175 req = vtscsi_dequeue_request(sc); 1176 if (req == NULL) { 1177 error = ENOBUFS; 1178 goto fail; 1179 } 1180 1181 tmf_req = &req->vsr_tmf_req; 1182 tmf_resp = &req->vsr_tmf_resp; 1183 1184 vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1185 (uintptr_t) to_ccbh, tmf_req); 1186 1187 sglist_reset(sg); 1188 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1189 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1190 1191 req->vsr_timedout_req = to_req; 1192 req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd; 1193 tmf_resp->response = -1; 1194 1195 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1196 VTSCSI_EXECUTE_ASYNC); 1197 if (error == 0) 1198 return (0); 1199 1200 vtscsi_enqueue_request(sc, req); 1201 1202fail: 1203 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p " 1204 "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh); 1205 1206 return (error); 1207} 1208 1209static void 1210vtscsi_timedout_scsi_cmd(void *xreq) 1211{ 1212 struct vtscsi_softc *sc; 1213 struct vtscsi_request *to_req; 1214 1215 to_req = xreq; 1216 sc = to_req->vsr_softc; 1217 1218 vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n", 1219 to_req, to_req->vsr_ccb, to_req->vsr_state); 1220 1221 /* Don't bother if the device is going away. */ 1222 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1223 return; 1224 1225 /* 1226 * Bail if the request is not in use. We likely raced when 1227 * stopping the callout handler or it has already been aborted. 1228 */ 1229 if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE || 1230 (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0) 1231 return; 1232 1233 /* 1234 * Complete the request queue in case the timedout request is 1235 * actually just pending. 1236 */ 1237 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1238 if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE) 1239 return; 1240 1241 sc->vtscsi_stats.scsi_cmd_timeouts++; 1242 to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT; 1243 1244 if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0) 1245 return; 1246 1247 vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n"); 1248 vtscsi_reset_bus(sc); 1249} 1250 1251static cam_status 1252vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp) 1253{ 1254 cam_status status; 1255 1256 switch (cmd_resp->response) { 1257 case VIRTIO_SCSI_S_OK: 1258 status = CAM_REQ_CMP; 1259 break; 1260 case VIRTIO_SCSI_S_OVERRUN: 1261 status = CAM_DATA_RUN_ERR; 1262 break; 1263 case VIRTIO_SCSI_S_ABORTED: 1264 status = CAM_REQ_ABORTED; 1265 break; 1266 case VIRTIO_SCSI_S_BAD_TARGET: 1267 status = CAM_SEL_TIMEOUT; 1268 break; 1269 case VIRTIO_SCSI_S_RESET: 1270 status = CAM_SCSI_BUS_RESET; 1271 break; 1272 case VIRTIO_SCSI_S_BUSY: 1273 status = CAM_SCSI_BUSY; 1274 break; 1275 case VIRTIO_SCSI_S_TRANSPORT_FAILURE: 1276 case VIRTIO_SCSI_S_TARGET_FAILURE: 1277 case VIRTIO_SCSI_S_NEXUS_FAILURE: 1278 status = CAM_SCSI_IT_NEXUS_LOST; 1279 break; 1280 default: /* VIRTIO_SCSI_S_FAILURE */ 1281 status = CAM_REQ_CMP_ERR; 1282 break; 1283 } 1284 1285 return (status); 1286} 1287 1288static cam_status 1289vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc, 1290 struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp) 1291{ 1292 cam_status status; 1293 1294 csio->scsi_status = cmd_resp->status; 1295 csio->resid = cmd_resp->resid; 1296 1297 if (csio->scsi_status == SCSI_STATUS_OK) 1298 status = CAM_REQ_CMP; 1299 else 1300 status = CAM_SCSI_STATUS_ERROR; 1301 1302 if (cmd_resp->sense_len > 0) { 1303 status |= CAM_AUTOSNS_VALID; 1304 1305 if (cmd_resp->sense_len < csio->sense_len) 1306 csio->sense_resid = csio->sense_len - 1307 cmd_resp->sense_len; 1308 else 1309 csio->sense_resid = 0; 1310 1311 bzero(&csio->sense_data, sizeof(csio->sense_data)); 1312 memcpy(cmd_resp->sense, &csio->sense_data, 1313 csio->sense_len - csio->sense_resid); 1314 } 1315 1316 vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR, 1317 "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n", 1318 csio, csio->scsi_status, csio->resid, csio->sense_resid); 1319 1320 return (status); 1321} 1322 1323static void 1324vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1325{ 1326 struct ccb_hdr *ccbh; 1327 struct ccb_scsiio *csio; 1328 struct virtio_scsi_cmd_resp *cmd_resp; 1329 cam_status status; 1330 1331 csio = &req->vsr_ccb->csio; 1332 ccbh = &csio->ccb_h; 1333 cmd_resp = &req->vsr_cmd_resp; 1334 1335 KASSERT(ccbh->ccbh_vtscsi_req == req, 1336 ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req)); 1337 1338 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1339 callout_stop(&req->vsr_callout); 1340 1341 status = vtscsi_scsi_cmd_cam_status(cmd_resp); 1342 if (status == CAM_REQ_ABORTED) { 1343 if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT) 1344 status = CAM_CMD_TIMEOUT; 1345 } else if (status == CAM_REQ_CMP) 1346 status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp); 1347 1348 if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1349 status |= CAM_DEV_QFRZN; 1350 xpt_freeze_devq(ccbh->path, 1); 1351 } 1352 1353 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 1354 status |= CAM_RELEASE_SIMQ; 1355 1356 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n", 1357 req, ccbh, status); 1358 1359 ccbh->status = status; 1360 xpt_done(req->vsr_ccb); 1361 vtscsi_enqueue_request(sc, req); 1362} 1363 1364static void 1365vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req) 1366{ 1367 1368 /* XXX We probably shouldn't poll forever. */ 1369 req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED; 1370 do 1371 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1372 while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0); 1373 1374 req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED; 1375} 1376 1377static int 1378vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req, 1379 struct sglist *sg, int readable, int writable, int flag) 1380{ 1381 struct virtqueue *vq; 1382 int error; 1383 1384 vq = sc->vtscsi_control_vq; 1385 1386 MPASS(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL); 1387 1388 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1389 if (error) { 1390 /* 1391 * Return EAGAIN when the virtqueue does not have enough 1392 * descriptors available. 1393 */ 1394 if (error == ENOSPC || error == EMSGSIZE) 1395 error = EAGAIN; 1396 1397 return (error); 1398 } 1399 1400 virtqueue_notify(vq); 1401 if (flag == VTSCSI_EXECUTE_POLL) 1402 vtscsi_poll_ctrl_req(sc, req); 1403 1404 return (0); 1405} 1406 1407static void 1408vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc, 1409 struct vtscsi_request *req) 1410{ 1411 union ccb *ccb; 1412 struct ccb_hdr *ccbh; 1413 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1414 1415 ccb = req->vsr_ccb; 1416 ccbh = &ccb->ccb_h; 1417 tmf_resp = &req->vsr_tmf_resp; 1418 1419 switch (tmf_resp->response) { 1420 case VIRTIO_SCSI_S_FUNCTION_COMPLETE: 1421 ccbh->status = CAM_REQ_CMP; 1422 break; 1423 case VIRTIO_SCSI_S_FUNCTION_REJECTED: 1424 ccbh->status = CAM_UA_ABORT; 1425 break; 1426 default: 1427 ccbh->status = CAM_REQ_CMP_ERR; 1428 break; 1429 } 1430 1431 xpt_done(ccb); 1432 vtscsi_enqueue_request(sc, req); 1433} 1434 1435static int 1436vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc, 1437 struct vtscsi_request *req) 1438{ 1439 struct sglist *sg; 1440 struct ccb_abort *cab; 1441 struct ccb_hdr *ccbh; 1442 struct ccb_hdr *abort_ccbh; 1443 struct vtscsi_request *abort_req; 1444 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1445 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1446 int error; 1447 1448 sg = sc->vtscsi_sglist; 1449 cab = &req->vsr_ccb->cab; 1450 ccbh = &cab->ccb_h; 1451 tmf_req = &req->vsr_tmf_req; 1452 tmf_resp = &req->vsr_tmf_resp; 1453 1454 /* CCB header and request that's to be aborted. */ 1455 abort_ccbh = &cab->abort_ccb->ccb_h; 1456 abort_req = abort_ccbh->ccbh_vtscsi_req; 1457 1458 if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) { 1459 error = EINVAL; 1460 goto fail; 1461 } 1462 1463 /* Only attempt to abort requests that could be in-flight. */ 1464 if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) { 1465 error = EALREADY; 1466 goto fail; 1467 } 1468 1469 abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED; 1470 if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1471 callout_stop(&abort_req->vsr_callout); 1472 1473 vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1474 (uintptr_t) abort_ccbh, tmf_req); 1475 1476 sglist_reset(sg); 1477 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1478 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1479 1480 req->vsr_complete = vtscsi_complete_abort_task_cmd; 1481 tmf_resp->response = -1; 1482 1483 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1484 VTSCSI_EXECUTE_ASYNC); 1485 1486fail: 1487 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p " 1488 "abort_req=%p\n", error, req, abort_ccbh, abort_req); 1489 1490 return (error); 1491} 1492 1493static void 1494vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc, 1495 struct vtscsi_request *req) 1496{ 1497 union ccb *ccb; 1498 struct ccb_hdr *ccbh; 1499 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1500 1501 ccb = req->vsr_ccb; 1502 ccbh = &ccb->ccb_h; 1503 tmf_resp = &req->vsr_tmf_resp; 1504 1505 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n", 1506 req, ccb, tmf_resp->response); 1507 1508 if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) { 1509 ccbh->status = CAM_REQ_CMP; 1510 vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id, 1511 ccbh->target_lun); 1512 } else 1513 ccbh->status = CAM_REQ_CMP_ERR; 1514 1515 xpt_done(ccb); 1516 vtscsi_enqueue_request(sc, req); 1517} 1518 1519static int 1520vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc, 1521 struct vtscsi_request *req) 1522{ 1523 struct sglist *sg; 1524 struct ccb_resetdev *crd; 1525 struct ccb_hdr *ccbh; 1526 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1527 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1528 uint32_t subtype; 1529 int error; 1530 1531 sg = sc->vtscsi_sglist; 1532 crd = &req->vsr_ccb->crd; 1533 ccbh = &crd->ccb_h; 1534 tmf_req = &req->vsr_tmf_req; 1535 tmf_resp = &req->vsr_tmf_resp; 1536 1537 if (ccbh->target_lun == CAM_LUN_WILDCARD) 1538 subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET; 1539 else 1540 subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET; 1541 1542 vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req); 1543 1544 sglist_reset(sg); 1545 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1546 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1547 1548 req->vsr_complete = vtscsi_complete_reset_dev_cmd; 1549 tmf_resp->response = -1; 1550 1551 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1552 VTSCSI_EXECUTE_ASYNC); 1553 1554 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n", 1555 error, req, ccbh); 1556 1557 return (error); 1558} 1559 1560static void 1561vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id) 1562{ 1563 1564 *target_id = lun[1]; 1565 *lun_id = (lun[2] << 8) | lun[3]; 1566} 1567 1568static void 1569vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[]) 1570{ 1571 1572 lun[0] = 1; 1573 lun[1] = ccbh->target_id; 1574 lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F); 1575 lun[3] = ccbh->target_lun & 0xFF; 1576} 1577 1578static void 1579vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio, 1580 struct virtio_scsi_cmd_req *cmd_req) 1581{ 1582 uint8_t attr; 1583 1584 switch (csio->tag_action) { 1585 case MSG_HEAD_OF_Q_TAG: 1586 attr = VIRTIO_SCSI_S_HEAD; 1587 break; 1588 case MSG_ORDERED_Q_TAG: 1589 attr = VIRTIO_SCSI_S_ORDERED; 1590 break; 1591 case MSG_ACA_TASK: 1592 attr = VIRTIO_SCSI_S_ACA; 1593 break; 1594 default: /* MSG_SIMPLE_Q_TAG */ 1595 attr = VIRTIO_SCSI_S_SIMPLE; 1596 break; 1597 } 1598 1599 vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun); 1600 cmd_req->tag = (uintptr_t) csio; 1601 cmd_req->task_attr = attr; 1602 1603 memcpy(cmd_req->cdb, 1604 csio->ccb_h.flags & CAM_CDB_POINTER ? 1605 csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes, 1606 csio->cdb_len); 1607} 1608 1609static void 1610vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype, 1611 uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) 1612{ 1613 1614 vtscsi_set_request_lun(ccbh, tmf_req->lun); 1615 1616 tmf_req->type = VIRTIO_SCSI_T_TMF; 1617 tmf_req->subtype = subtype; 1618 tmf_req->tag = tag; 1619} 1620 1621static void 1622vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason) 1623{ 1624 int frozen; 1625 1626 frozen = sc->vtscsi_frozen; 1627 1628 if (reason & VTSCSI_REQUEST && 1629 (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0) 1630 sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS; 1631 1632 if (reason & VTSCSI_REQUEST_VQ && 1633 (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0) 1634 sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL; 1635 1636 /* Freeze the SIMQ if transitioned to frozen. */ 1637 if (frozen == 0 && sc->vtscsi_frozen != 0) { 1638 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n"); 1639 xpt_freeze_simq(sc->vtscsi_sim, 1); 1640 } 1641} 1642 1643static int 1644vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason) 1645{ 1646 int thawed; 1647 1648 if (sc->vtscsi_frozen == 0 || reason == 0) 1649 return (0); 1650 1651 if (reason & VTSCSI_REQUEST && 1652 sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) 1653 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS; 1654 1655 if (reason & VTSCSI_REQUEST_VQ && 1656 sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) 1657 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL; 1658 1659 thawed = sc->vtscsi_frozen == 0; 1660 if (thawed != 0) 1661 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n"); 1662 1663 return (thawed); 1664} 1665 1666static void 1667vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code, 1668 target_id_t target_id, lun_id_t lun_id) 1669{ 1670 struct cam_path *path; 1671 1672 /* Use the wildcard path from our softc for bus announcements. */ 1673 if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) { 1674 xpt_async(ac_code, sc->vtscsi_path, NULL); 1675 return; 1676 } 1677 1678 if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim), 1679 target_id, lun_id) != CAM_REQ_CMP) { 1680 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n"); 1681 return; 1682 } 1683 1684 xpt_async(ac_code, path, NULL); 1685 xpt_free_path(path); 1686} 1687 1688static void 1689vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id, 1690 lun_id_t lun_id) 1691{ 1692 union ccb *ccb; 1693 cam_status status; 1694 1695 ccb = xpt_alloc_ccb_nowait(); 1696 if (ccb == NULL) { 1697 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); 1698 return; 1699 } 1700 1701 status = xpt_create_path(&ccb->ccb_h.path, NULL, 1702 cam_sim_path(sc->vtscsi_sim), target_id, lun_id); 1703 if (status != CAM_REQ_CMP) { 1704 xpt_free_ccb(ccb); 1705 return; 1706 } 1707 1708 xpt_rescan(ccb); 1709} 1710 1711static void 1712vtscsi_execute_rescan_bus(struct vtscsi_softc *sc) 1713{ 1714 1715 vtscsi_execute_rescan(sc, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); 1716} 1717 1718static void 1719vtscsi_transport_reset_event(struct vtscsi_softc *sc, 1720 struct virtio_scsi_event *event) 1721{ 1722 target_id_t target_id; 1723 lun_id_t lun_id; 1724 1725 vtscsi_get_request_lun(event->lun, &target_id, &lun_id); 1726 1727 switch (event->reason) { 1728 case VIRTIO_SCSI_EVT_RESET_RESCAN: 1729 case VIRTIO_SCSI_EVT_RESET_REMOVED: 1730 vtscsi_execute_rescan(sc, target_id, lun_id); 1731 break; 1732 default: 1733 device_printf(sc->vtscsi_dev, 1734 "unhandled transport event reason: %d\n", event->reason); 1735 break; 1736 } 1737} 1738 1739static void 1740vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) 1741{ 1742 int error; 1743 1744 if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) { 1745 switch (event->event) { 1746 case VIRTIO_SCSI_T_TRANSPORT_RESET: 1747 vtscsi_transport_reset_event(sc, event); 1748 break; 1749 default: 1750 device_printf(sc->vtscsi_dev, 1751 "unhandled event: %d\n", event->event); 1752 break; 1753 } 1754 } else 1755 vtscsi_execute_rescan_bus(sc); 1756 1757 /* 1758 * This should always be successful since the buffer 1759 * was just dequeued. 1760 */ 1761 error = vtscsi_enqueue_event_buf(sc, event); 1762 KASSERT(error == 0, 1763 ("cannot requeue event buffer: %d", error)); 1764} 1765 1766static int 1767vtscsi_enqueue_event_buf(struct vtscsi_softc *sc, 1768 struct virtio_scsi_event *event) 1769{ 1770 struct sglist *sg; 1771 struct virtqueue *vq; 1772 int size, error; 1773 1774 sg = sc->vtscsi_sglist; 1775 vq = sc->vtscsi_event_vq; 1776 size = sc->vtscsi_event_buf_size; 1777 1778 bzero(event, size); 1779 1780 sglist_reset(sg); 1781 error = sglist_append(sg, event, size); 1782 if (error) 1783 return (error); 1784 1785 error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg); 1786 if (error) 1787 return (error); 1788 1789 virtqueue_notify(vq); 1790 1791 return (0); 1792} 1793 1794static int 1795vtscsi_init_event_vq(struct vtscsi_softc *sc) 1796{ 1797 struct virtio_scsi_event *event; 1798 int i, size, error; 1799 1800 /* 1801 * The first release of QEMU with VirtIO SCSI support would crash 1802 * when attempting to notify the event virtqueue. This was fixed 1803 * when hotplug support was added. 1804 */ 1805 if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) 1806 size = sc->vtscsi_event_buf_size; 1807 else 1808 size = 0; 1809 1810 if (size < sizeof(struct virtio_scsi_event)) 1811 return (0); 1812 1813 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1814 event = &sc->vtscsi_event_bufs[i]; 1815 1816 error = vtscsi_enqueue_event_buf(sc, event); 1817 if (error) 1818 break; 1819 } 1820 1821 /* 1822 * Even just one buffer is enough. Missed events are 1823 * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag. 1824 */ 1825 if (i > 0) 1826 error = 0; 1827 1828 return (error); 1829} 1830 1831static void 1832vtscsi_reinit_event_vq(struct vtscsi_softc *sc) 1833{ 1834 struct virtio_scsi_event *event; 1835 int i, error; 1836 1837 if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 || 1838 sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event)) 1839 return; 1840 1841 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1842 event = &sc->vtscsi_event_bufs[i]; 1843 1844 error = vtscsi_enqueue_event_buf(sc, event); 1845 if (error) 1846 break; 1847 } 1848 1849 KASSERT(i > 0, ("cannot reinit event vq: %d", error)); 1850} 1851 1852static void 1853vtscsi_drain_event_vq(struct vtscsi_softc *sc) 1854{ 1855 struct virtqueue *vq; 1856 int last; 1857 1858 vq = sc->vtscsi_event_vq; 1859 last = 0; 1860 1861 while (virtqueue_drain(vq, &last) != NULL) 1862 ; 1863 1864 KASSERT(virtqueue_empty(vq), ("eventvq not empty")); 1865} 1866 1867static void 1868vtscsi_complete_vqs_locked(struct vtscsi_softc *sc) 1869{ 1870 1871 VTSCSI_LOCK_OWNED(sc); 1872 1873 if (sc->vtscsi_request_vq != NULL) 1874 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1875 if (sc->vtscsi_control_vq != NULL) 1876 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1877} 1878 1879static void 1880vtscsi_complete_vqs(struct vtscsi_softc *sc) 1881{ 1882 1883 VTSCSI_LOCK(sc); 1884 vtscsi_complete_vqs_locked(sc); 1885 VTSCSI_UNLOCK(sc); 1886} 1887 1888static void 1889vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 1890{ 1891 union ccb *ccb; 1892 int detach; 1893 1894 ccb = req->vsr_ccb; 1895 1896 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb); 1897 1898 /* 1899 * The callout must be drained when detaching since the request is 1900 * about to be freed. The VTSCSI_MTX must not be held for this in 1901 * case the callout is pending because there is a deadlock potential. 1902 * Otherwise, the virtqueue is being drained because of a bus reset 1903 * so we only need to attempt to stop the callouts. 1904 */ 1905 detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0; 1906 if (detach != 0) 1907 VTSCSI_LOCK_NOTOWNED(sc); 1908 else 1909 VTSCSI_LOCK_OWNED(sc); 1910 1911 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) { 1912 if (detach != 0) 1913 callout_drain(&req->vsr_callout); 1914 else 1915 callout_stop(&req->vsr_callout); 1916 } 1917 1918 if (ccb != NULL) { 1919 if (detach != 0) { 1920 VTSCSI_LOCK(sc); 1921 ccb->ccb_h.status = CAM_NO_HBA; 1922 } else 1923 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1924 xpt_done(ccb); 1925 if (detach != 0) 1926 VTSCSI_UNLOCK(sc); 1927 } 1928 1929 vtscsi_enqueue_request(sc, req); 1930} 1931 1932static void 1933vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 1934{ 1935 struct vtscsi_request *req; 1936 int last; 1937 1938 last = 0; 1939 1940 vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq); 1941 1942 while ((req = virtqueue_drain(vq, &last)) != NULL) 1943 vtscsi_cancel_request(sc, req); 1944 1945 KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); 1946} 1947 1948static void 1949vtscsi_drain_vqs(struct vtscsi_softc *sc) 1950{ 1951 1952 if (sc->vtscsi_control_vq != NULL) 1953 vtscsi_drain_vq(sc, sc->vtscsi_control_vq); 1954 if (sc->vtscsi_request_vq != NULL) 1955 vtscsi_drain_vq(sc, sc->vtscsi_request_vq); 1956 if (sc->vtscsi_event_vq != NULL) 1957 vtscsi_drain_event_vq(sc); 1958} 1959 1960static void 1961vtscsi_stop(struct vtscsi_softc *sc) 1962{ 1963 1964 vtscsi_disable_vqs_intr(sc); 1965 virtio_stop(sc->vtscsi_dev); 1966} 1967 1968static int 1969vtscsi_reset_bus(struct vtscsi_softc *sc) 1970{ 1971 int error; 1972 1973 VTSCSI_LOCK_OWNED(sc); 1974 1975 if (vtscsi_bus_reset_disable != 0) { 1976 device_printf(sc->vtscsi_dev, "bus reset disabled\n"); 1977 return (0); 1978 } 1979 1980 sc->vtscsi_flags |= VTSCSI_FLAG_RESET; 1981 1982 /* 1983 * vtscsi_stop() will cause the in-flight requests to be canceled. 1984 * Those requests are then completed here so CAM will retry them 1985 * after the reset is complete. 1986 */ 1987 vtscsi_stop(sc); 1988 vtscsi_complete_vqs_locked(sc); 1989 1990 /* Rid the virtqueues of any remaining requests. */ 1991 vtscsi_drain_vqs(sc); 1992 1993 /* 1994 * Any resource shortage that froze the SIMQ cannot persist across 1995 * a bus reset so ensure it gets thawed here. 1996 */ 1997 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 1998 xpt_release_simq(sc->vtscsi_sim, 0); 1999 2000 error = vtscsi_reinit(sc); 2001 if (error) { 2002 device_printf(sc->vtscsi_dev, 2003 "reinitialization failed, stopping device...\n"); 2004 vtscsi_stop(sc); 2005 } else 2006 vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, 2007 CAM_LUN_WILDCARD); 2008 2009 sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET; 2010 2011 return (error); 2012} 2013 2014static void 2015vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2016{ 2017 2018#ifdef INVARIANTS 2019 int req_nsegs, resp_nsegs; 2020 2021 req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2022 resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2023 2024 KASSERT(req_nsegs == 1, ("request crossed page boundary")); 2025 KASSERT(resp_nsegs == 1, ("response crossed page boundary")); 2026#endif 2027 2028 req->vsr_softc = sc; 2029 callout_init_mtx(&req->vsr_callout, VTSCSI_MTX(sc), 0); 2030} 2031 2032static int 2033vtscsi_alloc_requests(struct vtscsi_softc *sc) 2034{ 2035 struct vtscsi_request *req; 2036 int i, nreqs; 2037 2038 /* 2039 * Commands destined for either the request or control queues come 2040 * from the same SIM queue. Use the size of the request virtqueue 2041 * as it (should) be much more frequently used. Some additional 2042 * requests are allocated for internal (TMF) use. 2043 */ 2044 nreqs = virtqueue_size(sc->vtscsi_request_vq); 2045 if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) 2046 nreqs /= VTSCSI_MIN_SEGMENTS; 2047 nreqs += VTSCSI_RESERVED_REQUESTS; 2048 2049 for (i = 0; i < nreqs; i++) { 2050 req = malloc(sizeof(struct vtscsi_request), M_DEVBUF, 2051 M_NOWAIT); 2052 if (req == NULL) 2053 return (ENOMEM); 2054 2055 vtscsi_init_request(sc, req); 2056 2057 sc->vtscsi_nrequests++; 2058 vtscsi_enqueue_request(sc, req); 2059 } 2060 2061 return (0); 2062} 2063 2064static void 2065vtscsi_free_requests(struct vtscsi_softc *sc) 2066{ 2067 struct vtscsi_request *req; 2068 2069 while ((req = vtscsi_dequeue_request(sc)) != NULL) { 2070 KASSERT(callout_active(&req->vsr_callout) == 0, 2071 ("request callout still active")); 2072 2073 sc->vtscsi_nrequests--; 2074 free(req, M_DEVBUF); 2075 } 2076 2077 KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d", 2078 sc->vtscsi_nrequests)); 2079} 2080 2081static void 2082vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2083{ 2084 2085 KASSERT(req->vsr_softc == sc, 2086 ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc)); 2087 2088 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2089 2090 /* A request is available so the SIMQ could be released. */ 2091 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0) 2092 xpt_release_simq(sc->vtscsi_sim, 1); 2093 2094 req->vsr_ccb = NULL; 2095 req->vsr_complete = NULL; 2096 req->vsr_ptr0 = NULL; 2097 req->vsr_state = VTSCSI_REQ_STATE_FREE; 2098 req->vsr_flags = 0; 2099 2100 bzero(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2101 bzero(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2102 2103 /* 2104 * We insert at the tail of the queue in order to make it 2105 * very unlikely a request will be reused if we race with 2106 * stopping its callout handler. 2107 */ 2108 TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link); 2109} 2110 2111static struct vtscsi_request * 2112vtscsi_dequeue_request(struct vtscsi_softc *sc) 2113{ 2114 struct vtscsi_request *req; 2115 2116 req = TAILQ_FIRST(&sc->vtscsi_req_free); 2117 if (req != NULL) { 2118 req->vsr_state = VTSCSI_REQ_STATE_INUSE; 2119 TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link); 2120 } else 2121 sc->vtscsi_stats.dequeue_no_requests++; 2122 2123 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2124 2125 return (req); 2126} 2127 2128static void 2129vtscsi_complete_request(struct vtscsi_request *req) 2130{ 2131 2132 if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED) 2133 req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE; 2134 2135 if (req->vsr_complete != NULL) 2136 req->vsr_complete(req->vsr_softc, req); 2137} 2138 2139static void 2140vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 2141{ 2142 struct vtscsi_request *req; 2143 2144 VTSCSI_LOCK_OWNED(sc); 2145 2146 while ((req = virtqueue_dequeue(vq, NULL)) != NULL) 2147 vtscsi_complete_request(req); 2148} 2149 2150static void 2151vtscsi_control_vq_intr(void *xsc) 2152{ 2153 struct vtscsi_softc *sc; 2154 struct virtqueue *vq; 2155 2156 sc = xsc; 2157 vq = sc->vtscsi_control_vq; 2158 2159again: 2160 VTSCSI_LOCK(sc); 2161 2162 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 2163 2164 if (virtqueue_enable_intr(vq) != 0) { 2165 virtqueue_disable_intr(vq); 2166 VTSCSI_UNLOCK(sc); 2167 goto again; 2168 } 2169 2170 VTSCSI_UNLOCK(sc); 2171} 2172 2173static void 2174vtscsi_event_vq_intr(void *xsc) 2175{ 2176 struct vtscsi_softc *sc; 2177 struct virtqueue *vq; 2178 struct virtio_scsi_event *event; 2179 2180 sc = xsc; 2181 vq = sc->vtscsi_event_vq; 2182 2183again: 2184 VTSCSI_LOCK(sc); 2185 2186 while ((event = virtqueue_dequeue(vq, NULL)) != NULL) 2187 vtscsi_handle_event(sc, event); 2188 2189 if (virtqueue_enable_intr(vq) != 0) { 2190 virtqueue_disable_intr(vq); 2191 VTSCSI_UNLOCK(sc); 2192 goto again; 2193 } 2194 2195 VTSCSI_UNLOCK(sc); 2196} 2197 2198static void 2199vtscsi_request_vq_intr(void *xsc) 2200{ 2201 struct vtscsi_softc *sc; 2202 struct virtqueue *vq; 2203 2204 sc = xsc; 2205 vq = sc->vtscsi_request_vq; 2206 2207again: 2208 VTSCSI_LOCK(sc); 2209 2210 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 2211 2212 if (virtqueue_enable_intr(vq) != 0) { 2213 virtqueue_disable_intr(vq); 2214 VTSCSI_UNLOCK(sc); 2215 goto again; 2216 } 2217 2218 VTSCSI_UNLOCK(sc); 2219} 2220 2221static void 2222vtscsi_disable_vqs_intr(struct vtscsi_softc *sc) 2223{ 2224 2225 virtqueue_disable_intr(sc->vtscsi_control_vq); 2226 virtqueue_disable_intr(sc->vtscsi_event_vq); 2227 virtqueue_disable_intr(sc->vtscsi_request_vq); 2228} 2229 2230static void 2231vtscsi_enable_vqs_intr(struct vtscsi_softc *sc) 2232{ 2233 2234 virtqueue_enable_intr(sc->vtscsi_control_vq); 2235 virtqueue_enable_intr(sc->vtscsi_event_vq); 2236 virtqueue_enable_intr(sc->vtscsi_request_vq); 2237} 2238 2239static void 2240vtscsi_get_tunables(struct vtscsi_softc *sc) 2241{ 2242 char tmpstr[64]; 2243 2244 TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug); 2245 2246 snprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level", 2247 device_get_unit(sc->vtscsi_dev)); 2248 TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug); 2249} 2250 2251static void 2252vtscsi_add_sysctl(struct vtscsi_softc *sc) 2253{ 2254 device_t dev; 2255 struct vtscsi_statistics *stats; 2256 struct sysctl_ctx_list *ctx; 2257 struct sysctl_oid *tree; 2258 struct sysctl_oid_list *child; 2259 2260 dev = sc->vtscsi_dev; 2261 stats = &sc->vtscsi_stats; 2262 ctx = device_get_sysctl_ctx(dev); 2263 tree = device_get_sysctl_tree(dev); 2264 child = SYSCTL_CHILDREN(tree); 2265 2266 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", 2267 CTLFLAG_RW, &sc->vtscsi_debug, 0, 2268 "Debug level"); 2269 2270 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts", 2271 CTLFLAG_RD, &stats->scsi_cmd_timeouts, 2272 "SCSI command timeouts"); 2273 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests", 2274 CTLFLAG_RD, &stats->dequeue_no_requests, 2275 "No available requests to dequeue"); 2276} 2277 2278static void 2279vtscsi_printf_req(struct vtscsi_request *req, const char *func, 2280 const char *fmt, ...) 2281{ 2282 struct vtscsi_softc *sc; 2283 union ccb *ccb; 2284 struct sbuf sb; 2285 va_list ap; 2286 char str[192]; 2287 char path_str[64]; 2288 2289 if (req == NULL) 2290 return; 2291 2292 sc = req->vsr_softc; 2293 ccb = req->vsr_ccb; 2294 2295 va_start(ap, fmt); 2296 sbuf_new(&sb, str, sizeof(str), 0); 2297 2298 if (ccb == NULL) { 2299 sbuf_printf(&sb, "(noperiph:%s%d:%u): ", 2300 cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim), 2301 cam_sim_bus(sc->vtscsi_sim)); 2302 } else { 2303 xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str)); 2304 sbuf_cat(&sb, path_str); 2305 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 2306 scsi_command_string(&ccb->csio, &sb); 2307 sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len); 2308 } 2309 } 2310 2311 sbuf_vprintf(&sb, fmt, ap); 2312 va_end(ap); 2313 2314 sbuf_finish(&sb); 2315 printf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func, 2316 sbuf_data(&sb)); 2317} 2318