ctl_backend_block.c revision 275481
1229997Sken/*- 2229997Sken * Copyright (c) 2003 Silicon Graphics International Corp. 3229997Sken * Copyright (c) 2009-2011 Spectra Logic Corporation 4232604Strasz * Copyright (c) 2012 The FreeBSD Foundation 5229997Sken * All rights reserved. 6229997Sken * 7232604Strasz * Portions of this software were developed by Edward Tomasz Napierala 8232604Strasz * under sponsorship from the FreeBSD Foundation. 9232604Strasz * 10229997Sken * Redistribution and use in source and binary forms, with or without 11229997Sken * modification, are permitted provided that the following conditions 12229997Sken * are met: 13229997Sken * 1. Redistributions of source code must retain the above copyright 14229997Sken * notice, this list of conditions, and the following disclaimer, 15229997Sken * without modification. 16229997Sken * 2. Redistributions in binary form must reproduce at minimum a disclaimer 17229997Sken * substantially similar to the "NO WARRANTY" disclaimer below 18229997Sken * ("Disclaimer") and any redistribution must be conditioned upon 19229997Sken * including a substantially similar Disclaimer requirement for further 20229997Sken * binary redistribution. 21229997Sken * 22229997Sken * NO WARRANTY 23229997Sken * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 24229997Sken * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 25229997Sken * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 26229997Sken * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 27229997Sken * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28229997Sken * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29229997Sken * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30229997Sken * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 31229997Sken * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 32229997Sken * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33229997Sken * POSSIBILITY OF SUCH DAMAGES. 34229997Sken * 35229997Sken * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $ 36229997Sken */ 37229997Sken/* 38229997Sken * CAM Target Layer driver backend for block devices. 39229997Sken * 40229997Sken * Author: Ken Merry <ken@FreeBSD.org> 41229997Sken */ 42229997Sken#include <sys/cdefs.h> 43229997Sken__FBSDID("$FreeBSD: head/sys/cam/ctl/ctl_backend_block.c 275481 2014-12-04 18:37:42Z mav $"); 44229997Sken 45229997Sken#include <sys/param.h> 46229997Sken#include <sys/systm.h> 47229997Sken#include <sys/kernel.h> 48229997Sken#include <sys/types.h> 49229997Sken#include <sys/kthread.h> 50229997Sken#include <sys/bio.h> 51229997Sken#include <sys/fcntl.h> 52264274Smav#include <sys/limits.h> 53229997Sken#include <sys/lock.h> 54229997Sken#include <sys/mutex.h> 55229997Sken#include <sys/condvar.h> 56229997Sken#include <sys/malloc.h> 57229997Sken#include <sys/conf.h> 58229997Sken#include <sys/ioccom.h> 59229997Sken#include <sys/queue.h> 60229997Sken#include <sys/sbuf.h> 61229997Sken#include <sys/endian.h> 62229997Sken#include <sys/uio.h> 63229997Sken#include <sys/buf.h> 64229997Sken#include <sys/taskqueue.h> 65229997Sken#include <sys/vnode.h> 66229997Sken#include <sys/namei.h> 67229997Sken#include <sys/mount.h> 68229997Sken#include <sys/disk.h> 69229997Sken#include <sys/fcntl.h> 70229997Sken#include <sys/filedesc.h> 71275474Smav#include <sys/filio.h> 72229997Sken#include <sys/proc.h> 73229997Sken#include <sys/pcpu.h> 74229997Sken#include <sys/module.h> 75229997Sken#include <sys/sdt.h> 76229997Sken#include <sys/devicestat.h> 77229997Sken#include <sys/sysctl.h> 78229997Sken 79229997Sken#include <geom/geom.h> 80229997Sken 81229997Sken#include <cam/cam.h> 82229997Sken#include <cam/scsi/scsi_all.h> 83229997Sken#include <cam/scsi/scsi_da.h> 84229997Sken#include <cam/ctl/ctl_io.h> 85229997Sken#include <cam/ctl/ctl.h> 86229997Sken#include <cam/ctl/ctl_backend.h> 87229997Sken#include <cam/ctl/ctl_frontend_internal.h> 88229997Sken#include <cam/ctl/ctl_ioctl.h> 89229997Sken#include <cam/ctl/ctl_scsi_all.h> 90229997Sken#include <cam/ctl/ctl_error.h> 91229997Sken 92229997Sken/* 93264886Smav * The idea here is that we'll allocate enough S/G space to hold a 1MB 94264886Smav * I/O. If we get an I/O larger than that, we'll split it. 95229997Sken */ 96267537Smav#define CTLBLK_HALF_IO_SIZE (512 * 1024) 97267537Smav#define CTLBLK_MAX_IO_SIZE (CTLBLK_HALF_IO_SIZE * 2) 98264886Smav#define CTLBLK_MAX_SEG MAXPHYS 99267537Smav#define CTLBLK_HALF_SEGS MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1) 100267537Smav#define CTLBLK_MAX_SEGS (CTLBLK_HALF_SEGS * 2) 101229997Sken 102229997Sken#ifdef CTLBLK_DEBUG 103229997Sken#define DPRINTF(fmt, args...) \ 104229997Sken printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args) 105229997Sken#else 106229997Sken#define DPRINTF(fmt, args...) do {} while(0) 107229997Sken#endif 108229997Sken 109267519Smav#define PRIV(io) \ 110267519Smav ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND]) 111267537Smav#define ARGS(io) \ 112267537Smav ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]) 113267519Smav 114229997SkenSDT_PROVIDER_DEFINE(cbb); 115229997Sken 116229997Skentypedef enum { 117229997Sken CTL_BE_BLOCK_LUN_UNCONFIGURED = 0x01, 118229997Sken CTL_BE_BLOCK_LUN_CONFIG_ERR = 0x02, 119229997Sken CTL_BE_BLOCK_LUN_WAITING = 0x04, 120229997Sken CTL_BE_BLOCK_LUN_MULTI_THREAD = 0x08 121229997Sken} ctl_be_block_lun_flags; 122229997Sken 123229997Skentypedef enum { 124229997Sken CTL_BE_BLOCK_NONE, 125229997Sken CTL_BE_BLOCK_DEV, 126229997Sken CTL_BE_BLOCK_FILE 127229997Sken} ctl_be_block_type; 128229997Sken 129229997Skenstruct ctl_be_block_devdata { 130229997Sken struct cdev *cdev; 131229997Sken struct cdevsw *csw; 132229997Sken int dev_ref; 133229997Sken}; 134229997Sken 135229997Skenstruct ctl_be_block_filedata { 136229997Sken struct ucred *cred; 137229997Sken}; 138229997Sken 139229997Skenunion ctl_be_block_bedata { 140229997Sken struct ctl_be_block_devdata dev; 141229997Sken struct ctl_be_block_filedata file; 142229997Sken}; 143229997Sken 144229997Skenstruct ctl_be_block_io; 145229997Skenstruct ctl_be_block_lun; 146229997Sken 147229997Skentypedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun, 148229997Sken struct ctl_be_block_io *beio); 149274154Smavtypedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun, 150274154Smav const char *attrname); 151229997Sken 152229997Sken/* 153229997Sken * Backend LUN structure. There is a 1:1 mapping between a block device 154229997Sken * and a backend block LUN, and between a backend block LUN and a CTL LUN. 155229997Sken */ 156229997Skenstruct ctl_be_block_lun { 157272911Smav struct ctl_lun_create_params params; 158229997Sken struct ctl_block_disk *disk; 159229997Sken char lunname[32]; 160229997Sken char *dev_path; 161229997Sken ctl_be_block_type dev_type; 162229997Sken struct vnode *vn; 163229997Sken union ctl_be_block_bedata backend; 164229997Sken cbb_dispatch_t dispatch; 165229997Sken cbb_dispatch_t lun_flush; 166264274Smav cbb_dispatch_t unmap; 167275474Smav cbb_dispatch_t get_lba_status; 168274154Smav cbb_getattr_t getattr; 169229997Sken uma_zone_t lun_zone; 170229997Sken uint64_t size_blocks; 171229997Sken uint64_t size_bytes; 172229997Sken uint32_t blocksize; 173229997Sken int blocksize_shift; 174264191Smav uint16_t pblockexp; 175264191Smav uint16_t pblockoff; 176229997Sken struct ctl_be_block_softc *softc; 177229997Sken struct devstat *disk_stats; 178229997Sken ctl_be_block_lun_flags flags; 179229997Sken STAILQ_ENTRY(ctl_be_block_lun) links; 180229997Sken struct ctl_be_lun ctl_be_lun; 181229997Sken struct taskqueue *io_taskqueue; 182229997Sken struct task io_task; 183229997Sken int num_threads; 184229997Sken STAILQ_HEAD(, ctl_io_hdr) input_queue; 185275474Smav STAILQ_HEAD(, ctl_io_hdr) config_read_queue; 186229997Sken STAILQ_HEAD(, ctl_io_hdr) config_write_queue; 187229997Sken STAILQ_HEAD(, ctl_io_hdr) datamove_queue; 188267877Smav struct mtx_padalign io_lock; 189267877Smav struct mtx_padalign queue_lock; 190229997Sken}; 191229997Sken 192229997Sken/* 193229997Sken * Overall softc structure for the block backend module. 194229997Sken */ 195229997Skenstruct ctl_be_block_softc { 196229997Sken struct mtx lock; 197229997Sken int num_disks; 198229997Sken STAILQ_HEAD(, ctl_block_disk) disk_list; 199229997Sken int num_luns; 200229997Sken STAILQ_HEAD(, ctl_be_block_lun) lun_list; 201229997Sken}; 202229997Sken 203229997Skenstatic struct ctl_be_block_softc backend_block_softc; 204229997Sken 205229997Sken/* 206229997Sken * Per-I/O information. 207229997Sken */ 208229997Skenstruct ctl_be_block_io { 209229997Sken union ctl_io *io; 210229997Sken struct ctl_sg_entry sg_segs[CTLBLK_MAX_SEGS]; 211229997Sken struct iovec xiovecs[CTLBLK_MAX_SEGS]; 212229997Sken int bio_cmd; 213229997Sken int num_segs; 214229997Sken int num_bios_sent; 215229997Sken int num_bios_done; 216229997Sken int send_complete; 217229997Sken int num_errors; 218229997Sken struct bintime ds_t0; 219229997Sken devstat_tag_type ds_tag_type; 220229997Sken devstat_trans_flags ds_trans_type; 221229997Sken uint64_t io_len; 222229997Sken uint64_t io_offset; 223229997Sken struct ctl_be_block_softc *softc; 224229997Sken struct ctl_be_block_lun *lun; 225264274Smav void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */ 226229997Sken}; 227229997Sken 228229997Skenstatic int cbb_num_threads = 14; 229229997SkenSYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0, 230229997Sken "CAM Target Layer Block Backend"); 231267992ShselaskySYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN, 232229997Sken &cbb_num_threads, 0, "Number of threads per backing file"); 233229997Sken 234229997Skenstatic struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc); 235229997Skenstatic void ctl_free_beio(struct ctl_be_block_io *beio); 236229997Skenstatic void ctl_complete_beio(struct ctl_be_block_io *beio); 237229997Skenstatic int ctl_be_block_move_done(union ctl_io *io); 238229997Skenstatic void ctl_be_block_biodone(struct bio *bio); 239229997Skenstatic void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 240229997Sken struct ctl_be_block_io *beio); 241229997Skenstatic void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 242229997Sken struct ctl_be_block_io *beio); 243275474Smavstatic void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 244275474Smav struct ctl_be_block_io *beio); 245275481Smavstatic uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, 246275481Smav const char *attrname); 247229997Skenstatic void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 248229997Sken struct ctl_be_block_io *beio); 249264274Smavstatic void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 250264274Smav struct ctl_be_block_io *beio); 251229997Skenstatic void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 252229997Sken struct ctl_be_block_io *beio); 253274154Smavstatic uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, 254274154Smav const char *attrname); 255275474Smavstatic void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 256275474Smav union ctl_io *io); 257229997Skenstatic void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 258229997Sken union ctl_io *io); 259229997Skenstatic void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 260229997Sken union ctl_io *io); 261229997Skenstatic void ctl_be_block_worker(void *context, int pending); 262229997Skenstatic int ctl_be_block_submit(union ctl_io *io); 263229997Skenstatic int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 264229997Sken int flag, struct thread *td); 265229997Skenstatic int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, 266229997Sken struct ctl_lun_req *req); 267229997Skenstatic int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, 268229997Sken struct ctl_lun_req *req); 269229997Skenstatic int ctl_be_block_close(struct ctl_be_block_lun *be_lun); 270229997Skenstatic int ctl_be_block_open(struct ctl_be_block_softc *softc, 271229997Sken struct ctl_be_block_lun *be_lun, 272229997Sken struct ctl_lun_req *req); 273229997Skenstatic int ctl_be_block_create(struct ctl_be_block_softc *softc, 274229997Sken struct ctl_lun_req *req); 275229997Skenstatic int ctl_be_block_rm(struct ctl_be_block_softc *softc, 276229997Sken struct ctl_lun_req *req); 277232604Straszstatic int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun, 278232604Strasz struct ctl_lun_req *req); 279232604Straszstatic int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun, 280232604Strasz struct ctl_lun_req *req); 281232604Straszstatic int ctl_be_block_modify(struct ctl_be_block_softc *softc, 282232604Strasz struct ctl_lun_req *req); 283229997Skenstatic void ctl_be_block_lun_shutdown(void *be_lun); 284229997Skenstatic void ctl_be_block_lun_config_status(void *be_lun, 285229997Sken ctl_lun_config_status status); 286229997Skenstatic int ctl_be_block_config_write(union ctl_io *io); 287229997Skenstatic int ctl_be_block_config_read(union ctl_io *io); 288229997Skenstatic int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb); 289274154Smavstatic uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname); 290229997Skenint ctl_be_block_init(void); 291229997Sken 292229997Skenstatic struct ctl_backend_driver ctl_be_block_driver = 293229997Sken{ 294230334Sken .name = "block", 295230334Sken .flags = CTL_BE_FLAG_HAS_CONFIG, 296230334Sken .init = ctl_be_block_init, 297230334Sken .data_submit = ctl_be_block_submit, 298230334Sken .data_move_done = ctl_be_block_move_done, 299230334Sken .config_read = ctl_be_block_config_read, 300230334Sken .config_write = ctl_be_block_config_write, 301230334Sken .ioctl = ctl_be_block_ioctl, 302274154Smav .lun_info = ctl_be_block_lun_info, 303274154Smav .lun_attr = ctl_be_block_lun_attr 304229997Sken}; 305229997Sken 306229997SkenMALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend"); 307229997SkenCTL_BACKEND_DECLARE(cbb, ctl_be_block_driver); 308229997Sken 309264020Straszstatic uma_zone_t beio_zone; 310264020Strasz 311229997Skenstatic struct ctl_be_block_io * 312229997Skenctl_alloc_beio(struct ctl_be_block_softc *softc) 313229997Sken{ 314229997Sken struct ctl_be_block_io *beio; 315229997Sken 316264020Strasz beio = uma_zalloc(beio_zone, M_WAITOK | M_ZERO); 317264020Strasz beio->softc = softc; 318229997Sken return (beio); 319229997Sken} 320229997Sken 321229997Skenstatic void 322229997Skenctl_free_beio(struct ctl_be_block_io *beio) 323229997Sken{ 324229997Sken int duplicate_free; 325229997Sken int i; 326229997Sken 327229997Sken duplicate_free = 0; 328229997Sken 329229997Sken for (i = 0; i < beio->num_segs; i++) { 330229997Sken if (beio->sg_segs[i].addr == NULL) 331229997Sken duplicate_free++; 332229997Sken 333229997Sken uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr); 334229997Sken beio->sg_segs[i].addr = NULL; 335267537Smav 336267537Smav /* For compare we had two equal S/G lists. */ 337267537Smav if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) { 338267537Smav uma_zfree(beio->lun->lun_zone, 339267537Smav beio->sg_segs[i + CTLBLK_HALF_SEGS].addr); 340267537Smav beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL; 341267537Smav } 342229997Sken } 343229997Sken 344229997Sken if (duplicate_free > 0) { 345229997Sken printf("%s: %d duplicate frees out of %d segments\n", __func__, 346229997Sken duplicate_free, beio->num_segs); 347229997Sken } 348229997Sken 349264020Strasz uma_zfree(beio_zone, beio); 350229997Sken} 351229997Sken 352229997Skenstatic void 353229997Skenctl_complete_beio(struct ctl_be_block_io *beio) 354229997Sken{ 355267877Smav union ctl_io *io = beio->io; 356229997Sken 357264274Smav if (beio->beio_cont != NULL) { 358264274Smav beio->beio_cont(beio); 359264274Smav } else { 360264274Smav ctl_free_beio(beio); 361267537Smav ctl_data_submit_done(io); 362264274Smav } 363229997Sken} 364229997Sken 365229997Skenstatic int 366229997Skenctl_be_block_move_done(union ctl_io *io) 367229997Sken{ 368229997Sken struct ctl_be_block_io *beio; 369229997Sken struct ctl_be_block_lun *be_lun; 370267537Smav struct ctl_lba_len_flags *lbalen; 371229997Sken#ifdef CTL_TIME_IO 372229997Sken struct bintime cur_bt; 373267537Smav#endif 374267537Smav int i; 375229997Sken 376267519Smav beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 377229997Sken be_lun = beio->lun; 378229997Sken 379229997Sken DPRINTF("entered\n"); 380229997Sken 381229997Sken#ifdef CTL_TIME_IO 382229997Sken getbintime(&cur_bt); 383229997Sken bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); 384229997Sken bintime_add(&io->io_hdr.dma_bt, &cur_bt); 385229997Sken io->io_hdr.num_dmas++; 386229997Sken#endif 387267537Smav io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; 388229997Sken 389229997Sken /* 390229997Sken * We set status at this point for read commands, and write 391229997Sken * commands with errors. 392229997Sken */ 393275058Smav if (io->io_hdr.flags & CTL_FLAG_ABORT) { 394275058Smav ; 395275058Smav } else if ((io->io_hdr.port_status == 0) && 396267537Smav ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) { 397267537Smav lbalen = ARGS(beio->io); 398267537Smav if (lbalen->flags & CTL_LLF_READ) { 399267537Smav ctl_set_success(&io->scsiio); 400267537Smav } else if (lbalen->flags & CTL_LLF_COMPARE) { 401267537Smav /* We have two data blocks ready for comparison. */ 402267537Smav for (i = 0; i < beio->num_segs; i++) { 403267537Smav if (memcmp(beio->sg_segs[i].addr, 404267537Smav beio->sg_segs[i + CTLBLK_HALF_SEGS].addr, 405267537Smav beio->sg_segs[i].len) != 0) 406267537Smav break; 407267537Smav } 408267537Smav if (i < beio->num_segs) 409267537Smav ctl_set_sense(&io->scsiio, 410267537Smav /*current_error*/ 1, 411267537Smav /*sense_key*/ SSD_KEY_MISCOMPARE, 412267537Smav /*asc*/ 0x1D, 413267537Smav /*ascq*/ 0x00, 414267537Smav SSD_ELEM_NONE); 415267537Smav else 416267537Smav ctl_set_success(&io->scsiio); 417267537Smav } 418275058Smav } else if ((io->io_hdr.port_status != 0) && 419275058Smav ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || 420275058Smav (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { 421229997Sken /* 422229997Sken * For hardware error sense keys, the sense key 423229997Sken * specific value is defined to be a retry count, 424229997Sken * but we use it to pass back an internal FETD 425229997Sken * error code. XXX KDM Hopefully the FETD is only 426229997Sken * using 16 bits for an error code, since that's 427229997Sken * all the space we have in the sks field. 428229997Sken */ 429229997Sken ctl_set_internal_failure(&io->scsiio, 430229997Sken /*sks_valid*/ 1, 431229997Sken /*retry_count*/ 432229997Sken io->io_hdr.port_status); 433229997Sken } 434229997Sken 435229997Sken /* 436229997Sken * If this is a read, or a write with errors, it is done. 437229997Sken */ 438229997Sken if ((beio->bio_cmd == BIO_READ) 439229997Sken || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0) 440229997Sken || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) { 441229997Sken ctl_complete_beio(beio); 442229997Sken return (0); 443229997Sken } 444229997Sken 445229997Sken /* 446229997Sken * At this point, we have a write and the DMA completed 447229997Sken * successfully. We now have to queue it to the task queue to 448229997Sken * execute the backend I/O. That is because we do blocking 449229997Sken * memory allocations, and in the file backing case, blocking I/O. 450229997Sken * This move done routine is generally called in the SIM's 451229997Sken * interrupt context, and therefore we cannot block. 452229997Sken */ 453267877Smav mtx_lock(&be_lun->queue_lock); 454229997Sken /* 455229997Sken * XXX KDM make sure that links is okay to use at this point. 456229997Sken * Otherwise, we either need to add another field to ctl_io_hdr, 457229997Sken * or deal with resource allocation here. 458229997Sken */ 459229997Sken STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); 460267877Smav mtx_unlock(&be_lun->queue_lock); 461229997Sken 462229997Sken taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 463229997Sken 464229997Sken return (0); 465229997Sken} 466229997Sken 467229997Skenstatic void 468229997Skenctl_be_block_biodone(struct bio *bio) 469229997Sken{ 470229997Sken struct ctl_be_block_io *beio; 471229997Sken struct ctl_be_block_lun *be_lun; 472229997Sken union ctl_io *io; 473261538Smav int error; 474229997Sken 475229997Sken beio = bio->bio_caller1; 476229997Sken be_lun = beio->lun; 477229997Sken io = beio->io; 478229997Sken 479229997Sken DPRINTF("entered\n"); 480229997Sken 481261538Smav error = bio->bio_error; 482267877Smav mtx_lock(&be_lun->io_lock); 483261538Smav if (error != 0) 484229997Sken beio->num_errors++; 485229997Sken 486229997Sken beio->num_bios_done++; 487229997Sken 488229997Sken /* 489229997Sken * XXX KDM will this cause WITNESS to complain? Holding a lock 490229997Sken * during the free might cause it to complain. 491229997Sken */ 492229997Sken g_destroy_bio(bio); 493229997Sken 494229997Sken /* 495229997Sken * If the send complete bit isn't set, or we aren't the last I/O to 496229997Sken * complete, then we're done. 497229997Sken */ 498229997Sken if ((beio->send_complete == 0) 499229997Sken || (beio->num_bios_done < beio->num_bios_sent)) { 500267877Smav mtx_unlock(&be_lun->io_lock); 501229997Sken return; 502229997Sken } 503229997Sken 504229997Sken /* 505229997Sken * At this point, we've verified that we are the last I/O to 506229997Sken * complete, so it's safe to drop the lock. 507229997Sken */ 508267877Smav devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 509267877Smav beio->ds_tag_type, beio->ds_trans_type, 510267877Smav /*now*/ NULL, /*then*/&beio->ds_t0); 511267877Smav mtx_unlock(&be_lun->io_lock); 512229997Sken 513229997Sken /* 514229997Sken * If there are any errors from the backing device, we fail the 515229997Sken * entire I/O with a medium error. 516229997Sken */ 517229997Sken if (beio->num_errors > 0) { 518261538Smav if (error == EOPNOTSUPP) { 519261538Smav ctl_set_invalid_opcode(&io->scsiio); 520273809Smav } else if (error == ENOSPC) { 521273809Smav ctl_set_space_alloc_fail(&io->scsiio); 522261538Smav } else if (beio->bio_cmd == BIO_FLUSH) { 523229997Sken /* XXX KDM is there is a better error here? */ 524229997Sken ctl_set_internal_failure(&io->scsiio, 525229997Sken /*sks_valid*/ 1, 526229997Sken /*retry_count*/ 0xbad2); 527229997Sken } else 528229997Sken ctl_set_medium_error(&io->scsiio); 529229997Sken ctl_complete_beio(beio); 530229997Sken return; 531229997Sken } 532229997Sken 533229997Sken /* 534267537Smav * If this is a write, a flush, a delete or verify, we're all done. 535229997Sken * If this is a read, we can now send the data to the user. 536229997Sken */ 537229997Sken if ((beio->bio_cmd == BIO_WRITE) 538264274Smav || (beio->bio_cmd == BIO_FLUSH) 539267537Smav || (beio->bio_cmd == BIO_DELETE) 540267537Smav || (ARGS(io)->flags & CTL_LLF_VERIFY)) { 541229997Sken ctl_set_success(&io->scsiio); 542229997Sken ctl_complete_beio(beio); 543229997Sken } else { 544275058Smav if ((ARGS(io)->flags & CTL_LLF_READ) && 545275058Smav beio->beio_cont == NULL) 546275058Smav ctl_set_success(&io->scsiio); 547229997Sken#ifdef CTL_TIME_IO 548229997Sken getbintime(&io->io_hdr.dma_start_bt); 549229997Sken#endif 550229997Sken ctl_datamove(io); 551229997Sken } 552229997Sken} 553229997Sken 554229997Skenstatic void 555229997Skenctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, 556229997Sken struct ctl_be_block_io *beio) 557229997Sken{ 558267877Smav union ctl_io *io = beio->io; 559229997Sken struct mount *mountpoint; 560241896Skib int error, lock_flags; 561229997Sken 562229997Sken DPRINTF("entered\n"); 563229997Sken 564267877Smav binuptime(&beio->ds_t0); 565267877Smav mtx_lock(&be_lun->io_lock); 566267877Smav devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 567267877Smav mtx_unlock(&be_lun->io_lock); 568229997Sken 569267877Smav (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 570229997Sken 571229997Sken if (MNT_SHARED_WRITES(mountpoint) 572229997Sken || ((mountpoint == NULL) 573229997Sken && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 574229997Sken lock_flags = LK_SHARED; 575229997Sken else 576229997Sken lock_flags = LK_EXCLUSIVE; 577229997Sken 578229997Sken vn_lock(be_lun->vn, lock_flags | LK_RETRY); 579229997Sken 580229997Sken error = VOP_FSYNC(be_lun->vn, MNT_WAIT, curthread); 581229997Sken VOP_UNLOCK(be_lun->vn, 0); 582229997Sken 583229997Sken vn_finished_write(mountpoint); 584229997Sken 585267877Smav mtx_lock(&be_lun->io_lock); 586267877Smav devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 587267877Smav beio->ds_tag_type, beio->ds_trans_type, 588267877Smav /*now*/ NULL, /*then*/&beio->ds_t0); 589267877Smav mtx_unlock(&be_lun->io_lock); 590267877Smav 591229997Sken if (error == 0) 592229997Sken ctl_set_success(&io->scsiio); 593229997Sken else { 594229997Sken /* XXX KDM is there is a better error here? */ 595229997Sken ctl_set_internal_failure(&io->scsiio, 596229997Sken /*sks_valid*/ 1, 597229997Sken /*retry_count*/ 0xbad1); 598229997Sken } 599229997Sken 600229997Sken ctl_complete_beio(beio); 601229997Sken} 602229997Sken 603258622SavgSDT_PROBE_DEFINE1(cbb, kernel, read, file_start, "uint64_t"); 604258622SavgSDT_PROBE_DEFINE1(cbb, kernel, write, file_start, "uint64_t"); 605258622SavgSDT_PROBE_DEFINE1(cbb, kernel, read, file_done,"uint64_t"); 606258622SavgSDT_PROBE_DEFINE1(cbb, kernel, write, file_done, "uint64_t"); 607229997Sken 608229997Skenstatic void 609229997Skenctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, 610229997Sken struct ctl_be_block_io *beio) 611229997Sken{ 612229997Sken struct ctl_be_block_filedata *file_data; 613229997Sken union ctl_io *io; 614229997Sken struct uio xuio; 615229997Sken struct iovec *xiovec; 616241896Skib int flags; 617229997Sken int error, i; 618229997Sken 619229997Sken DPRINTF("entered\n"); 620229997Sken 621229997Sken file_data = &be_lun->backend.file; 622229997Sken io = beio->io; 623271309Smav flags = 0; 624271309Smav if (ARGS(io)->flags & CTL_LLF_DPO) 625271309Smav flags |= IO_DIRECT; 626271309Smav if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 627271309Smav flags |= IO_SYNC; 628229997Sken 629267537Smav bzero(&xuio, sizeof(xuio)); 630229997Sken if (beio->bio_cmd == BIO_READ) { 631229997Sken SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0); 632267537Smav xuio.uio_rw = UIO_READ; 633229997Sken } else { 634229997Sken SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0); 635267537Smav xuio.uio_rw = UIO_WRITE; 636229997Sken } 637229997Sken xuio.uio_offset = beio->io_offset; 638229997Sken xuio.uio_resid = beio->io_len; 639229997Sken xuio.uio_segflg = UIO_SYSSPACE; 640229997Sken xuio.uio_iov = beio->xiovecs; 641229997Sken xuio.uio_iovcnt = beio->num_segs; 642229997Sken xuio.uio_td = curthread; 643229997Sken 644229997Sken for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 645229997Sken xiovec->iov_base = beio->sg_segs[i].addr; 646229997Sken xiovec->iov_len = beio->sg_segs[i].len; 647229997Sken } 648229997Sken 649267877Smav binuptime(&beio->ds_t0); 650267877Smav mtx_lock(&be_lun->io_lock); 651267877Smav devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 652267877Smav mtx_unlock(&be_lun->io_lock); 653267877Smav 654229997Sken if (beio->bio_cmd == BIO_READ) { 655229997Sken vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 656229997Sken 657229997Sken /* 658229997Sken * UFS pays attention to IO_DIRECT for reads. If the 659229997Sken * DIRECTIO option is configured into the kernel, it calls 660229997Sken * ffs_rawread(). But that only works for single-segment 661229997Sken * uios with user space addresses. In our case, with a 662229997Sken * kernel uio, it still reads into the buffer cache, but it 663229997Sken * will just try to release the buffer from the cache later 664229997Sken * on in ffs_read(). 665229997Sken * 666229997Sken * ZFS does not pay attention to IO_DIRECT for reads. 667229997Sken * 668229997Sken * UFS does not pay attention to IO_SYNC for reads. 669229997Sken * 670229997Sken * ZFS pays attention to IO_SYNC (which translates into the 671229997Sken * Solaris define FRSYNC for zfs_read()) for reads. It 672229997Sken * attempts to sync the file before reading. 673229997Sken * 674229997Sken * So, to attempt to provide some barrier semantics in the 675229997Sken * BIO_ORDERED case, set both IO_DIRECT and IO_SYNC. 676229997Sken */ 677271309Smav error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred); 678229997Sken 679229997Sken VOP_UNLOCK(be_lun->vn, 0); 680267537Smav SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0); 681229997Sken } else { 682229997Sken struct mount *mountpoint; 683229997Sken int lock_flags; 684229997Sken 685229997Sken (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT); 686229997Sken 687229997Sken if (MNT_SHARED_WRITES(mountpoint) 688229997Sken || ((mountpoint == NULL) 689229997Sken && MNT_SHARED_WRITES(be_lun->vn->v_mount))) 690229997Sken lock_flags = LK_SHARED; 691229997Sken else 692229997Sken lock_flags = LK_EXCLUSIVE; 693229997Sken 694229997Sken vn_lock(be_lun->vn, lock_flags | LK_RETRY); 695229997Sken 696229997Sken /* 697229997Sken * UFS pays attention to IO_DIRECT for writes. The write 698229997Sken * is done asynchronously. (Normally the write would just 699229997Sken * get put into cache. 700229997Sken * 701229997Sken * UFS pays attention to IO_SYNC for writes. It will 702229997Sken * attempt to write the buffer out synchronously if that 703229997Sken * flag is set. 704229997Sken * 705229997Sken * ZFS does not pay attention to IO_DIRECT for writes. 706229997Sken * 707229997Sken * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC) 708229997Sken * for writes. It will flush the transaction from the 709229997Sken * cache before returning. 710229997Sken * 711229997Sken * So if we've got the BIO_ORDERED flag set, we want 712229997Sken * IO_SYNC in either the UFS or ZFS case. 713229997Sken */ 714271309Smav error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred); 715229997Sken VOP_UNLOCK(be_lun->vn, 0); 716229997Sken 717229997Sken vn_finished_write(mountpoint); 718267537Smav SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0); 719229997Sken } 720229997Sken 721267877Smav mtx_lock(&be_lun->io_lock); 722267877Smav devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 723267877Smav beio->ds_tag_type, beio->ds_trans_type, 724267877Smav /*now*/ NULL, /*then*/&beio->ds_t0); 725267877Smav mtx_unlock(&be_lun->io_lock); 726267877Smav 727229997Sken /* 728229997Sken * If we got an error, set the sense data to "MEDIUM ERROR" and 729229997Sken * return the I/O to the user. 730229997Sken */ 731229997Sken if (error != 0) { 732229997Sken char path_str[32]; 733229997Sken 734229997Sken ctl_scsi_path_string(io, path_str, sizeof(path_str)); 735229997Sken printf("%s%s command returned errno %d\n", path_str, 736229997Sken (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error); 737273809Smav if (error == ENOSPC) { 738273809Smav ctl_set_space_alloc_fail(&io->scsiio); 739273809Smav } else 740273809Smav ctl_set_medium_error(&io->scsiio); 741229997Sken ctl_complete_beio(beio); 742229997Sken return; 743229997Sken } 744229997Sken 745229997Sken /* 746269122Smav * If this is a write or a verify, we're all done. 747229997Sken * If this is a read, we can now send the data to the user. 748229997Sken */ 749269122Smav if ((beio->bio_cmd == BIO_WRITE) || 750269122Smav (ARGS(io)->flags & CTL_LLF_VERIFY)) { 751229997Sken ctl_set_success(&io->scsiio); 752229997Sken ctl_complete_beio(beio); 753229997Sken } else { 754275058Smav if ((ARGS(io)->flags & CTL_LLF_READ) && 755275058Smav beio->beio_cont == NULL) 756275058Smav ctl_set_success(&io->scsiio); 757229997Sken#ifdef CTL_TIME_IO 758229997Sken getbintime(&io->io_hdr.dma_start_bt); 759229997Sken#endif 760229997Sken ctl_datamove(io); 761229997Sken } 762229997Sken} 763229997Sken 764229997Skenstatic void 765275474Smavctl_be_block_gls_file(struct ctl_be_block_lun *be_lun, 766275474Smav struct ctl_be_block_io *beio) 767275474Smav{ 768275474Smav union ctl_io *io = beio->io; 769275474Smav struct ctl_lba_len_flags *lbalen = ARGS(io); 770275474Smav struct scsi_get_lba_status_data *data; 771275474Smav off_t roff, off; 772275474Smav int error, status; 773275474Smav 774275474Smav DPRINTF("entered\n"); 775275474Smav 776275474Smav off = roff = ((off_t)lbalen->lba) << be_lun->blocksize_shift; 777275474Smav vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 778275474Smav error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off, 779275474Smav 0, curthread->td_ucred, curthread); 780275474Smav if (error == 0 && off > roff) 781275474Smav status = 0; /* mapped up to off */ 782275474Smav else { 783275474Smav error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off, 784275474Smav 0, curthread->td_ucred, curthread); 785275474Smav if (error == 0 && off > roff) 786275474Smav status = 1; /* deallocated up to off */ 787275474Smav else { 788275474Smav status = 0; /* unknown up to the end */ 789275474Smav off = be_lun->size_bytes; 790275474Smav } 791275474Smav } 792275474Smav VOP_UNLOCK(be_lun->vn, 0); 793275474Smav 794275474Smav off >>= be_lun->blocksize_shift; 795275474Smav data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 796275474Smav scsi_u64to8b(lbalen->lba, data->descr[0].addr); 797275474Smav scsi_ulto4b(MIN(UINT32_MAX, off - lbalen->lba), 798275474Smav data->descr[0].length); 799275474Smav data->descr[0].status = status; 800275474Smav 801275474Smav ctl_complete_beio(beio); 802275474Smav} 803275474Smav 804275481Smavstatic uint64_t 805275481Smavctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname) 806275481Smav{ 807275481Smav struct vattr vattr; 808275481Smav struct statfs statfs; 809275481Smav int error; 810275481Smav 811275481Smav if (be_lun->vn == NULL) 812275481Smav return (UINT64_MAX); 813275481Smav if (strcmp(attrname, "blocksused") == 0) { 814275481Smav error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 815275481Smav if (error != 0) 816275481Smav return (UINT64_MAX); 817275481Smav return (vattr.va_bytes >> be_lun->blocksize_shift); 818275481Smav } 819275481Smav if (strcmp(attrname, "blocksavail") == 0) { 820275481Smav error = VFS_STATFS(be_lun->vn->v_mount, &statfs); 821275481Smav if (error != 0) 822275481Smav return (UINT64_MAX); 823275481Smav return ((statfs.f_bavail * statfs.f_bsize) >> 824275481Smav be_lun->blocksize_shift); 825275481Smav } 826275481Smav return (UINT64_MAX); 827275481Smav} 828275481Smav 829275474Smavstatic void 830269123Smavctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun, 831269123Smav struct ctl_be_block_io *beio) 832269123Smav{ 833269123Smav struct ctl_be_block_devdata *dev_data; 834269123Smav union ctl_io *io; 835269123Smav struct uio xuio; 836269123Smav struct iovec *xiovec; 837269123Smav int flags; 838269123Smav int error, i; 839269123Smav 840269123Smav DPRINTF("entered\n"); 841269123Smav 842269123Smav dev_data = &be_lun->backend.dev; 843269123Smav io = beio->io; 844271309Smav flags = 0; 845271309Smav if (ARGS(io)->flags & CTL_LLF_DPO) 846271309Smav flags |= IO_DIRECT; 847271309Smav if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA) 848271309Smav flags |= IO_SYNC; 849269123Smav 850269123Smav bzero(&xuio, sizeof(xuio)); 851269123Smav if (beio->bio_cmd == BIO_READ) { 852269123Smav SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0); 853269123Smav xuio.uio_rw = UIO_READ; 854269123Smav } else { 855269123Smav SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0); 856269123Smav xuio.uio_rw = UIO_WRITE; 857269123Smav } 858269123Smav xuio.uio_offset = beio->io_offset; 859269123Smav xuio.uio_resid = beio->io_len; 860269123Smav xuio.uio_segflg = UIO_SYSSPACE; 861269123Smav xuio.uio_iov = beio->xiovecs; 862269123Smav xuio.uio_iovcnt = beio->num_segs; 863269123Smav xuio.uio_td = curthread; 864269123Smav 865269123Smav for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) { 866269123Smav xiovec->iov_base = beio->sg_segs[i].addr; 867269123Smav xiovec->iov_len = beio->sg_segs[i].len; 868269123Smav } 869269123Smav 870269123Smav binuptime(&beio->ds_t0); 871269123Smav mtx_lock(&be_lun->io_lock); 872269123Smav devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0); 873269123Smav mtx_unlock(&be_lun->io_lock); 874269123Smav 875269123Smav if (beio->bio_cmd == BIO_READ) { 876271309Smav error = (*dev_data->csw->d_read)(dev_data->cdev, &xuio, flags); 877269123Smav SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0); 878269123Smav } else { 879271309Smav error = (*dev_data->csw->d_write)(dev_data->cdev, &xuio, flags); 880269123Smav SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0); 881269123Smav } 882269123Smav 883269123Smav mtx_lock(&be_lun->io_lock); 884269123Smav devstat_end_transaction(beio->lun->disk_stats, beio->io_len, 885269123Smav beio->ds_tag_type, beio->ds_trans_type, 886269123Smav /*now*/ NULL, /*then*/&beio->ds_t0); 887269123Smav mtx_unlock(&be_lun->io_lock); 888269123Smav 889269123Smav /* 890269123Smav * If we got an error, set the sense data to "MEDIUM ERROR" and 891269123Smav * return the I/O to the user. 892269123Smav */ 893269123Smav if (error != 0) { 894273809Smav if (error == ENOSPC) { 895273809Smav ctl_set_space_alloc_fail(&io->scsiio); 896273809Smav } else 897273809Smav ctl_set_medium_error(&io->scsiio); 898269123Smav ctl_complete_beio(beio); 899269123Smav return; 900269123Smav } 901269123Smav 902269123Smav /* 903269123Smav * If this is a write or a verify, we're all done. 904269123Smav * If this is a read, we can now send the data to the user. 905269123Smav */ 906269123Smav if ((beio->bio_cmd == BIO_WRITE) || 907269123Smav (ARGS(io)->flags & CTL_LLF_VERIFY)) { 908269123Smav ctl_set_success(&io->scsiio); 909269123Smav ctl_complete_beio(beio); 910269123Smav } else { 911275058Smav if ((ARGS(io)->flags & CTL_LLF_READ) && 912275058Smav beio->beio_cont == NULL) 913275058Smav ctl_set_success(&io->scsiio); 914269123Smav#ifdef CTL_TIME_IO 915269123Smav getbintime(&io->io_hdr.dma_start_bt); 916269123Smav#endif 917269123Smav ctl_datamove(io); 918269123Smav } 919269123Smav} 920269123Smav 921269123Smavstatic void 922275474Smavctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun, 923275474Smav struct ctl_be_block_io *beio) 924275474Smav{ 925275474Smav struct ctl_be_block_devdata *dev_data = &be_lun->backend.dev; 926275474Smav union ctl_io *io = beio->io; 927275474Smav struct ctl_lba_len_flags *lbalen = ARGS(io); 928275474Smav struct scsi_get_lba_status_data *data; 929275474Smav off_t roff, off; 930275474Smav int error, status; 931275474Smav 932275474Smav DPRINTF("entered\n"); 933275474Smav 934275474Smav off = roff = ((off_t)lbalen->lba) << be_lun->blocksize_shift; 935275474Smav error = (*dev_data->csw->d_ioctl)(dev_data->cdev, FIOSEEKHOLE, 936275474Smav (caddr_t)&off, FREAD, curthread); 937275474Smav if (error == 0 && off > roff) 938275474Smav status = 0; /* mapped up to off */ 939275474Smav else { 940275474Smav error = (*dev_data->csw->d_ioctl)(dev_data->cdev, FIOSEEKDATA, 941275474Smav (caddr_t)&off, FREAD, curthread); 942275474Smav if (error == 0 && off > roff) 943275474Smav status = 1; /* deallocated up to off */ 944275474Smav else { 945275474Smav status = 0; /* unknown up to the end */ 946275474Smav off = be_lun->size_bytes; 947275474Smav } 948275474Smav } 949275474Smav 950275474Smav off >>= be_lun->blocksize_shift; 951275474Smav data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr; 952275474Smav scsi_u64to8b(lbalen->lba, data->descr[0].addr); 953275474Smav scsi_ulto4b(MIN(UINT32_MAX, off - lbalen->lba), 954275474Smav data->descr[0].length); 955275474Smav data->descr[0].status = status; 956275474Smav 957275474Smav ctl_complete_beio(beio); 958275474Smav} 959275474Smav 960275474Smavstatic void 961229997Skenctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun, 962229997Sken struct ctl_be_block_io *beio) 963229997Sken{ 964229997Sken struct bio *bio; 965229997Sken union ctl_io *io; 966229997Sken struct ctl_be_block_devdata *dev_data; 967229997Sken 968229997Sken dev_data = &be_lun->backend.dev; 969229997Sken io = beio->io; 970229997Sken 971229997Sken DPRINTF("entered\n"); 972229997Sken 973229997Sken /* This can't fail, it's a blocking allocation. */ 974229997Sken bio = g_alloc_bio(); 975229997Sken 976229997Sken bio->bio_cmd = BIO_FLUSH; 977229997Sken bio->bio_flags |= BIO_ORDERED; 978229997Sken bio->bio_dev = dev_data->cdev; 979229997Sken bio->bio_offset = 0; 980229997Sken bio->bio_data = 0; 981229997Sken bio->bio_done = ctl_be_block_biodone; 982229997Sken bio->bio_caller1 = beio; 983229997Sken bio->bio_pblkno = 0; 984229997Sken 985229997Sken /* 986229997Sken * We don't need to acquire the LUN lock here, because we are only 987229997Sken * sending one bio, and so there is no other context to synchronize 988229997Sken * with. 989229997Sken */ 990229997Sken beio->num_bios_sent = 1; 991229997Sken beio->send_complete = 1; 992229997Sken 993229997Sken binuptime(&beio->ds_t0); 994267877Smav mtx_lock(&be_lun->io_lock); 995229997Sken devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 996267877Smav mtx_unlock(&be_lun->io_lock); 997229997Sken 998229997Sken (*dev_data->csw->d_strategy)(bio); 999229997Sken} 1000229997Sken 1001229997Skenstatic void 1002264274Smavctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun, 1003264274Smav struct ctl_be_block_io *beio, 1004264274Smav uint64_t off, uint64_t len, int last) 1005264274Smav{ 1006264274Smav struct bio *bio; 1007264274Smav struct ctl_be_block_devdata *dev_data; 1008264296Smav uint64_t maxlen; 1009264274Smav 1010264274Smav dev_data = &be_lun->backend.dev; 1011264296Smav maxlen = LONG_MAX - (LONG_MAX % be_lun->blocksize); 1012264274Smav while (len > 0) { 1013264274Smav bio = g_alloc_bio(); 1014264274Smav bio->bio_cmd = BIO_DELETE; 1015264274Smav bio->bio_dev = dev_data->cdev; 1016264274Smav bio->bio_offset = off; 1017264296Smav bio->bio_length = MIN(len, maxlen); 1018264274Smav bio->bio_data = 0; 1019264274Smav bio->bio_done = ctl_be_block_biodone; 1020264274Smav bio->bio_caller1 = beio; 1021264296Smav bio->bio_pblkno = off / be_lun->blocksize; 1022264274Smav 1023264274Smav off += bio->bio_length; 1024264274Smav len -= bio->bio_length; 1025264274Smav 1026267877Smav mtx_lock(&be_lun->io_lock); 1027264274Smav beio->num_bios_sent++; 1028264274Smav if (last && len == 0) 1029264274Smav beio->send_complete = 1; 1030267877Smav mtx_unlock(&be_lun->io_lock); 1031264274Smav 1032264274Smav (*dev_data->csw->d_strategy)(bio); 1033264274Smav } 1034264274Smav} 1035264274Smav 1036264274Smavstatic void 1037264274Smavctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun, 1038264274Smav struct ctl_be_block_io *beio) 1039264274Smav{ 1040264274Smav union ctl_io *io; 1041264274Smav struct ctl_be_block_devdata *dev_data; 1042267515Smav struct ctl_ptr_len_flags *ptrlen; 1043264274Smav struct scsi_unmap_desc *buf, *end; 1044264274Smav uint64_t len; 1045264274Smav 1046264274Smav dev_data = &be_lun->backend.dev; 1047264274Smav io = beio->io; 1048264274Smav 1049264274Smav DPRINTF("entered\n"); 1050264274Smav 1051264274Smav binuptime(&beio->ds_t0); 1052267877Smav mtx_lock(&be_lun->io_lock); 1053264274Smav devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 1054267877Smav mtx_unlock(&be_lun->io_lock); 1055264274Smav 1056264274Smav if (beio->io_offset == -1) { 1057264274Smav beio->io_len = 0; 1058267515Smav ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 1059267515Smav buf = (struct scsi_unmap_desc *)ptrlen->ptr; 1060267515Smav end = buf + ptrlen->len / sizeof(*buf); 1061264274Smav for (; buf < end; buf++) { 1062264274Smav len = (uint64_t)scsi_4btoul(buf->length) * 1063264274Smav be_lun->blocksize; 1064264274Smav beio->io_len += len; 1065264274Smav ctl_be_block_unmap_dev_range(be_lun, beio, 1066264274Smav scsi_8btou64(buf->lba) * be_lun->blocksize, len, 1067264283Smav (end - buf < 2) ? TRUE : FALSE); 1068264274Smav } 1069264274Smav } else 1070264274Smav ctl_be_block_unmap_dev_range(be_lun, beio, 1071264274Smav beio->io_offset, beio->io_len, TRUE); 1072264274Smav} 1073264274Smav 1074264274Smavstatic void 1075229997Skenctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun, 1076229997Sken struct ctl_be_block_io *beio) 1077229997Sken{ 1078267877Smav TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue); 1079229997Sken int i; 1080229997Sken struct bio *bio; 1081229997Sken struct ctl_be_block_devdata *dev_data; 1082229997Sken off_t cur_offset; 1083229997Sken int max_iosize; 1084229997Sken 1085229997Sken DPRINTF("entered\n"); 1086229997Sken 1087229997Sken dev_data = &be_lun->backend.dev; 1088229997Sken 1089229997Sken /* 1090229997Sken * We have to limit our I/O size to the maximum supported by the 1091229997Sken * backend device. Hopefully it is MAXPHYS. If the driver doesn't 1092229997Sken * set it properly, use DFLTPHYS. 1093229997Sken */ 1094229997Sken max_iosize = dev_data->cdev->si_iosize_max; 1095229997Sken if (max_iosize < PAGE_SIZE) 1096229997Sken max_iosize = DFLTPHYS; 1097229997Sken 1098229997Sken cur_offset = beio->io_offset; 1099229997Sken for (i = 0; i < beio->num_segs; i++) { 1100229997Sken size_t cur_size; 1101229997Sken uint8_t *cur_ptr; 1102229997Sken 1103229997Sken cur_size = beio->sg_segs[i].len; 1104229997Sken cur_ptr = beio->sg_segs[i].addr; 1105229997Sken 1106229997Sken while (cur_size > 0) { 1107229997Sken /* This can't fail, it's a blocking allocation. */ 1108229997Sken bio = g_alloc_bio(); 1109229997Sken 1110229997Sken KASSERT(bio != NULL, ("g_alloc_bio() failed!\n")); 1111229997Sken 1112229997Sken bio->bio_cmd = beio->bio_cmd; 1113229997Sken bio->bio_dev = dev_data->cdev; 1114229997Sken bio->bio_caller1 = beio; 1115229997Sken bio->bio_length = min(cur_size, max_iosize); 1116229997Sken bio->bio_offset = cur_offset; 1117229997Sken bio->bio_data = cur_ptr; 1118229997Sken bio->bio_done = ctl_be_block_biodone; 1119229997Sken bio->bio_pblkno = cur_offset / be_lun->blocksize; 1120229997Sken 1121229997Sken cur_offset += bio->bio_length; 1122229997Sken cur_ptr += bio->bio_length; 1123229997Sken cur_size -= bio->bio_length; 1124229997Sken 1125267877Smav TAILQ_INSERT_TAIL(&queue, bio, bio_queue); 1126229997Sken beio->num_bios_sent++; 1127229997Sken } 1128229997Sken } 1129267877Smav binuptime(&beio->ds_t0); 1130267877Smav mtx_lock(&be_lun->io_lock); 1131267877Smav devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0); 1132267877Smav beio->send_complete = 1; 1133267877Smav mtx_unlock(&be_lun->io_lock); 1134267877Smav 1135267877Smav /* 1136267877Smav * Fire off all allocated requests! 1137267877Smav */ 1138267877Smav while ((bio = TAILQ_FIRST(&queue)) != NULL) { 1139267877Smav TAILQ_REMOVE(&queue, bio, bio_queue); 1140267877Smav (*dev_data->csw->d_strategy)(bio); 1141267877Smav } 1142229997Sken} 1143229997Sken 1144274154Smavstatic uint64_t 1145274154Smavctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname) 1146274154Smav{ 1147274154Smav struct ctl_be_block_devdata *dev_data = &be_lun->backend.dev; 1148274154Smav struct diocgattr_arg arg; 1149274154Smav int error; 1150274154Smav 1151274154Smav if (dev_data->csw == NULL || dev_data->csw->d_ioctl == NULL) 1152274154Smav return (UINT64_MAX); 1153274154Smav strlcpy(arg.name, attrname, sizeof(arg.name)); 1154274154Smav arg.len = sizeof(arg.value.off); 1155274154Smav error = dev_data->csw->d_ioctl(dev_data->cdev, 1156274154Smav DIOCGATTR, (caddr_t)&arg, FREAD, curthread); 1157274154Smav if (error != 0) 1158274154Smav return (UINT64_MAX); 1159274154Smav return (arg.value.off); 1160274154Smav} 1161274154Smav 1162229997Skenstatic void 1163264274Smavctl_be_block_cw_done_ws(struct ctl_be_block_io *beio) 1164264274Smav{ 1165264274Smav union ctl_io *io; 1166264274Smav 1167264274Smav io = beio->io; 1168264274Smav ctl_free_beio(beio); 1169267641Smav if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1170267641Smav ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1171267641Smav (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 1172264274Smav ctl_config_write_done(io); 1173264274Smav return; 1174264274Smav } 1175264274Smav 1176264274Smav ctl_be_block_config_write(io); 1177264274Smav} 1178264274Smav 1179264274Smavstatic void 1180264274Smavctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun, 1181264274Smav union ctl_io *io) 1182264274Smav{ 1183264274Smav struct ctl_be_block_io *beio; 1184264274Smav struct ctl_be_block_softc *softc; 1185267515Smav struct ctl_lba_len_flags *lbalen; 1186264274Smav uint64_t len_left, lba; 1187264274Smav int i, seglen; 1188264274Smav uint8_t *buf, *end; 1189264274Smav 1190264274Smav DPRINTF("entered\n"); 1191264274Smav 1192267519Smav beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1193264274Smav softc = be_lun->softc; 1194267537Smav lbalen = ARGS(beio->io); 1195264274Smav 1196271839Smav if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) || 1197269622Smav (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) { 1198264274Smav ctl_free_beio(beio); 1199264274Smav ctl_set_invalid_field(&io->scsiio, 1200264274Smav /*sks_valid*/ 1, 1201264274Smav /*command*/ 1, 1202264274Smav /*field*/ 1, 1203264274Smav /*bit_valid*/ 0, 1204264274Smav /*bit*/ 0); 1205264274Smav ctl_config_write_done(io); 1206264274Smav return; 1207264274Smav } 1208264274Smav 1209264274Smav switch (io->scsiio.tag_type) { 1210264274Smav case CTL_TAG_ORDERED: 1211264274Smav beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1212264274Smav break; 1213264274Smav case CTL_TAG_HEAD_OF_QUEUE: 1214264274Smav beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1215264274Smav break; 1216264274Smav case CTL_TAG_UNTAGGED: 1217264274Smav case CTL_TAG_SIMPLE: 1218264274Smav case CTL_TAG_ACA: 1219264274Smav default: 1220264274Smav beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1221264274Smav break; 1222264274Smav } 1223264274Smav 1224269622Smav if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) { 1225267515Smav beio->io_offset = lbalen->lba * be_lun->blocksize; 1226267515Smav beio->io_len = (uint64_t)lbalen->len * be_lun->blocksize; 1227264274Smav beio->bio_cmd = BIO_DELETE; 1228264274Smav beio->ds_trans_type = DEVSTAT_FREE; 1229264274Smav 1230264274Smav be_lun->unmap(be_lun, beio); 1231264274Smav return; 1232264274Smav } 1233264274Smav 1234264274Smav beio->bio_cmd = BIO_WRITE; 1235264274Smav beio->ds_trans_type = DEVSTAT_WRITE; 1236264274Smav 1237264274Smav DPRINTF("WRITE SAME at LBA %jx len %u\n", 1238267515Smav (uintmax_t)lbalen->lba, lbalen->len); 1239264274Smav 1240267515Smav len_left = (uint64_t)lbalen->len * be_lun->blocksize; 1241264274Smav for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) { 1242264274Smav 1243264274Smav /* 1244264274Smav * Setup the S/G entry for this chunk. 1245264274Smav */ 1246264886Smav seglen = MIN(CTLBLK_MAX_SEG, len_left); 1247264274Smav seglen -= seglen % be_lun->blocksize; 1248264274Smav beio->sg_segs[i].len = seglen; 1249264274Smav beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1250264274Smav 1251264274Smav DPRINTF("segment %d addr %p len %zd\n", i, 1252264274Smav beio->sg_segs[i].addr, beio->sg_segs[i].len); 1253264274Smav 1254264274Smav beio->num_segs++; 1255264274Smav len_left -= seglen; 1256264274Smav 1257264274Smav buf = beio->sg_segs[i].addr; 1258264274Smav end = buf + seglen; 1259264274Smav for (; buf < end; buf += be_lun->blocksize) { 1260264274Smav memcpy(buf, io->scsiio.kern_data_ptr, be_lun->blocksize); 1261267515Smav if (lbalen->flags & SWS_LBDATA) 1262267515Smav scsi_ulto4b(lbalen->lba + lba, buf); 1263264274Smav lba++; 1264264274Smav } 1265264274Smav } 1266264274Smav 1267267515Smav beio->io_offset = lbalen->lba * be_lun->blocksize; 1268264274Smav beio->io_len = lba * be_lun->blocksize; 1269264274Smav 1270264274Smav /* We can not do all in one run. Correct and schedule rerun. */ 1271264274Smav if (len_left > 0) { 1272267515Smav lbalen->lba += lba; 1273267515Smav lbalen->len -= lba; 1274264274Smav beio->beio_cont = ctl_be_block_cw_done_ws; 1275264274Smav } 1276264274Smav 1277264274Smav be_lun->dispatch(be_lun, beio); 1278264274Smav} 1279264274Smav 1280264274Smavstatic void 1281264274Smavctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun, 1282264274Smav union ctl_io *io) 1283264274Smav{ 1284264274Smav struct ctl_be_block_io *beio; 1285264274Smav struct ctl_be_block_softc *softc; 1286267515Smav struct ctl_ptr_len_flags *ptrlen; 1287264274Smav 1288264274Smav DPRINTF("entered\n"); 1289264274Smav 1290267519Smav beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1291264274Smav softc = be_lun->softc; 1292267515Smav ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN]; 1293264274Smav 1294269622Smav if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) { 1295264274Smav ctl_free_beio(beio); 1296264274Smav ctl_set_invalid_field(&io->scsiio, 1297264274Smav /*sks_valid*/ 0, 1298264274Smav /*command*/ 1, 1299264274Smav /*field*/ 0, 1300264274Smav /*bit_valid*/ 0, 1301264274Smav /*bit*/ 0); 1302264274Smav ctl_config_write_done(io); 1303264274Smav return; 1304264274Smav } 1305264274Smav 1306264274Smav switch (io->scsiio.tag_type) { 1307264274Smav case CTL_TAG_ORDERED: 1308264274Smav beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1309264274Smav break; 1310264274Smav case CTL_TAG_HEAD_OF_QUEUE: 1311264274Smav beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1312264274Smav break; 1313264274Smav case CTL_TAG_UNTAGGED: 1314264274Smav case CTL_TAG_SIMPLE: 1315264274Smav case CTL_TAG_ACA: 1316264274Smav default: 1317264274Smav beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1318264274Smav break; 1319264274Smav } 1320264274Smav 1321264274Smav beio->io_len = 0; 1322264274Smav beio->io_offset = -1; 1323264274Smav 1324264274Smav beio->bio_cmd = BIO_DELETE; 1325264274Smav beio->ds_trans_type = DEVSTAT_FREE; 1326264274Smav 1327267515Smav DPRINTF("UNMAP\n"); 1328264274Smav 1329264274Smav be_lun->unmap(be_lun, beio); 1330264274Smav} 1331264274Smav 1332264274Smavstatic void 1333275474Smavctl_be_block_cr_done(struct ctl_be_block_io *beio) 1334275474Smav{ 1335275474Smav union ctl_io *io; 1336275474Smav 1337275474Smav io = beio->io; 1338275474Smav ctl_free_beio(beio); 1339275474Smav ctl_config_read_done(io); 1340275474Smav} 1341275474Smav 1342275474Smavstatic void 1343275474Smavctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun, 1344275474Smav union ctl_io *io) 1345275474Smav{ 1346275474Smav struct ctl_be_block_io *beio; 1347275474Smav struct ctl_be_block_softc *softc; 1348275474Smav 1349275474Smav DPRINTF("entered\n"); 1350275474Smav 1351275474Smav softc = be_lun->softc; 1352275474Smav beio = ctl_alloc_beio(softc); 1353275474Smav beio->io = io; 1354275474Smav beio->lun = be_lun; 1355275474Smav beio->beio_cont = ctl_be_block_cr_done; 1356275474Smav PRIV(io)->ptr = (void *)beio; 1357275474Smav 1358275474Smav switch (io->scsiio.cdb[0]) { 1359275474Smav case SERVICE_ACTION_IN: /* GET LBA STATUS */ 1360275474Smav beio->bio_cmd = -1; 1361275474Smav beio->ds_trans_type = DEVSTAT_NO_DATA; 1362275474Smav beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1363275474Smav beio->io_len = 0; 1364275474Smav if (be_lun->get_lba_status) 1365275474Smav be_lun->get_lba_status(be_lun, beio); 1366275474Smav else 1367275474Smav ctl_be_block_cr_done(beio); 1368275474Smav break; 1369275474Smav default: 1370275474Smav panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1371275474Smav break; 1372275474Smav } 1373275474Smav} 1374275474Smav 1375275474Smavstatic void 1376264274Smavctl_be_block_cw_done(struct ctl_be_block_io *beio) 1377264274Smav{ 1378264274Smav union ctl_io *io; 1379264274Smav 1380264274Smav io = beio->io; 1381264274Smav ctl_free_beio(beio); 1382264274Smav ctl_config_write_done(io); 1383264274Smav} 1384264274Smav 1385264274Smavstatic void 1386229997Skenctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun, 1387229997Sken union ctl_io *io) 1388229997Sken{ 1389229997Sken struct ctl_be_block_io *beio; 1390229997Sken struct ctl_be_block_softc *softc; 1391229997Sken 1392229997Sken DPRINTF("entered\n"); 1393229997Sken 1394229997Sken softc = be_lun->softc; 1395229997Sken beio = ctl_alloc_beio(softc); 1396229997Sken beio->io = io; 1397229997Sken beio->lun = be_lun; 1398264274Smav beio->beio_cont = ctl_be_block_cw_done; 1399267519Smav PRIV(io)->ptr = (void *)beio; 1400229997Sken 1401229997Sken switch (io->scsiio.cdb[0]) { 1402229997Sken case SYNCHRONIZE_CACHE: 1403229997Sken case SYNCHRONIZE_CACHE_16: 1404249194Strasz beio->bio_cmd = BIO_FLUSH; 1405229997Sken beio->ds_trans_type = DEVSTAT_NO_DATA; 1406229997Sken beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1407229997Sken beio->io_len = 0; 1408229997Sken be_lun->lun_flush(be_lun, beio); 1409229997Sken break; 1410264274Smav case WRITE_SAME_10: 1411264274Smav case WRITE_SAME_16: 1412264274Smav ctl_be_block_cw_dispatch_ws(be_lun, io); 1413264274Smav break; 1414264274Smav case UNMAP: 1415264274Smav ctl_be_block_cw_dispatch_unmap(be_lun, io); 1416264274Smav break; 1417229997Sken default: 1418229997Sken panic("Unhandled CDB type %#x", io->scsiio.cdb[0]); 1419229997Sken break; 1420229997Sken } 1421229997Sken} 1422229997Sken 1423258622SavgSDT_PROBE_DEFINE1(cbb, kernel, read, start, "uint64_t"); 1424258622SavgSDT_PROBE_DEFINE1(cbb, kernel, write, start, "uint64_t"); 1425258622SavgSDT_PROBE_DEFINE1(cbb, kernel, read, alloc_done, "uint64_t"); 1426258622SavgSDT_PROBE_DEFINE1(cbb, kernel, write, alloc_done, "uint64_t"); 1427229997Sken 1428229997Skenstatic void 1429264886Smavctl_be_block_next(struct ctl_be_block_io *beio) 1430264886Smav{ 1431264886Smav struct ctl_be_block_lun *be_lun; 1432264886Smav union ctl_io *io; 1433264886Smav 1434264886Smav io = beio->io; 1435264886Smav be_lun = beio->lun; 1436264886Smav ctl_free_beio(beio); 1437267641Smav if ((io->io_hdr.flags & CTL_FLAG_ABORT) || 1438267641Smav ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE && 1439267641Smav (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) { 1440267537Smav ctl_data_submit_done(io); 1441264886Smav return; 1442264886Smav } 1443264886Smav 1444264886Smav io->io_hdr.status &= ~CTL_STATUS_MASK; 1445264886Smav io->io_hdr.status |= CTL_STATUS_NONE; 1446264886Smav 1447267877Smav mtx_lock(&be_lun->queue_lock); 1448264886Smav /* 1449264886Smav * XXX KDM make sure that links is okay to use at this point. 1450264886Smav * Otherwise, we either need to add another field to ctl_io_hdr, 1451264886Smav * or deal with resource allocation here. 1452264886Smav */ 1453264886Smav STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 1454267877Smav mtx_unlock(&be_lun->queue_lock); 1455264886Smav 1456264886Smav taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 1457264886Smav} 1458264886Smav 1459264886Smavstatic void 1460229997Skenctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, 1461229997Sken union ctl_io *io) 1462229997Sken{ 1463229997Sken struct ctl_be_block_io *beio; 1464229997Sken struct ctl_be_block_softc *softc; 1465267537Smav struct ctl_lba_len_flags *lbalen; 1466267519Smav struct ctl_ptr_len_flags *bptrlen; 1467267519Smav uint64_t len_left, lbas; 1468229997Sken int i; 1469229997Sken 1470229997Sken softc = be_lun->softc; 1471229997Sken 1472229997Sken DPRINTF("entered\n"); 1473229997Sken 1474267537Smav lbalen = ARGS(io); 1475267537Smav if (lbalen->flags & CTL_LLF_WRITE) { 1476267537Smav SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0); 1477267537Smav } else { 1478229997Sken SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0); 1479229997Sken } 1480229997Sken 1481229997Sken beio = ctl_alloc_beio(softc); 1482229997Sken beio->io = io; 1483229997Sken beio->lun = be_lun; 1484267519Smav bptrlen = PRIV(io); 1485267519Smav bptrlen->ptr = (void *)beio; 1486229997Sken 1487229997Sken switch (io->scsiio.tag_type) { 1488229997Sken case CTL_TAG_ORDERED: 1489229997Sken beio->ds_tag_type = DEVSTAT_TAG_ORDERED; 1490229997Sken break; 1491229997Sken case CTL_TAG_HEAD_OF_QUEUE: 1492229997Sken beio->ds_tag_type = DEVSTAT_TAG_HEAD; 1493229997Sken break; 1494229997Sken case CTL_TAG_UNTAGGED: 1495229997Sken case CTL_TAG_SIMPLE: 1496229997Sken case CTL_TAG_ACA: 1497229997Sken default: 1498229997Sken beio->ds_tag_type = DEVSTAT_TAG_SIMPLE; 1499229997Sken break; 1500229997Sken } 1501229997Sken 1502267537Smav if (lbalen->flags & CTL_LLF_WRITE) { 1503267537Smav beio->bio_cmd = BIO_WRITE; 1504267537Smav beio->ds_trans_type = DEVSTAT_WRITE; 1505267537Smav } else { 1506229997Sken beio->bio_cmd = BIO_READ; 1507229997Sken beio->ds_trans_type = DEVSTAT_READ; 1508229997Sken } 1509229997Sken 1510264886Smav DPRINTF("%s at LBA %jx len %u @%ju\n", 1511229997Sken (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", 1512267519Smav (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len); 1513267537Smav if (lbalen->flags & CTL_LLF_COMPARE) 1514267537Smav lbas = CTLBLK_HALF_IO_SIZE; 1515267537Smav else 1516267537Smav lbas = CTLBLK_MAX_IO_SIZE; 1517267537Smav lbas = MIN(lbalen->len - bptrlen->len, lbas / be_lun->blocksize); 1518267519Smav beio->io_offset = (lbalen->lba + bptrlen->len) * be_lun->blocksize; 1519267519Smav beio->io_len = lbas * be_lun->blocksize; 1520267519Smav bptrlen->len += lbas; 1521229997Sken 1522264886Smav for (i = 0, len_left = beio->io_len; len_left > 0; i++) { 1523264886Smav KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)", 1524264886Smav i, CTLBLK_MAX_SEGS)); 1525229997Sken 1526229997Sken /* 1527229997Sken * Setup the S/G entry for this chunk. 1528229997Sken */ 1529264886Smav beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left); 1530229997Sken beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK); 1531229997Sken 1532229997Sken DPRINTF("segment %d addr %p len %zd\n", i, 1533229997Sken beio->sg_segs[i].addr, beio->sg_segs[i].len); 1534229997Sken 1535267537Smav /* Set up second segment for compare operation. */ 1536267537Smav if (lbalen->flags & CTL_LLF_COMPARE) { 1537267537Smav beio->sg_segs[i + CTLBLK_HALF_SEGS].len = 1538267537Smav beio->sg_segs[i].len; 1539267537Smav beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = 1540267537Smav uma_zalloc(be_lun->lun_zone, M_WAITOK); 1541267537Smav } 1542267537Smav 1543229997Sken beio->num_segs++; 1544229997Sken len_left -= beio->sg_segs[i].len; 1545229997Sken } 1546267519Smav if (bptrlen->len < lbalen->len) 1547264886Smav beio->beio_cont = ctl_be_block_next; 1548264886Smav io->scsiio.be_move_done = ctl_be_block_move_done; 1549267537Smav /* For compare we have separate S/G lists for read and datamove. */ 1550267537Smav if (lbalen->flags & CTL_LLF_COMPARE) 1551267537Smav io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS]; 1552267537Smav else 1553267537Smav io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs; 1554264886Smav io->scsiio.kern_data_len = beio->io_len; 1555264886Smav io->scsiio.kern_data_resid = 0; 1556264886Smav io->scsiio.kern_sg_entries = beio->num_segs; 1557264886Smav io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST; 1558229997Sken 1559229997Sken /* 1560229997Sken * For the read case, we need to read the data into our buffers and 1561229997Sken * then we can send it back to the user. For the write case, we 1562229997Sken * need to get the data from the user first. 1563229997Sken */ 1564229997Sken if (beio->bio_cmd == BIO_READ) { 1565229997Sken SDT_PROBE(cbb, kernel, read, alloc_done, 0, 0, 0, 0, 0); 1566229997Sken be_lun->dispatch(be_lun, beio); 1567229997Sken } else { 1568229997Sken SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0); 1569229997Sken#ifdef CTL_TIME_IO 1570229997Sken getbintime(&io->io_hdr.dma_start_bt); 1571229997Sken#endif 1572229997Sken ctl_datamove(io); 1573229997Sken } 1574229997Sken} 1575229997Sken 1576229997Skenstatic void 1577229997Skenctl_be_block_worker(void *context, int pending) 1578229997Sken{ 1579229997Sken struct ctl_be_block_lun *be_lun; 1580229997Sken struct ctl_be_block_softc *softc; 1581229997Sken union ctl_io *io; 1582229997Sken 1583229997Sken be_lun = (struct ctl_be_block_lun *)context; 1584229997Sken softc = be_lun->softc; 1585229997Sken 1586229997Sken DPRINTF("entered\n"); 1587229997Sken 1588267877Smav mtx_lock(&be_lun->queue_lock); 1589229997Sken for (;;) { 1590229997Sken io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue); 1591229997Sken if (io != NULL) { 1592229997Sken struct ctl_be_block_io *beio; 1593229997Sken 1594229997Sken DPRINTF("datamove queue\n"); 1595229997Sken 1596229997Sken STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr, 1597229997Sken ctl_io_hdr, links); 1598229997Sken 1599267877Smav mtx_unlock(&be_lun->queue_lock); 1600229997Sken 1601267519Smav beio = (struct ctl_be_block_io *)PRIV(io)->ptr; 1602229997Sken 1603229997Sken be_lun->dispatch(be_lun, beio); 1604229997Sken 1605267877Smav mtx_lock(&be_lun->queue_lock); 1606229997Sken continue; 1607229997Sken } 1608229997Sken io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue); 1609229997Sken if (io != NULL) { 1610229997Sken DPRINTF("config write queue\n"); 1611229997Sken STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr, 1612229997Sken ctl_io_hdr, links); 1613267877Smav mtx_unlock(&be_lun->queue_lock); 1614229997Sken ctl_be_block_cw_dispatch(be_lun, io); 1615267877Smav mtx_lock(&be_lun->queue_lock); 1616229997Sken continue; 1617229997Sken } 1618275474Smav io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue); 1619275474Smav if (io != NULL) { 1620275474Smav DPRINTF("config read queue\n"); 1621275474Smav STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr, 1622275474Smav ctl_io_hdr, links); 1623275474Smav mtx_unlock(&be_lun->queue_lock); 1624275474Smav ctl_be_block_cr_dispatch(be_lun, io); 1625275474Smav mtx_lock(&be_lun->queue_lock); 1626275474Smav continue; 1627275474Smav } 1628229997Sken io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue); 1629229997Sken if (io != NULL) { 1630229997Sken DPRINTF("input queue\n"); 1631229997Sken 1632229997Sken STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr, 1633229997Sken ctl_io_hdr, links); 1634267877Smav mtx_unlock(&be_lun->queue_lock); 1635229997Sken 1636229997Sken /* 1637229997Sken * We must drop the lock, since this routine and 1638229997Sken * its children may sleep. 1639229997Sken */ 1640229997Sken ctl_be_block_dispatch(be_lun, io); 1641229997Sken 1642267877Smav mtx_lock(&be_lun->queue_lock); 1643229997Sken continue; 1644229997Sken } 1645229997Sken 1646229997Sken /* 1647229997Sken * If we get here, there is no work left in the queues, so 1648229997Sken * just break out and let the task queue go to sleep. 1649229997Sken */ 1650229997Sken break; 1651229997Sken } 1652267877Smav mtx_unlock(&be_lun->queue_lock); 1653229997Sken} 1654229997Sken 1655229997Sken/* 1656229997Sken * Entry point from CTL to the backend for I/O. We queue everything to a 1657229997Sken * work thread, so this just puts the I/O on a queue and wakes up the 1658229997Sken * thread. 1659229997Sken */ 1660229997Skenstatic int 1661229997Skenctl_be_block_submit(union ctl_io *io) 1662229997Sken{ 1663229997Sken struct ctl_be_block_lun *be_lun; 1664229997Sken struct ctl_be_lun *ctl_be_lun; 1665229997Sken 1666229997Sken DPRINTF("entered\n"); 1667229997Sken 1668229997Sken ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 1669229997Sken CTL_PRIV_BACKEND_LUN].ptr; 1670229997Sken be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 1671229997Sken 1672229997Sken /* 1673229997Sken * Make sure we only get SCSI I/O. 1674229997Sken */ 1675229997Sken KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type " 1676229997Sken "%#x) encountered", io->io_hdr.io_type)); 1677229997Sken 1678267519Smav PRIV(io)->len = 0; 1679267519Smav 1680267877Smav mtx_lock(&be_lun->queue_lock); 1681229997Sken /* 1682229997Sken * XXX KDM make sure that links is okay to use at this point. 1683229997Sken * Otherwise, we either need to add another field to ctl_io_hdr, 1684229997Sken * or deal with resource allocation here. 1685229997Sken */ 1686229997Sken STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links); 1687267877Smav mtx_unlock(&be_lun->queue_lock); 1688229997Sken taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 1689229997Sken 1690267514Smav return (CTL_RETVAL_COMPLETE); 1691229997Sken} 1692229997Sken 1693229997Skenstatic int 1694229997Skenctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 1695229997Sken int flag, struct thread *td) 1696229997Sken{ 1697229997Sken struct ctl_be_block_softc *softc; 1698229997Sken int error; 1699229997Sken 1700229997Sken softc = &backend_block_softc; 1701229997Sken 1702229997Sken error = 0; 1703229997Sken 1704229997Sken switch (cmd) { 1705229997Sken case CTL_LUN_REQ: { 1706229997Sken struct ctl_lun_req *lun_req; 1707229997Sken 1708229997Sken lun_req = (struct ctl_lun_req *)addr; 1709229997Sken 1710229997Sken switch (lun_req->reqtype) { 1711229997Sken case CTL_LUNREQ_CREATE: 1712229997Sken error = ctl_be_block_create(softc, lun_req); 1713229997Sken break; 1714229997Sken case CTL_LUNREQ_RM: 1715229997Sken error = ctl_be_block_rm(softc, lun_req); 1716229997Sken break; 1717232604Strasz case CTL_LUNREQ_MODIFY: 1718232604Strasz error = ctl_be_block_modify(softc, lun_req); 1719232604Strasz break; 1720229997Sken default: 1721229997Sken lun_req->status = CTL_LUN_ERROR; 1722229997Sken snprintf(lun_req->error_str, sizeof(lun_req->error_str), 1723272911Smav "invalid LUN request type %d", 1724229997Sken lun_req->reqtype); 1725229997Sken break; 1726229997Sken } 1727229997Sken break; 1728229997Sken } 1729229997Sken default: 1730229997Sken error = ENOTTY; 1731229997Sken break; 1732229997Sken } 1733229997Sken 1734229997Sken return (error); 1735229997Sken} 1736229997Sken 1737229997Skenstatic int 1738229997Skenctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1739229997Sken{ 1740229997Sken struct ctl_be_block_filedata *file_data; 1741229997Sken struct ctl_lun_create_params *params; 1742229997Sken struct vattr vattr; 1743273029Smav off_t pss; 1744229997Sken int error; 1745229997Sken 1746229997Sken error = 0; 1747229997Sken file_data = &be_lun->backend.file; 1748272911Smav params = &be_lun->params; 1749229997Sken 1750229997Sken be_lun->dev_type = CTL_BE_BLOCK_FILE; 1751229997Sken be_lun->dispatch = ctl_be_block_dispatch_file; 1752229997Sken be_lun->lun_flush = ctl_be_block_flush_file; 1753275474Smav be_lun->get_lba_status = ctl_be_block_gls_file; 1754275481Smav be_lun->getattr = ctl_be_block_getattr_file; 1755229997Sken 1756229997Sken error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 1757229997Sken if (error != 0) { 1758229997Sken snprintf(req->error_str, sizeof(req->error_str), 1759229997Sken "error calling VOP_GETATTR() for file %s", 1760229997Sken be_lun->dev_path); 1761229997Sken return (error); 1762229997Sken } 1763229997Sken 1764229997Sken /* 1765229997Sken * Verify that we have the ability to upgrade to exclusive 1766229997Sken * access on this file so we can trap errors at open instead 1767229997Sken * of reporting them during first access. 1768229997Sken */ 1769229997Sken if (VOP_ISLOCKED(be_lun->vn) != LK_EXCLUSIVE) { 1770229997Sken vn_lock(be_lun->vn, LK_UPGRADE | LK_RETRY); 1771229997Sken if (be_lun->vn->v_iflag & VI_DOOMED) { 1772229997Sken error = EBADF; 1773229997Sken snprintf(req->error_str, sizeof(req->error_str), 1774229997Sken "error locking file %s", be_lun->dev_path); 1775229997Sken return (error); 1776229997Sken } 1777229997Sken } 1778229997Sken 1779229997Sken 1780229997Sken file_data->cred = crhold(curthread->td_ucred); 1781232604Strasz if (params->lun_size_bytes != 0) 1782232604Strasz be_lun->size_bytes = params->lun_size_bytes; 1783232604Strasz else 1784232604Strasz be_lun->size_bytes = vattr.va_size; 1785229997Sken /* 1786229997Sken * We set the multi thread flag for file operations because all 1787229997Sken * filesystems (in theory) are capable of allowing multiple readers 1788229997Sken * of a file at once. So we want to get the maximum possible 1789229997Sken * concurrency. 1790229997Sken */ 1791229997Sken be_lun->flags |= CTL_BE_BLOCK_LUN_MULTI_THREAD; 1792229997Sken 1793229997Sken /* 1794273029Smav * For files we can use any logical block size. Prefer 512 bytes 1795273029Smav * for compatibility reasons. If file's vattr.va_blocksize 1796273029Smav * (preferred I/O block size) is bigger and multiple to chosen 1797273029Smav * logical block size -- report it as physical block size. 1798229997Sken */ 1799229997Sken if (params->blocksize_bytes != 0) 1800229997Sken be_lun->blocksize = params->blocksize_bytes; 1801229997Sken else 1802229997Sken be_lun->blocksize = 512; 1803273029Smav pss = vattr.va_blocksize / be_lun->blocksize; 1804273029Smav if ((pss > 0) && (pss * be_lun->blocksize == vattr.va_blocksize) && 1805273029Smav ((pss & (pss - 1)) == 0)) { 1806273029Smav be_lun->pblockexp = fls(pss) - 1; 1807273029Smav be_lun->pblockoff = 0; 1808273029Smav } 1809229997Sken 1810229997Sken /* 1811229997Sken * Sanity check. The media size has to be at least one 1812229997Sken * sector long. 1813229997Sken */ 1814229997Sken if (be_lun->size_bytes < be_lun->blocksize) { 1815229997Sken error = EINVAL; 1816229997Sken snprintf(req->error_str, sizeof(req->error_str), 1817229997Sken "file %s size %ju < block size %u", be_lun->dev_path, 1818229997Sken (uintmax_t)be_lun->size_bytes, be_lun->blocksize); 1819229997Sken } 1820229997Sken return (error); 1821229997Sken} 1822229997Sken 1823229997Skenstatic int 1824229997Skenctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 1825229997Sken{ 1826229997Sken struct ctl_lun_create_params *params; 1827229997Sken struct vattr vattr; 1828229997Sken struct cdev *dev; 1829229997Sken struct cdevsw *devsw; 1830229997Sken int error; 1831264191Smav off_t ps, pss, po, pos; 1832229997Sken 1833272911Smav params = &be_lun->params; 1834229997Sken 1835229997Sken be_lun->dev_type = CTL_BE_BLOCK_DEV; 1836229997Sken be_lun->backend.dev.cdev = be_lun->vn->v_rdev; 1837229997Sken be_lun->backend.dev.csw = dev_refthread(be_lun->backend.dev.cdev, 1838229997Sken &be_lun->backend.dev.dev_ref); 1839229997Sken if (be_lun->backend.dev.csw == NULL) 1840229997Sken panic("Unable to retrieve device switch"); 1841275474Smav if (strcmp(be_lun->backend.dev.csw->d_name, "zvol") == 0) { 1842269123Smav be_lun->dispatch = ctl_be_block_dispatch_zvol; 1843275474Smav be_lun->get_lba_status = ctl_be_block_gls_zvol; 1844275474Smav } else 1845269123Smav be_lun->dispatch = ctl_be_block_dispatch_dev; 1846269123Smav be_lun->lun_flush = ctl_be_block_flush_dev; 1847269123Smav be_lun->unmap = ctl_be_block_unmap_dev; 1848274154Smav be_lun->getattr = ctl_be_block_getattr_dev; 1849229997Sken 1850229997Sken error = VOP_GETATTR(be_lun->vn, &vattr, NOCRED); 1851229997Sken if (error) { 1852229997Sken snprintf(req->error_str, sizeof(req->error_str), 1853272911Smav "error getting vnode attributes for device %s", 1854272911Smav be_lun->dev_path); 1855229997Sken return (error); 1856229997Sken } 1857229997Sken 1858229997Sken dev = be_lun->vn->v_rdev; 1859229997Sken devsw = dev->si_devsw; 1860229997Sken if (!devsw->d_ioctl) { 1861229997Sken snprintf(req->error_str, sizeof(req->error_str), 1862272911Smav "no d_ioctl for device %s!", 1863229997Sken be_lun->dev_path); 1864229997Sken return (ENODEV); 1865229997Sken } 1866229997Sken 1867229997Sken error = devsw->d_ioctl(dev, DIOCGSECTORSIZE, 1868229997Sken (caddr_t)&be_lun->blocksize, FREAD, 1869229997Sken curthread); 1870229997Sken if (error) { 1871229997Sken snprintf(req->error_str, sizeof(req->error_str), 1872272911Smav "error %d returned for DIOCGSECTORSIZE ioctl " 1873272911Smav "on %s!", error, be_lun->dev_path); 1874229997Sken return (error); 1875229997Sken } 1876229997Sken 1877229997Sken /* 1878229997Sken * If the user has asked for a blocksize that is greater than the 1879229997Sken * backing device's blocksize, we can do it only if the blocksize 1880229997Sken * the user is asking for is an even multiple of the underlying 1881229997Sken * device's blocksize. 1882229997Sken */ 1883229997Sken if ((params->blocksize_bytes != 0) 1884229997Sken && (params->blocksize_bytes > be_lun->blocksize)) { 1885229997Sken uint32_t bs_multiple, tmp_blocksize; 1886229997Sken 1887229997Sken bs_multiple = params->blocksize_bytes / be_lun->blocksize; 1888229997Sken 1889229997Sken tmp_blocksize = bs_multiple * be_lun->blocksize; 1890229997Sken 1891229997Sken if (tmp_blocksize == params->blocksize_bytes) { 1892229997Sken be_lun->blocksize = params->blocksize_bytes; 1893229997Sken } else { 1894229997Sken snprintf(req->error_str, sizeof(req->error_str), 1895272911Smav "requested blocksize %u is not an even " 1896229997Sken "multiple of backing device blocksize %u", 1897272911Smav params->blocksize_bytes, 1898229997Sken be_lun->blocksize); 1899229997Sken return (EINVAL); 1900229997Sken 1901229997Sken } 1902229997Sken } else if ((params->blocksize_bytes != 0) 1903229997Sken && (params->blocksize_bytes != be_lun->blocksize)) { 1904229997Sken snprintf(req->error_str, sizeof(req->error_str), 1905272911Smav "requested blocksize %u < backing device " 1906272911Smav "blocksize %u", params->blocksize_bytes, 1907229997Sken be_lun->blocksize); 1908229997Sken return (EINVAL); 1909229997Sken } 1910229997Sken 1911229997Sken error = devsw->d_ioctl(dev, DIOCGMEDIASIZE, 1912229997Sken (caddr_t)&be_lun->size_bytes, FREAD, 1913229997Sken curthread); 1914229997Sken if (error) { 1915229997Sken snprintf(req->error_str, sizeof(req->error_str), 1916272911Smav "error %d returned for DIOCGMEDIASIZE " 1917272911Smav " ioctl on %s!", error, 1918232604Strasz be_lun->dev_path); 1919229997Sken return (error); 1920229997Sken } 1921229997Sken 1922232604Strasz if (params->lun_size_bytes != 0) { 1923232604Strasz if (params->lun_size_bytes > be_lun->size_bytes) { 1924232604Strasz snprintf(req->error_str, sizeof(req->error_str), 1925272911Smav "requested LUN size %ju > backing device " 1926272911Smav "size %ju", 1927232604Strasz (uintmax_t)params->lun_size_bytes, 1928232604Strasz (uintmax_t)be_lun->size_bytes); 1929232604Strasz return (EINVAL); 1930232604Strasz } 1931232604Strasz 1932232604Strasz be_lun->size_bytes = params->lun_size_bytes; 1933232604Strasz } 1934232604Strasz 1935264191Smav error = devsw->d_ioctl(dev, DIOCGSTRIPESIZE, 1936264191Smav (caddr_t)&ps, FREAD, curthread); 1937264191Smav if (error) 1938264191Smav ps = po = 0; 1939264191Smav else { 1940264191Smav error = devsw->d_ioctl(dev, DIOCGSTRIPEOFFSET, 1941264191Smav (caddr_t)&po, FREAD, curthread); 1942264191Smav if (error) 1943264191Smav po = 0; 1944264191Smav } 1945264191Smav pss = ps / be_lun->blocksize; 1946264191Smav pos = po / be_lun->blocksize; 1947264191Smav if ((pss > 0) && (pss * be_lun->blocksize == ps) && (pss >= pos) && 1948264191Smav ((pss & (pss - 1)) == 0) && (pos * be_lun->blocksize == po)) { 1949264191Smav be_lun->pblockexp = fls(pss) - 1; 1950264191Smav be_lun->pblockoff = (pss - pos) % pss; 1951264191Smav } 1952264191Smav 1953229997Sken return (0); 1954229997Sken} 1955229997Sken 1956229997Skenstatic int 1957229997Skenctl_be_block_close(struct ctl_be_block_lun *be_lun) 1958229997Sken{ 1959229997Sken DROP_GIANT(); 1960229997Sken if (be_lun->vn) { 1961229997Sken int flags = FREAD | FWRITE; 1962229997Sken 1963229997Sken switch (be_lun->dev_type) { 1964229997Sken case CTL_BE_BLOCK_DEV: 1965229997Sken if (be_lun->backend.dev.csw) { 1966229997Sken dev_relthread(be_lun->backend.dev.cdev, 1967229997Sken be_lun->backend.dev.dev_ref); 1968229997Sken be_lun->backend.dev.csw = NULL; 1969229997Sken be_lun->backend.dev.cdev = NULL; 1970229997Sken } 1971229997Sken break; 1972229997Sken case CTL_BE_BLOCK_FILE: 1973229997Sken break; 1974229997Sken case CTL_BE_BLOCK_NONE: 1975258871Strasz break; 1976229997Sken default: 1977229997Sken panic("Unexpected backend type."); 1978229997Sken break; 1979229997Sken } 1980229997Sken 1981229997Sken (void)vn_close(be_lun->vn, flags, NOCRED, curthread); 1982229997Sken be_lun->vn = NULL; 1983229997Sken 1984229997Sken switch (be_lun->dev_type) { 1985229997Sken case CTL_BE_BLOCK_DEV: 1986229997Sken break; 1987229997Sken case CTL_BE_BLOCK_FILE: 1988229997Sken if (be_lun->backend.file.cred != NULL) { 1989229997Sken crfree(be_lun->backend.file.cred); 1990229997Sken be_lun->backend.file.cred = NULL; 1991229997Sken } 1992229997Sken break; 1993229997Sken case CTL_BE_BLOCK_NONE: 1994258871Strasz break; 1995229997Sken default: 1996229997Sken panic("Unexpected backend type."); 1997229997Sken break; 1998229997Sken } 1999272911Smav be_lun->dev_type = CTL_BE_BLOCK_NONE; 2000229997Sken } 2001229997Sken PICKUP_GIANT(); 2002229997Sken 2003229997Sken return (0); 2004229997Sken} 2005229997Sken 2006229997Skenstatic int 2007229997Skenctl_be_block_open(struct ctl_be_block_softc *softc, 2008229997Sken struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req) 2009229997Sken{ 2010229997Sken struct nameidata nd; 2011229997Sken int flags; 2012229997Sken int error; 2013229997Sken 2014229997Sken /* 2015229997Sken * XXX KDM allow a read-only option? 2016229997Sken */ 2017229997Sken flags = FREAD | FWRITE; 2018229997Sken error = 0; 2019229997Sken 2020229997Sken if (rootvnode == NULL) { 2021229997Sken snprintf(req->error_str, sizeof(req->error_str), 2022272911Smav "Root filesystem is not mounted"); 2023229997Sken return (1); 2024229997Sken } 2025229997Sken 2026229997Sken if (!curthread->td_proc->p_fd->fd_cdir) { 2027229997Sken curthread->td_proc->p_fd->fd_cdir = rootvnode; 2028229997Sken VREF(rootvnode); 2029229997Sken } 2030229997Sken if (!curthread->td_proc->p_fd->fd_rdir) { 2031229997Sken curthread->td_proc->p_fd->fd_rdir = rootvnode; 2032229997Sken VREF(rootvnode); 2033229997Sken } 2034229997Sken if (!curthread->td_proc->p_fd->fd_jdir) { 2035229997Sken curthread->td_proc->p_fd->fd_jdir = rootvnode; 2036229997Sken VREF(rootvnode); 2037229997Sken } 2038229997Sken 2039229997Sken again: 2040229997Sken NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread); 2041229997Sken error = vn_open(&nd, &flags, 0, NULL); 2042229997Sken if (error) { 2043229997Sken /* 2044229997Sken * This is the only reasonable guess we can make as far as 2045229997Sken * path if the user doesn't give us a fully qualified path. 2046229997Sken * If they want to specify a file, they need to specify the 2047229997Sken * full path. 2048229997Sken */ 2049229997Sken if (be_lun->dev_path[0] != '/') { 2050229997Sken char *dev_path = "/dev/"; 2051229997Sken char *dev_name; 2052229997Sken 2053229997Sken /* Try adding device path at beginning of name */ 2054229997Sken dev_name = malloc(strlen(be_lun->dev_path) 2055229997Sken + strlen(dev_path) + 1, 2056229997Sken M_CTLBLK, M_WAITOK); 2057229997Sken if (dev_name) { 2058229997Sken sprintf(dev_name, "%s%s", dev_path, 2059229997Sken be_lun->dev_path); 2060229997Sken free(be_lun->dev_path, M_CTLBLK); 2061229997Sken be_lun->dev_path = dev_name; 2062229997Sken goto again; 2063229997Sken } 2064229997Sken } 2065229997Sken snprintf(req->error_str, sizeof(req->error_str), 2066272911Smav "error opening %s: %d", be_lun->dev_path, error); 2067229997Sken return (error); 2068229997Sken } 2069229997Sken 2070229997Sken NDFREE(&nd, NDF_ONLY_PNBUF); 2071229997Sken 2072229997Sken be_lun->vn = nd.ni_vp; 2073229997Sken 2074229997Sken /* We only support disks and files. */ 2075229997Sken if (vn_isdisk(be_lun->vn, &error)) { 2076229997Sken error = ctl_be_block_open_dev(be_lun, req); 2077229997Sken } else if (be_lun->vn->v_type == VREG) { 2078229997Sken error = ctl_be_block_open_file(be_lun, req); 2079229997Sken } else { 2080229997Sken error = EINVAL; 2081229997Sken snprintf(req->error_str, sizeof(req->error_str), 2082258871Strasz "%s is not a disk or plain file", be_lun->dev_path); 2083229997Sken } 2084229997Sken VOP_UNLOCK(be_lun->vn, 0); 2085229997Sken 2086229997Sken if (error != 0) { 2087229997Sken ctl_be_block_close(be_lun); 2088229997Sken return (error); 2089229997Sken } 2090229997Sken 2091229997Sken be_lun->blocksize_shift = fls(be_lun->blocksize) - 1; 2092229997Sken be_lun->size_blocks = be_lun->size_bytes >> be_lun->blocksize_shift; 2093229997Sken 2094229997Sken return (0); 2095229997Sken} 2096229997Sken 2097229997Skenstatic int 2098229997Skenctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2099229997Sken{ 2100229997Sken struct ctl_be_block_lun *be_lun; 2101229997Sken struct ctl_lun_create_params *params; 2102267481Smav char num_thread_str[16]; 2103229997Sken char tmpstr[32]; 2104267481Smav char *value; 2105264274Smav int retval, num_threads, unmap; 2106267481Smav int tmp_num_threads; 2107229997Sken 2108229997Sken params = &req->reqdata.create; 2109229997Sken retval = 0; 2110272911Smav req->status = CTL_LUN_OK; 2111229997Sken 2112229997Sken num_threads = cbb_num_threads; 2113229997Sken 2114229997Sken be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK); 2115229997Sken 2116272911Smav be_lun->params = req->reqdata.create; 2117229997Sken be_lun->softc = softc; 2118229997Sken STAILQ_INIT(&be_lun->input_queue); 2119275474Smav STAILQ_INIT(&be_lun->config_read_queue); 2120229997Sken STAILQ_INIT(&be_lun->config_write_queue); 2121229997Sken STAILQ_INIT(&be_lun->datamove_queue); 2122229997Sken sprintf(be_lun->lunname, "cblk%d", softc->num_luns); 2123267877Smav mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF); 2124267877Smav mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF); 2125268280Smav ctl_init_opts(&be_lun->ctl_be_lun.options, 2126268280Smav req->num_be_args, req->kern_be_args); 2127229997Sken 2128264886Smav be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG, 2129256995Smav NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0); 2130229997Sken 2131229997Sken if (be_lun->lun_zone == NULL) { 2132229997Sken snprintf(req->error_str, sizeof(req->error_str), 2133272911Smav "error allocating UMA zone"); 2134229997Sken goto bailout_error; 2135229997Sken } 2136229997Sken 2137229997Sken if (params->flags & CTL_LUN_FLAG_DEV_TYPE) 2138229997Sken be_lun->ctl_be_lun.lun_type = params->device_type; 2139229997Sken else 2140229997Sken be_lun->ctl_be_lun.lun_type = T_DIRECT; 2141229997Sken 2142229997Sken if (be_lun->ctl_be_lun.lun_type == T_DIRECT) { 2143268280Smav value = ctl_get_opt(&be_lun->ctl_be_lun.options, "file"); 2144267499Smav if (value == NULL) { 2145229997Sken snprintf(req->error_str, sizeof(req->error_str), 2146272911Smav "no file argument specified"); 2147229997Sken goto bailout_error; 2148229997Sken } 2149267499Smav be_lun->dev_path = strdup(value, M_CTLBLK); 2150272911Smav be_lun->blocksize = 512; 2151272911Smav be_lun->blocksize_shift = fls(be_lun->blocksize) - 1; 2152229997Sken 2153229997Sken retval = ctl_be_block_open(softc, be_lun, req); 2154229997Sken if (retval != 0) { 2155229997Sken retval = 0; 2156272911Smav req->status = CTL_LUN_WARNING; 2157229997Sken } 2158229997Sken } else { 2159229997Sken /* 2160229997Sken * For processor devices, we don't have any size. 2161229997Sken */ 2162229997Sken be_lun->blocksize = 0; 2163264191Smav be_lun->pblockexp = 0; 2164264191Smav be_lun->pblockoff = 0; 2165229997Sken be_lun->size_blocks = 0; 2166229997Sken be_lun->size_bytes = 0; 2167229997Sken be_lun->ctl_be_lun.maxlba = 0; 2168229997Sken 2169229997Sken /* 2170229997Sken * Default to just 1 thread for processor devices. 2171229997Sken */ 2172229997Sken num_threads = 1; 2173229997Sken } 2174229997Sken 2175229997Sken /* 2176229997Sken * XXX This searching loop might be refactored to be combined with 2177229997Sken * the loop above, 2178229997Sken */ 2179268280Smav value = ctl_get_opt(&be_lun->ctl_be_lun.options, "num_threads"); 2180267481Smav if (value != NULL) { 2181267481Smav tmp_num_threads = strtol(value, NULL, 0); 2182229997Sken 2183267481Smav /* 2184267481Smav * We don't let the user specify less than one 2185267481Smav * thread, but hope he's clueful enough not to 2186267481Smav * specify 1000 threads. 2187267481Smav */ 2188267481Smav if (tmp_num_threads < 1) { 2189267481Smav snprintf(req->error_str, sizeof(req->error_str), 2190272911Smav "invalid number of threads %s", 2191272911Smav num_thread_str); 2192267481Smav goto bailout_error; 2193229997Sken } 2194267481Smav num_threads = tmp_num_threads; 2195229997Sken } 2196274154Smav unmap = (be_lun->dispatch == ctl_be_block_dispatch_zvol); 2197268280Smav value = ctl_get_opt(&be_lun->ctl_be_lun.options, "unmap"); 2198274154Smav if (value != NULL) 2199274154Smav unmap = (strcmp(value, "on") == 0); 2200229997Sken 2201229997Sken be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED; 2202229997Sken be_lun->ctl_be_lun.flags = CTL_LUN_FLAG_PRIMARY; 2203272911Smav if (be_lun->vn == NULL) 2204272911Smav be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_OFFLINE; 2205264274Smav if (unmap) 2206264274Smav be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_UNMAP; 2207229997Sken be_lun->ctl_be_lun.be_lun = be_lun; 2208272911Smav be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ? 2209272911Smav 0 : (be_lun->size_blocks - 1); 2210229997Sken be_lun->ctl_be_lun.blocksize = be_lun->blocksize; 2211264191Smav be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp; 2212264191Smav be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff; 2213272911Smav if (be_lun->dispatch == ctl_be_block_dispatch_zvol && 2214272911Smav be_lun->blocksize != 0) 2215272911Smav be_lun->ctl_be_lun.atomicblock = CTLBLK_MAX_IO_SIZE / 2216272911Smav be_lun->blocksize; 2217229997Sken /* Tell the user the blocksize we ended up using */ 2218272911Smav params->lun_size_bytes = be_lun->size_bytes; 2219229997Sken params->blocksize_bytes = be_lun->blocksize; 2220229997Sken if (params->flags & CTL_LUN_FLAG_ID_REQ) { 2221229997Sken be_lun->ctl_be_lun.req_lun_id = params->req_lun_id; 2222229997Sken be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_ID_REQ; 2223229997Sken } else 2224229997Sken be_lun->ctl_be_lun.req_lun_id = 0; 2225229997Sken 2226229997Sken be_lun->ctl_be_lun.lun_shutdown = ctl_be_block_lun_shutdown; 2227229997Sken be_lun->ctl_be_lun.lun_config_status = 2228229997Sken ctl_be_block_lun_config_status; 2229229997Sken be_lun->ctl_be_lun.be = &ctl_be_block_driver; 2230229997Sken 2231229997Sken if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) { 2232229997Sken snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%4d", 2233229997Sken softc->num_luns); 2234229997Sken strncpy((char *)be_lun->ctl_be_lun.serial_num, tmpstr, 2235229997Sken ctl_min(sizeof(be_lun->ctl_be_lun.serial_num), 2236229997Sken sizeof(tmpstr))); 2237229997Sken 2238229997Sken /* Tell the user what we used for a serial number */ 2239229997Sken strncpy((char *)params->serial_num, tmpstr, 2240229997Sken ctl_min(sizeof(params->serial_num), sizeof(tmpstr))); 2241229997Sken } else { 2242229997Sken strncpy((char *)be_lun->ctl_be_lun.serial_num, 2243229997Sken params->serial_num, 2244229997Sken ctl_min(sizeof(be_lun->ctl_be_lun.serial_num), 2245229997Sken sizeof(params->serial_num))); 2246229997Sken } 2247229997Sken if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) { 2248229997Sken snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%4d", softc->num_luns); 2249229997Sken strncpy((char *)be_lun->ctl_be_lun.device_id, tmpstr, 2250229997Sken ctl_min(sizeof(be_lun->ctl_be_lun.device_id), 2251229997Sken sizeof(tmpstr))); 2252229997Sken 2253229997Sken /* Tell the user what we used for a device ID */ 2254229997Sken strncpy((char *)params->device_id, tmpstr, 2255229997Sken ctl_min(sizeof(params->device_id), sizeof(tmpstr))); 2256229997Sken } else { 2257229997Sken strncpy((char *)be_lun->ctl_be_lun.device_id, 2258229997Sken params->device_id, 2259229997Sken ctl_min(sizeof(be_lun->ctl_be_lun.device_id), 2260229997Sken sizeof(params->device_id))); 2261229997Sken } 2262229997Sken 2263229997Sken TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun); 2264229997Sken 2265229997Sken be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK, 2266229997Sken taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue); 2267229997Sken 2268229997Sken if (be_lun->io_taskqueue == NULL) { 2269229997Sken snprintf(req->error_str, sizeof(req->error_str), 2270272911Smav "unable to create taskqueue"); 2271229997Sken goto bailout_error; 2272229997Sken } 2273229997Sken 2274229997Sken /* 2275229997Sken * Note that we start the same number of threads by default for 2276229997Sken * both the file case and the block device case. For the file 2277229997Sken * case, we need multiple threads to allow concurrency, because the 2278229997Sken * vnode interface is designed to be a blocking interface. For the 2279229997Sken * block device case, ZFS zvols at least will block the caller's 2280229997Sken * context in many instances, and so we need multiple threads to 2281229997Sken * overcome that problem. Other block devices don't need as many 2282229997Sken * threads, but they shouldn't cause too many problems. 2283229997Sken * 2284229997Sken * If the user wants to just have a single thread for a block 2285229997Sken * device, he can specify that when the LUN is created, or change 2286229997Sken * the tunable/sysctl to alter the default number of threads. 2287229997Sken */ 2288229997Sken retval = taskqueue_start_threads(&be_lun->io_taskqueue, 2289229997Sken /*num threads*/num_threads, 2290229997Sken /*priority*/PWAIT, 2291229997Sken /*thread name*/ 2292229997Sken "%s taskq", be_lun->lunname); 2293229997Sken 2294229997Sken if (retval != 0) 2295229997Sken goto bailout_error; 2296229997Sken 2297229997Sken be_lun->num_threads = num_threads; 2298229997Sken 2299229997Sken mtx_lock(&softc->lock); 2300229997Sken softc->num_luns++; 2301229997Sken STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links); 2302229997Sken 2303229997Sken mtx_unlock(&softc->lock); 2304229997Sken 2305229997Sken retval = ctl_add_lun(&be_lun->ctl_be_lun); 2306229997Sken if (retval != 0) { 2307229997Sken mtx_lock(&softc->lock); 2308229997Sken STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2309229997Sken links); 2310229997Sken softc->num_luns--; 2311229997Sken mtx_unlock(&softc->lock); 2312229997Sken snprintf(req->error_str, sizeof(req->error_str), 2313272911Smav "ctl_add_lun() returned error %d, see dmesg for " 2314272911Smav "details", retval); 2315229997Sken retval = 0; 2316229997Sken goto bailout_error; 2317229997Sken } 2318229997Sken 2319229997Sken mtx_lock(&softc->lock); 2320229997Sken 2321229997Sken /* 2322229997Sken * Tell the config_status routine that we're waiting so it won't 2323229997Sken * clean up the LUN in the event of an error. 2324229997Sken */ 2325229997Sken be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2326229997Sken 2327229997Sken while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) { 2328229997Sken retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2329229997Sken if (retval == EINTR) 2330229997Sken break; 2331229997Sken } 2332229997Sken be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2333229997Sken 2334229997Sken if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) { 2335229997Sken snprintf(req->error_str, sizeof(req->error_str), 2336272911Smav "LUN configuration error, see dmesg for details"); 2337229997Sken STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, 2338229997Sken links); 2339229997Sken softc->num_luns--; 2340229997Sken mtx_unlock(&softc->lock); 2341229997Sken goto bailout_error; 2342229997Sken } else { 2343229997Sken params->req_lun_id = be_lun->ctl_be_lun.lun_id; 2344229997Sken } 2345229997Sken 2346229997Sken mtx_unlock(&softc->lock); 2347229997Sken 2348229997Sken be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id, 2349229997Sken be_lun->blocksize, 2350229997Sken DEVSTAT_ALL_SUPPORTED, 2351229997Sken be_lun->ctl_be_lun.lun_type 2352229997Sken | DEVSTAT_TYPE_IF_OTHER, 2353229997Sken DEVSTAT_PRIORITY_OTHER); 2354229997Sken 2355229997Sken return (retval); 2356229997Sken 2357229997Skenbailout_error: 2358229997Sken req->status = CTL_LUN_ERROR; 2359229997Sken 2360267429Smav if (be_lun->io_taskqueue != NULL) 2361267429Smav taskqueue_free(be_lun->io_taskqueue); 2362229997Sken ctl_be_block_close(be_lun); 2363267429Smav if (be_lun->dev_path != NULL) 2364267429Smav free(be_lun->dev_path, M_CTLBLK); 2365267429Smav if (be_lun->lun_zone != NULL) 2366267429Smav uma_zdestroy(be_lun->lun_zone); 2367268280Smav ctl_free_opts(&be_lun->ctl_be_lun.options); 2368267877Smav mtx_destroy(&be_lun->queue_lock); 2369267877Smav mtx_destroy(&be_lun->io_lock); 2370229997Sken free(be_lun, M_CTLBLK); 2371229997Sken 2372229997Sken return (retval); 2373229997Sken} 2374229997Sken 2375229997Skenstatic int 2376229997Skenctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2377229997Sken{ 2378229997Sken struct ctl_lun_rm_params *params; 2379229997Sken struct ctl_be_block_lun *be_lun; 2380229997Sken int retval; 2381229997Sken 2382229997Sken params = &req->reqdata.rm; 2383229997Sken 2384229997Sken mtx_lock(&softc->lock); 2385229997Sken 2386229997Sken be_lun = NULL; 2387229997Sken 2388229997Sken STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 2389229997Sken if (be_lun->ctl_be_lun.lun_id == params->lun_id) 2390229997Sken break; 2391229997Sken } 2392229997Sken mtx_unlock(&softc->lock); 2393229997Sken 2394229997Sken if (be_lun == NULL) { 2395229997Sken snprintf(req->error_str, sizeof(req->error_str), 2396272911Smav "LUN %u is not managed by the block backend", 2397272911Smav params->lun_id); 2398229997Sken goto bailout_error; 2399229997Sken } 2400229997Sken 2401229997Sken retval = ctl_disable_lun(&be_lun->ctl_be_lun); 2402229997Sken 2403229997Sken if (retval != 0) { 2404229997Sken snprintf(req->error_str, sizeof(req->error_str), 2405272911Smav "error %d returned from ctl_disable_lun() for " 2406272911Smav "LUN %d", retval, params->lun_id); 2407229997Sken goto bailout_error; 2408229997Sken 2409229997Sken } 2410229997Sken 2411229997Sken retval = ctl_invalidate_lun(&be_lun->ctl_be_lun); 2412229997Sken if (retval != 0) { 2413229997Sken snprintf(req->error_str, sizeof(req->error_str), 2414272911Smav "error %d returned from ctl_invalidate_lun() for " 2415272911Smav "LUN %d", retval, params->lun_id); 2416229997Sken goto bailout_error; 2417229997Sken } 2418229997Sken 2419229997Sken mtx_lock(&softc->lock); 2420229997Sken 2421229997Sken be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; 2422229997Sken 2423229997Sken while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2424229997Sken retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); 2425229997Sken if (retval == EINTR) 2426229997Sken break; 2427229997Sken } 2428229997Sken 2429229997Sken be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; 2430229997Sken 2431229997Sken if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { 2432229997Sken snprintf(req->error_str, sizeof(req->error_str), 2433272911Smav "interrupted waiting for LUN to be freed"); 2434229997Sken mtx_unlock(&softc->lock); 2435229997Sken goto bailout_error; 2436229997Sken } 2437229997Sken 2438229997Sken STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links); 2439229997Sken 2440229997Sken softc->num_luns--; 2441229997Sken mtx_unlock(&softc->lock); 2442229997Sken 2443229997Sken taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task); 2444229997Sken 2445229997Sken taskqueue_free(be_lun->io_taskqueue); 2446229997Sken 2447229997Sken ctl_be_block_close(be_lun); 2448229997Sken 2449229997Sken if (be_lun->disk_stats != NULL) 2450229997Sken devstat_remove_entry(be_lun->disk_stats); 2451229997Sken 2452229997Sken uma_zdestroy(be_lun->lun_zone); 2453229997Sken 2454268280Smav ctl_free_opts(&be_lun->ctl_be_lun.options); 2455229997Sken free(be_lun->dev_path, M_CTLBLK); 2456267877Smav mtx_destroy(&be_lun->queue_lock); 2457267877Smav mtx_destroy(&be_lun->io_lock); 2458229997Sken free(be_lun, M_CTLBLK); 2459229997Sken 2460229997Sken req->status = CTL_LUN_OK; 2461229997Sken 2462229997Sken return (0); 2463229997Sken 2464229997Skenbailout_error: 2465229997Sken 2466229997Sken req->status = CTL_LUN_ERROR; 2467229997Sken 2468229997Sken return (0); 2469229997Sken} 2470229997Sken 2471232604Straszstatic int 2472232604Straszctl_be_block_modify_file(struct ctl_be_block_lun *be_lun, 2473232604Strasz struct ctl_lun_req *req) 2474232604Strasz{ 2475232604Strasz struct vattr vattr; 2476232604Strasz int error; 2477272911Smav struct ctl_lun_create_params *params = &be_lun->params; 2478232604Strasz 2479232604Strasz if (params->lun_size_bytes != 0) { 2480232604Strasz be_lun->size_bytes = params->lun_size_bytes; 2481232604Strasz } else { 2482271794Smav vn_lock(be_lun->vn, LK_SHARED | LK_RETRY); 2483232604Strasz error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred); 2484271794Smav VOP_UNLOCK(be_lun->vn, 0); 2485232604Strasz if (error != 0) { 2486232604Strasz snprintf(req->error_str, sizeof(req->error_str), 2487232604Strasz "error calling VOP_GETATTR() for file %s", 2488232604Strasz be_lun->dev_path); 2489232604Strasz return (error); 2490232604Strasz } 2491232604Strasz 2492232604Strasz be_lun->size_bytes = vattr.va_size; 2493232604Strasz } 2494232604Strasz 2495232604Strasz return (0); 2496232604Strasz} 2497232604Strasz 2498232604Straszstatic int 2499232604Straszctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun, 2500232604Strasz struct ctl_lun_req *req) 2501232604Strasz{ 2502271794Smav struct ctl_be_block_devdata *dev_data; 2503232604Strasz int error; 2504272911Smav struct ctl_lun_create_params *params = &be_lun->params; 2505232604Strasz uint64_t size_bytes; 2506232604Strasz 2507271794Smav dev_data = &be_lun->backend.dev; 2508271794Smav if (!dev_data->csw->d_ioctl) { 2509232604Strasz snprintf(req->error_str, sizeof(req->error_str), 2510272911Smav "no d_ioctl for device %s!", be_lun->dev_path); 2511232604Strasz return (ENODEV); 2512232604Strasz } 2513232604Strasz 2514271794Smav error = dev_data->csw->d_ioctl(dev_data->cdev, DIOCGMEDIASIZE, 2515232604Strasz (caddr_t)&size_bytes, FREAD, 2516232604Strasz curthread); 2517232604Strasz if (error) { 2518232604Strasz snprintf(req->error_str, sizeof(req->error_str), 2519272911Smav "error %d returned for DIOCGMEDIASIZE ioctl " 2520272911Smav "on %s!", error, be_lun->dev_path); 2521232604Strasz return (error); 2522232604Strasz } 2523232604Strasz 2524232604Strasz if (params->lun_size_bytes != 0) { 2525232604Strasz if (params->lun_size_bytes > size_bytes) { 2526232604Strasz snprintf(req->error_str, sizeof(req->error_str), 2527272911Smav "requested LUN size %ju > backing device " 2528272911Smav "size %ju", 2529232604Strasz (uintmax_t)params->lun_size_bytes, 2530232604Strasz (uintmax_t)size_bytes); 2531232604Strasz return (EINVAL); 2532232604Strasz } 2533232604Strasz 2534232604Strasz be_lun->size_bytes = params->lun_size_bytes; 2535232604Strasz } else { 2536232604Strasz be_lun->size_bytes = size_bytes; 2537232604Strasz } 2538232604Strasz 2539232604Strasz return (0); 2540232604Strasz} 2541232604Strasz 2542232604Straszstatic int 2543232604Straszctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req) 2544232604Strasz{ 2545232604Strasz struct ctl_lun_modify_params *params; 2546232604Strasz struct ctl_be_block_lun *be_lun; 2547271794Smav uint64_t oldsize; 2548241896Skib int error; 2549232604Strasz 2550232604Strasz params = &req->reqdata.modify; 2551232604Strasz 2552232604Strasz mtx_lock(&softc->lock); 2553232604Strasz be_lun = NULL; 2554232604Strasz STAILQ_FOREACH(be_lun, &softc->lun_list, links) { 2555232604Strasz if (be_lun->ctl_be_lun.lun_id == params->lun_id) 2556232604Strasz break; 2557232604Strasz } 2558232604Strasz mtx_unlock(&softc->lock); 2559232604Strasz 2560232604Strasz if (be_lun == NULL) { 2561232604Strasz snprintf(req->error_str, sizeof(req->error_str), 2562272911Smav "LUN %u is not managed by the block backend", 2563272911Smav params->lun_id); 2564232604Strasz goto bailout_error; 2565232604Strasz } 2566232604Strasz 2567272911Smav be_lun->params.lun_size_bytes = params->lun_size_bytes; 2568232604Strasz 2569274253Smav oldsize = be_lun->size_bytes; 2570272911Smav if (be_lun->vn == NULL) 2571272911Smav error = ctl_be_block_open(softc, be_lun, req); 2572272911Smav else if (be_lun->vn->v_type == VREG) 2573232604Strasz error = ctl_be_block_modify_file(be_lun, req); 2574232604Strasz else 2575232604Strasz error = ctl_be_block_modify_dev(be_lun, req); 2576232604Strasz 2577274253Smav if (error == 0 && be_lun->size_bytes != oldsize) { 2578271794Smav be_lun->size_blocks = be_lun->size_bytes >> 2579271794Smav be_lun->blocksize_shift; 2580232604Strasz 2581271794Smav /* 2582271794Smav * The maximum LBA is the size - 1. 2583271794Smav * 2584271794Smav * XXX: Note that this field is being updated without locking, 2585271794Smav * which might cause problems on 32-bit architectures. 2586271794Smav */ 2587272911Smav be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ? 2588272911Smav 0 : (be_lun->size_blocks - 1); 2589272911Smav be_lun->ctl_be_lun.blocksize = be_lun->blocksize; 2590272911Smav be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp; 2591272911Smav be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff; 2592272911Smav if (be_lun->dispatch == ctl_be_block_dispatch_zvol && 2593272911Smav be_lun->blocksize != 0) 2594272911Smav be_lun->ctl_be_lun.atomicblock = CTLBLK_MAX_IO_SIZE / 2595272911Smav be_lun->blocksize; 2596271794Smav ctl_lun_capacity_changed(&be_lun->ctl_be_lun); 2597272911Smav if (oldsize == 0 && be_lun->size_blocks != 0) 2598272911Smav ctl_lun_online(&be_lun->ctl_be_lun); 2599271794Smav } 2600232604Strasz 2601232604Strasz /* Tell the user the exact size we ended up using */ 2602232604Strasz params->lun_size_bytes = be_lun->size_bytes; 2603232604Strasz 2604272911Smav req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK; 2605232604Strasz 2606232604Strasz return (0); 2607232604Strasz 2608232604Straszbailout_error: 2609232604Strasz req->status = CTL_LUN_ERROR; 2610232604Strasz 2611232604Strasz return (0); 2612232604Strasz} 2613232604Strasz 2614229997Skenstatic void 2615229997Skenctl_be_block_lun_shutdown(void *be_lun) 2616229997Sken{ 2617229997Sken struct ctl_be_block_lun *lun; 2618229997Sken struct ctl_be_block_softc *softc; 2619229997Sken 2620229997Sken lun = (struct ctl_be_block_lun *)be_lun; 2621229997Sken 2622229997Sken softc = lun->softc; 2623229997Sken 2624229997Sken mtx_lock(&softc->lock); 2625229997Sken lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED; 2626229997Sken if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2627229997Sken wakeup(lun); 2628229997Sken mtx_unlock(&softc->lock); 2629229997Sken 2630229997Sken} 2631229997Sken 2632229997Skenstatic void 2633229997Skenctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status) 2634229997Sken{ 2635229997Sken struct ctl_be_block_lun *lun; 2636229997Sken struct ctl_be_block_softc *softc; 2637229997Sken 2638229997Sken lun = (struct ctl_be_block_lun *)be_lun; 2639229997Sken softc = lun->softc; 2640229997Sken 2641229997Sken if (status == CTL_LUN_CONFIG_OK) { 2642229997Sken mtx_lock(&softc->lock); 2643229997Sken lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2644229997Sken if (lun->flags & CTL_BE_BLOCK_LUN_WAITING) 2645229997Sken wakeup(lun); 2646229997Sken mtx_unlock(&softc->lock); 2647229997Sken 2648229997Sken /* 2649229997Sken * We successfully added the LUN, attempt to enable it. 2650229997Sken */ 2651229997Sken if (ctl_enable_lun(&lun->ctl_be_lun) != 0) { 2652229997Sken printf("%s: ctl_enable_lun() failed!\n", __func__); 2653229997Sken if (ctl_invalidate_lun(&lun->ctl_be_lun) != 0) { 2654229997Sken printf("%s: ctl_invalidate_lun() failed!\n", 2655229997Sken __func__); 2656229997Sken } 2657229997Sken } 2658229997Sken 2659229997Sken return; 2660229997Sken } 2661229997Sken 2662229997Sken 2663229997Sken mtx_lock(&softc->lock); 2664229997Sken lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED; 2665229997Sken lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR; 2666229997Sken wakeup(lun); 2667229997Sken mtx_unlock(&softc->lock); 2668229997Sken} 2669229997Sken 2670229997Sken 2671229997Skenstatic int 2672229997Skenctl_be_block_config_write(union ctl_io *io) 2673229997Sken{ 2674229997Sken struct ctl_be_block_lun *be_lun; 2675229997Sken struct ctl_be_lun *ctl_be_lun; 2676229997Sken int retval; 2677229997Sken 2678229997Sken retval = 0; 2679229997Sken 2680229997Sken DPRINTF("entered\n"); 2681229997Sken 2682229997Sken ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 2683229997Sken CTL_PRIV_BACKEND_LUN].ptr; 2684229997Sken be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 2685229997Sken 2686229997Sken switch (io->scsiio.cdb[0]) { 2687229997Sken case SYNCHRONIZE_CACHE: 2688229997Sken case SYNCHRONIZE_CACHE_16: 2689264274Smav case WRITE_SAME_10: 2690264274Smav case WRITE_SAME_16: 2691264274Smav case UNMAP: 2692229997Sken /* 2693229997Sken * The upper level CTL code will filter out any CDBs with 2694229997Sken * the immediate bit set and return the proper error. 2695229997Sken * 2696229997Sken * We don't really need to worry about what LBA range the 2697229997Sken * user asked to be synced out. When they issue a sync 2698229997Sken * cache command, we'll sync out the whole thing. 2699229997Sken */ 2700267877Smav mtx_lock(&be_lun->queue_lock); 2701229997Sken STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr, 2702229997Sken links); 2703267877Smav mtx_unlock(&be_lun->queue_lock); 2704229997Sken taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); 2705229997Sken break; 2706229997Sken case START_STOP_UNIT: { 2707229997Sken struct scsi_start_stop_unit *cdb; 2708229997Sken 2709229997Sken cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb; 2710229997Sken 2711229997Sken if (cdb->how & SSS_START) 2712229997Sken retval = ctl_start_lun(ctl_be_lun); 2713229997Sken else { 2714229997Sken retval = ctl_stop_lun(ctl_be_lun); 2715229997Sken /* 2716229997Sken * XXX KDM Copan-specific offline behavior. 2717229997Sken * Figure out a reasonable way to port this? 2718229997Sken */ 2719229997Sken#ifdef NEEDTOPORT 2720229997Sken if ((retval == 0) 2721229997Sken && (cdb->byte2 & SSS_ONOFFLINE)) 2722229997Sken retval = ctl_lun_offline(ctl_be_lun); 2723229997Sken#endif 2724229997Sken } 2725229997Sken 2726229997Sken /* 2727229997Sken * In general, the above routines should not fail. They 2728229997Sken * just set state for the LUN. So we've got something 2729229997Sken * pretty wrong here if we can't start or stop the LUN. 2730229997Sken */ 2731229997Sken if (retval != 0) { 2732229997Sken ctl_set_internal_failure(&io->scsiio, 2733229997Sken /*sks_valid*/ 1, 2734229997Sken /*retry_count*/ 0xf051); 2735229997Sken retval = CTL_RETVAL_COMPLETE; 2736229997Sken } else { 2737229997Sken ctl_set_success(&io->scsiio); 2738229997Sken } 2739229997Sken ctl_config_write_done(io); 2740229997Sken break; 2741229997Sken } 2742229997Sken default: 2743229997Sken ctl_set_invalid_opcode(&io->scsiio); 2744229997Sken ctl_config_write_done(io); 2745229997Sken retval = CTL_RETVAL_COMPLETE; 2746229997Sken break; 2747229997Sken } 2748229997Sken 2749229997Sken return (retval); 2750229997Sken} 2751229997Sken 2752229997Skenstatic int 2753229997Skenctl_be_block_config_read(union ctl_io *io) 2754229997Sken{ 2755275474Smav struct ctl_be_block_lun *be_lun; 2756275474Smav struct ctl_be_lun *ctl_be_lun; 2757275474Smav int retval = 0; 2758275474Smav 2759275474Smav DPRINTF("entered\n"); 2760275474Smav 2761275474Smav ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[ 2762275474Smav CTL_PRIV_BACKEND_LUN].ptr; 2763275474Smav be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun; 2764275474Smav 2765275474Smav switch (io->scsiio.cdb[0]) { 2766275474Smav case SERVICE_ACTION_IN: 2767275474Smav if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) { 2768275474Smav mtx_lock(&be_lun->queue_lock); 2769275474Smav STAILQ_INSERT_TAIL(&be_lun->config_read_queue, 2770275474Smav &io->io_hdr, links); 2771275474Smav mtx_unlock(&be_lun->queue_lock); 2772275474Smav taskqueue_enqueue(be_lun->io_taskqueue, 2773275474Smav &be_lun->io_task); 2774275474Smav retval = CTL_RETVAL_QUEUED; 2775275474Smav break; 2776275474Smav } 2777275474Smav ctl_set_invalid_field(&io->scsiio, 2778275474Smav /*sks_valid*/ 1, 2779275474Smav /*command*/ 1, 2780275474Smav /*field*/ 1, 2781275474Smav /*bit_valid*/ 1, 2782275474Smav /*bit*/ 4); 2783275474Smav ctl_config_read_done(io); 2784275474Smav retval = CTL_RETVAL_COMPLETE; 2785275474Smav break; 2786275474Smav default: 2787275474Smav ctl_set_invalid_opcode(&io->scsiio); 2788275474Smav ctl_config_read_done(io); 2789275474Smav retval = CTL_RETVAL_COMPLETE; 2790275474Smav break; 2791275474Smav } 2792275474Smav 2793275474Smav return (retval); 2794229997Sken} 2795229997Sken 2796229997Skenstatic int 2797229997Skenctl_be_block_lun_info(void *be_lun, struct sbuf *sb) 2798229997Sken{ 2799229997Sken struct ctl_be_block_lun *lun; 2800229997Sken int retval; 2801229997Sken 2802229997Sken lun = (struct ctl_be_block_lun *)be_lun; 2803229997Sken retval = 0; 2804229997Sken 2805268283Smav retval = sbuf_printf(sb, "\t<num_threads>"); 2806229997Sken 2807229997Sken if (retval != 0) 2808229997Sken goto bailout; 2809229997Sken 2810229997Sken retval = sbuf_printf(sb, "%d", lun->num_threads); 2811229997Sken 2812229997Sken if (retval != 0) 2813229997Sken goto bailout; 2814229997Sken 2815268283Smav retval = sbuf_printf(sb, "</num_threads>\n"); 2816229997Sken 2817229997Skenbailout: 2818229997Sken 2819229997Sken return (retval); 2820229997Sken} 2821229997Sken 2822274154Smavstatic uint64_t 2823274154Smavctl_be_block_lun_attr(void *be_lun, const char *attrname) 2824274154Smav{ 2825274154Smav struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun; 2826274154Smav 2827274154Smav if (lun->getattr == NULL) 2828274154Smav return (UINT64_MAX); 2829274154Smav return (lun->getattr(lun, attrname)); 2830274154Smav} 2831274154Smav 2832229997Skenint 2833229997Skenctl_be_block_init(void) 2834229997Sken{ 2835229997Sken struct ctl_be_block_softc *softc; 2836229997Sken int retval; 2837229997Sken 2838229997Sken softc = &backend_block_softc; 2839229997Sken retval = 0; 2840229997Sken 2841267877Smav mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF); 2842264020Strasz beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io), 2843264020Strasz NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 2844229997Sken STAILQ_INIT(&softc->disk_list); 2845229997Sken STAILQ_INIT(&softc->lun_list); 2846229997Sken 2847229997Sken return (retval); 2848229997Sken} 2849