camdd.c revision 334603
1/*- 2 * Copyright (c) 1997-2007 Kenneth D. Merry 3 * Copyright (c) 2013, 2014, 2015 Spectra Logic Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions, and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * substantially similar to the "NO WARRANTY" disclaimer below 14 * ("Disclaimer") and any redistribution must be conditioned upon 15 * including a substantially similar Disclaimer requirement for further 16 * binary redistribution. 17 * 18 * NO WARRANTY 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 28 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGES. 30 * 31 * Authors: Ken Merry (Spectra Logic Corporation) 32 */ 33 34/* 35 * This is eventually intended to be: 36 * - A basic data transfer/copy utility 37 * - A simple benchmark utility 38 * - An example of how to use the asynchronous pass(4) driver interface. 39 */ 40#include <sys/cdefs.h> 41__FBSDID("$FreeBSD: stable/11/usr.sbin/camdd/camdd.c 334603 2018-06-04 05:23:06Z delphij $"); 42 43#include <sys/ioctl.h> 44#include <sys/stdint.h> 45#include <sys/types.h> 46#include <sys/endian.h> 47#include <sys/param.h> 48#include <sys/sbuf.h> 49#include <sys/stat.h> 50#include <sys/event.h> 51#include <sys/time.h> 52#include <sys/uio.h> 53#include <vm/vm.h> 54#include <machine/bus.h> 55#include <sys/bus.h> 56#include <sys/bus_dma.h> 57#include <sys/mtio.h> 58#include <sys/conf.h> 59#include <sys/disk.h> 60 61#include <stdio.h> 62#include <stdlib.h> 63#include <semaphore.h> 64#include <string.h> 65#include <unistd.h> 66#include <inttypes.h> 67#include <limits.h> 68#include <fcntl.h> 69#include <ctype.h> 70#include <err.h> 71#include <libutil.h> 72#include <pthread.h> 73#include <assert.h> 74#include <bsdxml.h> 75 76#include <cam/cam.h> 77#include <cam/cam_debug.h> 78#include <cam/cam_ccb.h> 79#include <cam/scsi/scsi_all.h> 80#include <cam/scsi/scsi_da.h> 81#include <cam/scsi/scsi_pass.h> 82#include <cam/scsi/scsi_message.h> 83#include <cam/scsi/smp_all.h> 84#include <camlib.h> 85#include <mtlib.h> 86#include <zlib.h> 87 88typedef enum { 89 CAMDD_CMD_NONE = 0x00000000, 90 CAMDD_CMD_HELP = 0x00000001, 91 CAMDD_CMD_WRITE = 0x00000002, 92 CAMDD_CMD_READ = 0x00000003 93} camdd_cmdmask; 94 95typedef enum { 96 CAMDD_ARG_NONE = 0x00000000, 97 CAMDD_ARG_VERBOSE = 0x00000001, 98 CAMDD_ARG_DEVICE = 0x00000002, 99 CAMDD_ARG_BUS = 0x00000004, 100 CAMDD_ARG_TARGET = 0x00000008, 101 CAMDD_ARG_LUN = 0x00000010, 102 CAMDD_ARG_UNIT = 0x00000020, 103 CAMDD_ARG_TIMEOUT = 0x00000040, 104 CAMDD_ARG_ERR_RECOVER = 0x00000080, 105 CAMDD_ARG_RETRIES = 0x00000100 106} camdd_argmask; 107 108typedef enum { 109 CAMDD_DEV_NONE = 0x00, 110 CAMDD_DEV_PASS = 0x01, 111 CAMDD_DEV_FILE = 0x02 112} camdd_dev_type; 113 114struct camdd_io_opts { 115 camdd_dev_type dev_type; 116 char *dev_name; 117 uint64_t blocksize; 118 uint64_t queue_depth; 119 uint64_t offset; 120 int min_cmd_size; 121 int write_dev; 122 uint64_t debug; 123}; 124 125typedef enum { 126 CAMDD_BUF_NONE, 127 CAMDD_BUF_DATA, 128 CAMDD_BUF_INDIRECT 129} camdd_buf_type; 130 131struct camdd_buf_indirect { 132 /* 133 * Pointer to the source buffer. 134 */ 135 struct camdd_buf *src_buf; 136 137 /* 138 * Offset into the source buffer, in bytes. 139 */ 140 uint64_t offset; 141 /* 142 * Pointer to the starting point in the source buffer. 143 */ 144 uint8_t *start_ptr; 145 146 /* 147 * Length of this chunk in bytes. 148 */ 149 size_t len; 150}; 151 152struct camdd_buf_data { 153 /* 154 * Buffer allocated when we allocate this camdd_buf. This should 155 * be the size of the blocksize for this device. 156 */ 157 uint8_t *buf; 158 159 /* 160 * The amount of backing store allocated in buf. Generally this 161 * will be the blocksize of the device. 162 */ 163 uint32_t alloc_len; 164 165 /* 166 * The amount of data that was put into the buffer (on reads) or 167 * the amount of data we have put onto the src_list so far (on 168 * writes). 169 */ 170 uint32_t fill_len; 171 172 /* 173 * The amount of data that was not transferred. 174 */ 175 uint32_t resid; 176 177 /* 178 * Starting byte offset on the reader. 179 */ 180 uint64_t src_start_offset; 181 182 /* 183 * CCB used for pass(4) device targets. 184 */ 185 union ccb ccb; 186 187 /* 188 * Number of scatter/gather segments. 189 */ 190 int sg_count; 191 192 /* 193 * Set if we had to tack on an extra buffer to round the transfer 194 * up to a sector size. 195 */ 196 int extra_buf; 197 198 /* 199 * Scatter/gather list used generally when we're the writer for a 200 * pass(4) device. 201 */ 202 bus_dma_segment_t *segs; 203 204 /* 205 * Scatter/gather list used generally when we're the writer for a 206 * file or block device; 207 */ 208 struct iovec *iovec; 209}; 210 211union camdd_buf_types { 212 struct camdd_buf_indirect indirect; 213 struct camdd_buf_data data; 214}; 215 216typedef enum { 217 CAMDD_STATUS_NONE, 218 CAMDD_STATUS_OK, 219 CAMDD_STATUS_SHORT_IO, 220 CAMDD_STATUS_EOF, 221 CAMDD_STATUS_ERROR 222} camdd_buf_status; 223 224struct camdd_buf { 225 camdd_buf_type buf_type; 226 union camdd_buf_types buf_type_spec; 227 228 camdd_buf_status status; 229 230 uint64_t lba; 231 size_t len; 232 233 /* 234 * A reference count of how many indirect buffers point to this 235 * buffer. 236 */ 237 int refcount; 238 239 /* 240 * A link back to our parent device. 241 */ 242 struct camdd_dev *dev; 243 STAILQ_ENTRY(camdd_buf) links; 244 STAILQ_ENTRY(camdd_buf) work_links; 245 246 /* 247 * A count of the buffers on the src_list. 248 */ 249 int src_count; 250 251 /* 252 * List of buffers from our partner thread that are the components 253 * of this buffer for the I/O. Uses src_links. 254 */ 255 STAILQ_HEAD(,camdd_buf) src_list; 256 STAILQ_ENTRY(camdd_buf) src_links; 257}; 258 259#define NUM_DEV_TYPES 2 260 261struct camdd_dev_pass { 262 int scsi_dev_type; 263 struct cam_device *dev; 264 uint64_t max_sector; 265 uint32_t block_len; 266 uint32_t cpi_maxio; 267}; 268 269typedef enum { 270 CAMDD_FILE_NONE, 271 CAMDD_FILE_REG, 272 CAMDD_FILE_STD, 273 CAMDD_FILE_PIPE, 274 CAMDD_FILE_DISK, 275 CAMDD_FILE_TAPE, 276 CAMDD_FILE_TTY, 277 CAMDD_FILE_MEM 278} camdd_file_type; 279 280typedef enum { 281 CAMDD_FF_NONE = 0x00, 282 CAMDD_FF_CAN_SEEK = 0x01 283} camdd_file_flags; 284 285struct camdd_dev_file { 286 int fd; 287 struct stat sb; 288 char filename[MAXPATHLEN + 1]; 289 camdd_file_type file_type; 290 camdd_file_flags file_flags; 291 uint8_t *tmp_buf; 292}; 293 294struct camdd_dev_block { 295 int fd; 296 uint64_t size_bytes; 297 uint32_t block_len; 298}; 299 300union camdd_dev_spec { 301 struct camdd_dev_pass pass; 302 struct camdd_dev_file file; 303 struct camdd_dev_block block; 304}; 305 306typedef enum { 307 CAMDD_DEV_FLAG_NONE = 0x00, 308 CAMDD_DEV_FLAG_EOF = 0x01, 309 CAMDD_DEV_FLAG_PEER_EOF = 0x02, 310 CAMDD_DEV_FLAG_ACTIVE = 0x04, 311 CAMDD_DEV_FLAG_EOF_SENT = 0x08, 312 CAMDD_DEV_FLAG_EOF_QUEUED = 0x10 313} camdd_dev_flags; 314 315struct camdd_dev { 316 camdd_dev_type dev_type; 317 union camdd_dev_spec dev_spec; 318 camdd_dev_flags flags; 319 char device_name[MAXPATHLEN+1]; 320 uint32_t blocksize; 321 uint32_t sector_size; 322 uint64_t max_sector; 323 uint64_t sector_io_limit; 324 int min_cmd_size; 325 int write_dev; 326 int retry_count; 327 int io_timeout; 328 int debug; 329 uint64_t start_offset_bytes; 330 uint64_t next_io_pos_bytes; 331 uint64_t next_peer_pos_bytes; 332 uint64_t next_completion_pos_bytes; 333 uint64_t peer_bytes_queued; 334 uint64_t bytes_transferred; 335 uint32_t target_queue_depth; 336 uint32_t cur_active_io; 337 uint8_t *extra_buf; 338 uint32_t extra_buf_len; 339 struct camdd_dev *peer_dev; 340 pthread_mutex_t mutex; 341 pthread_cond_t cond; 342 int kq; 343 344 int (*run)(struct camdd_dev *dev); 345 int (*fetch)(struct camdd_dev *dev); 346 347 /* 348 * Buffers that are available for I/O. Uses links. 349 */ 350 STAILQ_HEAD(,camdd_buf) free_queue; 351 352 /* 353 * Free indirect buffers. These are used for breaking a large 354 * buffer into multiple pieces. 355 */ 356 STAILQ_HEAD(,camdd_buf) free_indirect_queue; 357 358 /* 359 * Buffers that have been queued to the kernel. Uses links. 360 */ 361 STAILQ_HEAD(,camdd_buf) active_queue; 362 363 /* 364 * Will generally contain one of our buffers that is waiting for enough 365 * I/O from our partner thread to be able to execute. This will 366 * generally happen when our per-I/O-size is larger than the 367 * partner thread's per-I/O-size. Uses links. 368 */ 369 STAILQ_HEAD(,camdd_buf) pending_queue; 370 371 /* 372 * Number of buffers on the pending queue 373 */ 374 int num_pending_queue; 375 376 /* 377 * Buffers that are filled and ready to execute. This is used when 378 * our partner (reader) thread sends us blocks that are larger than 379 * our blocksize, and so we have to split them into multiple pieces. 380 */ 381 STAILQ_HEAD(,camdd_buf) run_queue; 382 383 /* 384 * Number of buffers on the run queue. 385 */ 386 int num_run_queue; 387 388 STAILQ_HEAD(,camdd_buf) reorder_queue; 389 390 int num_reorder_queue; 391 392 /* 393 * Buffers that have been queued to us by our partner thread 394 * (generally the reader thread) to be written out. Uses 395 * work_links. 396 */ 397 STAILQ_HEAD(,camdd_buf) work_queue; 398 399 /* 400 * Buffers that have been completed by our partner thread. Uses 401 * work_links. 402 */ 403 STAILQ_HEAD(,camdd_buf) peer_done_queue; 404 405 /* 406 * Number of buffers on the peer done queue. 407 */ 408 uint32_t num_peer_done_queue; 409 410 /* 411 * A list of buffers that we have queued to our peer thread. Uses 412 * links. 413 */ 414 STAILQ_HEAD(,camdd_buf) peer_work_queue; 415 416 /* 417 * Number of buffers on the peer work queue. 418 */ 419 uint32_t num_peer_work_queue; 420}; 421 422static sem_t camdd_sem; 423static sig_atomic_t need_exit = 0; 424static sig_atomic_t error_exit = 0; 425static sig_atomic_t need_status = 0; 426 427#ifndef min 428#define min(a, b) (a < b) ? a : b 429#endif 430 431/* 432 * XXX KDM private copy of timespecsub(). This is normally defined in 433 * sys/time.h, but is only enabled in the kernel. If that definition is 434 * enabled in userland, it breaks the build of libnetbsd. 435 */ 436#ifndef timespecsub 437#define timespecsub(vvp, uvp) \ 438 do { \ 439 (vvp)->tv_sec -= (uvp)->tv_sec; \ 440 (vvp)->tv_nsec -= (uvp)->tv_nsec; \ 441 if ((vvp)->tv_nsec < 0) { \ 442 (vvp)->tv_sec--; \ 443 (vvp)->tv_nsec += 1000000000; \ 444 } \ 445 } while (0) 446#endif 447 448 449/* Generically useful offsets into the peripheral private area */ 450#define ppriv_ptr0 periph_priv.entries[0].ptr 451#define ppriv_ptr1 periph_priv.entries[1].ptr 452#define ppriv_field0 periph_priv.entries[0].field 453#define ppriv_field1 periph_priv.entries[1].field 454 455#define ccb_buf ppriv_ptr0 456 457#define CAMDD_FILE_DEFAULT_BLOCK 524288 458#define CAMDD_FILE_DEFAULT_DEPTH 1 459#define CAMDD_PASS_MAX_BLOCK 1048576 460#define CAMDD_PASS_DEFAULT_DEPTH 6 461#define CAMDD_PASS_RW_TIMEOUT 60 * 1000 462 463static int parse_btl(char *tstr, int *bus, int *target, int *lun, 464 camdd_argmask *arglst); 465void camdd_free_dev(struct camdd_dev *dev); 466struct camdd_dev *camdd_alloc_dev(camdd_dev_type dev_type, 467 struct kevent *new_ke, int num_ke, 468 int retry_count, int timeout); 469static struct camdd_buf *camdd_alloc_buf(struct camdd_dev *dev, 470 camdd_buf_type buf_type); 471void camdd_release_buf(struct camdd_buf *buf); 472struct camdd_buf *camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type); 473int camdd_buf_sg_create(struct camdd_buf *buf, int iovec, 474 uint32_t sector_size, uint32_t *num_sectors_used, 475 int *double_buf_needed); 476uint32_t camdd_buf_get_len(struct camdd_buf *buf); 477void camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf); 478int camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize, 479 uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran); 480struct camdd_dev *camdd_probe_file(int fd, struct camdd_io_opts *io_opts, 481 int retry_count, int timeout); 482struct camdd_dev *camdd_probe_pass(struct cam_device *cam_dev, 483 struct camdd_io_opts *io_opts, 484 camdd_argmask arglist, int probe_retry_count, 485 int probe_timeout, int io_retry_count, 486 int io_timeout); 487void *camdd_file_worker(void *arg); 488camdd_buf_status camdd_ccb_status(union ccb *ccb); 489int camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf); 490int camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf); 491void camdd_peer_done(struct camdd_buf *buf); 492void camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf, 493 int *error_count); 494int camdd_pass_fetch(struct camdd_dev *dev); 495int camdd_file_run(struct camdd_dev *dev); 496int camdd_pass_run(struct camdd_dev *dev); 497int camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len); 498int camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf); 499void camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth, 500 uint32_t *peer_depth, uint32_t *our_bytes, 501 uint32_t *peer_bytes); 502void *camdd_worker(void *arg); 503void camdd_sig_handler(int sig); 504void camdd_print_status(struct camdd_dev *camdd_dev, 505 struct camdd_dev *other_dev, 506 struct timespec *start_time); 507int camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts, 508 uint64_t max_io, int retry_count, int timeout); 509int camdd_parse_io_opts(char *args, int is_write, 510 struct camdd_io_opts *io_opts); 511void usage(void); 512 513/* 514 * Parse out a bus, or a bus, target and lun in the following 515 * format: 516 * bus 517 * bus:target 518 * bus:target:lun 519 * 520 * Returns the number of parsed components, or 0. 521 */ 522static int 523parse_btl(char *tstr, int *bus, int *target, int *lun, camdd_argmask *arglst) 524{ 525 char *tmpstr; 526 int convs = 0; 527 528 while (isspace(*tstr) && (*tstr != '\0')) 529 tstr++; 530 531 tmpstr = (char *)strtok(tstr, ":"); 532 if ((tmpstr != NULL) && (*tmpstr != '\0')) { 533 *bus = strtol(tmpstr, NULL, 0); 534 *arglst |= CAMDD_ARG_BUS; 535 convs++; 536 tmpstr = (char *)strtok(NULL, ":"); 537 if ((tmpstr != NULL) && (*tmpstr != '\0')) { 538 *target = strtol(tmpstr, NULL, 0); 539 *arglst |= CAMDD_ARG_TARGET; 540 convs++; 541 tmpstr = (char *)strtok(NULL, ":"); 542 if ((tmpstr != NULL) && (*tmpstr != '\0')) { 543 *lun = strtol(tmpstr, NULL, 0); 544 *arglst |= CAMDD_ARG_LUN; 545 convs++; 546 } 547 } 548 } 549 550 return convs; 551} 552 553/* 554 * XXX KDM clean up and free all of the buffers on the queue! 555 */ 556void 557camdd_free_dev(struct camdd_dev *dev) 558{ 559 if (dev == NULL) 560 return; 561 562 switch (dev->dev_type) { 563 case CAMDD_DEV_FILE: { 564 struct camdd_dev_file *file_dev = &dev->dev_spec.file; 565 566 if (file_dev->fd != -1) 567 close(file_dev->fd); 568 free(file_dev->tmp_buf); 569 break; 570 } 571 case CAMDD_DEV_PASS: { 572 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass; 573 574 if (pass_dev->dev != NULL) 575 cam_close_device(pass_dev->dev); 576 break; 577 } 578 default: 579 break; 580 } 581 582 free(dev); 583} 584 585struct camdd_dev * 586camdd_alloc_dev(camdd_dev_type dev_type, struct kevent *new_ke, int num_ke, 587 int retry_count, int timeout) 588{ 589 struct camdd_dev *dev = NULL; 590 struct kevent *ke; 591 size_t ke_size; 592 int retval = 0; 593 594 dev = calloc(1, sizeof(*dev)); 595 if (dev == NULL) { 596 warn("%s: unable to malloc %zu bytes", __func__, sizeof(*dev)); 597 goto bailout; 598 } 599 600 dev->dev_type = dev_type; 601 dev->io_timeout = timeout; 602 dev->retry_count = retry_count; 603 STAILQ_INIT(&dev->free_queue); 604 STAILQ_INIT(&dev->free_indirect_queue); 605 STAILQ_INIT(&dev->active_queue); 606 STAILQ_INIT(&dev->pending_queue); 607 STAILQ_INIT(&dev->run_queue); 608 STAILQ_INIT(&dev->reorder_queue); 609 STAILQ_INIT(&dev->work_queue); 610 STAILQ_INIT(&dev->peer_done_queue); 611 STAILQ_INIT(&dev->peer_work_queue); 612 retval = pthread_mutex_init(&dev->mutex, NULL); 613 if (retval != 0) { 614 warnc(retval, "%s: failed to initialize mutex", __func__); 615 goto bailout; 616 } 617 618 retval = pthread_cond_init(&dev->cond, NULL); 619 if (retval != 0) { 620 warnc(retval, "%s: failed to initialize condition variable", 621 __func__); 622 goto bailout; 623 } 624 625 dev->kq = kqueue(); 626 if (dev->kq == -1) { 627 warn("%s: Unable to create kqueue", __func__); 628 goto bailout; 629 } 630 631 ke_size = sizeof(struct kevent) * (num_ke + 4); 632 ke = calloc(1, ke_size); 633 if (ke == NULL) { 634 warn("%s: unable to malloc %zu bytes", __func__, ke_size); 635 goto bailout; 636 } 637 if (num_ke > 0) 638 bcopy(new_ke, ke, num_ke * sizeof(struct kevent)); 639 640 EV_SET(&ke[num_ke++], (uintptr_t)&dev->work_queue, EVFILT_USER, 641 EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0); 642 EV_SET(&ke[num_ke++], (uintptr_t)&dev->peer_done_queue, EVFILT_USER, 643 EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0); 644 EV_SET(&ke[num_ke++], SIGINFO, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0); 645 EV_SET(&ke[num_ke++], SIGINT, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0); 646 647 retval = kevent(dev->kq, ke, num_ke, NULL, 0, NULL); 648 if (retval == -1) { 649 warn("%s: Unable to register kevents", __func__); 650 goto bailout; 651 } 652 653 654 return (dev); 655 656bailout: 657 free(dev); 658 659 return (NULL); 660} 661 662static struct camdd_buf * 663camdd_alloc_buf(struct camdd_dev *dev, camdd_buf_type buf_type) 664{ 665 struct camdd_buf *buf = NULL; 666 uint8_t *data_ptr = NULL; 667 668 /* 669 * We only need to allocate data space for data buffers. 670 */ 671 switch (buf_type) { 672 case CAMDD_BUF_DATA: 673 data_ptr = malloc(dev->blocksize); 674 if (data_ptr == NULL) { 675 warn("unable to allocate %u bytes", dev->blocksize); 676 goto bailout_error; 677 } 678 break; 679 default: 680 break; 681 } 682 683 buf = calloc(1, sizeof(*buf)); 684 if (buf == NULL) { 685 warn("unable to allocate %zu bytes", sizeof(*buf)); 686 goto bailout_error; 687 } 688 689 buf->buf_type = buf_type; 690 buf->dev = dev; 691 switch (buf_type) { 692 case CAMDD_BUF_DATA: { 693 struct camdd_buf_data *data; 694 695 data = &buf->buf_type_spec.data; 696 697 data->alloc_len = dev->blocksize; 698 data->buf = data_ptr; 699 break; 700 } 701 case CAMDD_BUF_INDIRECT: 702 break; 703 default: 704 break; 705 } 706 STAILQ_INIT(&buf->src_list); 707 708 return (buf); 709 710bailout_error: 711 free(data_ptr); 712 713 return (NULL); 714} 715 716void 717camdd_release_buf(struct camdd_buf *buf) 718{ 719 struct camdd_dev *dev; 720 721 dev = buf->dev; 722 723 switch (buf->buf_type) { 724 case CAMDD_BUF_DATA: { 725 struct camdd_buf_data *data; 726 727 data = &buf->buf_type_spec.data; 728 729 if (data->segs != NULL) { 730 if (data->extra_buf != 0) { 731 void *extra_buf; 732 733 extra_buf = (void *) 734 data->segs[data->sg_count - 1].ds_addr; 735 free(extra_buf); 736 data->extra_buf = 0; 737 } 738 free(data->segs); 739 data->segs = NULL; 740 data->sg_count = 0; 741 } else if (data->iovec != NULL) { 742 if (data->extra_buf != 0) { 743 free(data->iovec[data->sg_count - 1].iov_base); 744 data->extra_buf = 0; 745 } 746 free(data->iovec); 747 data->iovec = NULL; 748 data->sg_count = 0; 749 } 750 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 751 break; 752 } 753 case CAMDD_BUF_INDIRECT: 754 STAILQ_INSERT_TAIL(&dev->free_indirect_queue, buf, links); 755 break; 756 default: 757 err(1, "%s: Invalid buffer type %d for released buffer", 758 __func__, buf->buf_type); 759 break; 760 } 761} 762 763struct camdd_buf * 764camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type) 765{ 766 struct camdd_buf *buf = NULL; 767 768 switch (buf_type) { 769 case CAMDD_BUF_DATA: 770 buf = STAILQ_FIRST(&dev->free_queue); 771 if (buf != NULL) { 772 struct camdd_buf_data *data; 773 uint8_t *data_ptr; 774 uint32_t alloc_len; 775 776 STAILQ_REMOVE_HEAD(&dev->free_queue, links); 777 data = &buf->buf_type_spec.data; 778 data_ptr = data->buf; 779 alloc_len = data->alloc_len; 780 bzero(buf, sizeof(*buf)); 781 data->buf = data_ptr; 782 data->alloc_len = alloc_len; 783 } 784 break; 785 case CAMDD_BUF_INDIRECT: 786 buf = STAILQ_FIRST(&dev->free_indirect_queue); 787 if (buf != NULL) { 788 STAILQ_REMOVE_HEAD(&dev->free_indirect_queue, links); 789 790 bzero(buf, sizeof(*buf)); 791 } 792 break; 793 default: 794 warnx("Unknown buffer type %d requested", buf_type); 795 break; 796 } 797 798 799 if (buf == NULL) 800 return (camdd_alloc_buf(dev, buf_type)); 801 else { 802 STAILQ_INIT(&buf->src_list); 803 buf->dev = dev; 804 buf->buf_type = buf_type; 805 806 return (buf); 807 } 808} 809 810int 811camdd_buf_sg_create(struct camdd_buf *buf, int iovec, uint32_t sector_size, 812 uint32_t *num_sectors_used, int *double_buf_needed) 813{ 814 struct camdd_buf *tmp_buf; 815 struct camdd_buf_data *data; 816 uint8_t *extra_buf = NULL; 817 size_t extra_buf_len = 0; 818 int i, retval = 0; 819 820 data = &buf->buf_type_spec.data; 821 822 data->sg_count = buf->src_count; 823 /* 824 * Compose a scatter/gather list from all of the buffers in the list. 825 * If the length of the buffer isn't a multiple of the sector size, 826 * we'll have to add an extra buffer. This should only happen 827 * at the end of a transfer. 828 */ 829 if ((data->fill_len % sector_size) != 0) { 830 extra_buf_len = sector_size - (data->fill_len % sector_size); 831 extra_buf = calloc(extra_buf_len, 1); 832 if (extra_buf == NULL) { 833 warn("%s: unable to allocate %zu bytes for extra " 834 "buffer space", __func__, extra_buf_len); 835 retval = 1; 836 goto bailout; 837 } 838 data->extra_buf = 1; 839 data->sg_count++; 840 } 841 if (iovec == 0) { 842 data->segs = calloc(data->sg_count, sizeof(bus_dma_segment_t)); 843 if (data->segs == NULL) { 844 warn("%s: unable to allocate %zu bytes for S/G list", 845 __func__, sizeof(bus_dma_segment_t) * 846 data->sg_count); 847 retval = 1; 848 goto bailout; 849 } 850 851 } else { 852 data->iovec = calloc(data->sg_count, sizeof(struct iovec)); 853 if (data->iovec == NULL) { 854 warn("%s: unable to allocate %zu bytes for S/G list", 855 __func__, sizeof(struct iovec) * data->sg_count); 856 retval = 1; 857 goto bailout; 858 } 859 } 860 861 for (i = 0, tmp_buf = STAILQ_FIRST(&buf->src_list); 862 i < buf->src_count && tmp_buf != NULL; i++, 863 tmp_buf = STAILQ_NEXT(tmp_buf, src_links)) { 864 865 if (tmp_buf->buf_type == CAMDD_BUF_DATA) { 866 struct camdd_buf_data *tmp_data; 867 868 tmp_data = &tmp_buf->buf_type_spec.data; 869 if (iovec == 0) { 870 data->segs[i].ds_addr = 871 (bus_addr_t) tmp_data->buf; 872 data->segs[i].ds_len = tmp_data->fill_len - 873 tmp_data->resid; 874 } else { 875 data->iovec[i].iov_base = tmp_data->buf; 876 data->iovec[i].iov_len = tmp_data->fill_len - 877 tmp_data->resid; 878 } 879 if (((tmp_data->fill_len - tmp_data->resid) % 880 sector_size) != 0) 881 *double_buf_needed = 1; 882 } else { 883 struct camdd_buf_indirect *tmp_ind; 884 885 tmp_ind = &tmp_buf->buf_type_spec.indirect; 886 if (iovec == 0) { 887 data->segs[i].ds_addr = 888 (bus_addr_t)tmp_ind->start_ptr; 889 data->segs[i].ds_len = tmp_ind->len; 890 } else { 891 data->iovec[i].iov_base = tmp_ind->start_ptr; 892 data->iovec[i].iov_len = tmp_ind->len; 893 } 894 if ((tmp_ind->len % sector_size) != 0) 895 *double_buf_needed = 1; 896 } 897 } 898 899 if (extra_buf != NULL) { 900 if (iovec == 0) { 901 data->segs[i].ds_addr = (bus_addr_t)extra_buf; 902 data->segs[i].ds_len = extra_buf_len; 903 } else { 904 data->iovec[i].iov_base = extra_buf; 905 data->iovec[i].iov_len = extra_buf_len; 906 } 907 i++; 908 } 909 if ((tmp_buf != NULL) || (i != data->sg_count)) { 910 warnx("buffer source count does not match " 911 "number of buffers in list!"); 912 retval = 1; 913 goto bailout; 914 } 915 916bailout: 917 if (retval == 0) { 918 *num_sectors_used = (data->fill_len + extra_buf_len) / 919 sector_size; 920 } 921 return (retval); 922} 923 924uint32_t 925camdd_buf_get_len(struct camdd_buf *buf) 926{ 927 uint32_t len = 0; 928 929 if (buf->buf_type != CAMDD_BUF_DATA) { 930 struct camdd_buf_indirect *indirect; 931 932 indirect = &buf->buf_type_spec.indirect; 933 len = indirect->len; 934 } else { 935 struct camdd_buf_data *data; 936 937 data = &buf->buf_type_spec.data; 938 len = data->fill_len; 939 } 940 941 return (len); 942} 943 944void 945camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf) 946{ 947 struct camdd_buf_data *data; 948 949 assert(buf->buf_type == CAMDD_BUF_DATA); 950 951 data = &buf->buf_type_spec.data; 952 953 STAILQ_INSERT_TAIL(&buf->src_list, child_buf, src_links); 954 buf->src_count++; 955 956 data->fill_len += camdd_buf_get_len(child_buf); 957} 958 959typedef enum { 960 CAMDD_TS_MAX_BLK, 961 CAMDD_TS_MIN_BLK, 962 CAMDD_TS_BLK_GRAN, 963 CAMDD_TS_EFF_IOSIZE 964} camdd_status_item_index; 965 966static struct camdd_status_items { 967 const char *name; 968 struct mt_status_entry *entry; 969} req_status_items[] = { 970 { "max_blk", NULL }, 971 { "min_blk", NULL }, 972 { "blk_gran", NULL }, 973 { "max_effective_iosize", NULL } 974}; 975 976int 977camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize, 978 uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran) 979{ 980 struct mt_status_data status_data; 981 char *xml_str = NULL; 982 unsigned int i; 983 int retval = 0; 984 985 retval = mt_get_xml_str(fd, MTIOCEXTGET, &xml_str); 986 if (retval != 0) 987 err(1, "Couldn't get XML string from %s", filename); 988 989 retval = mt_get_status(xml_str, &status_data); 990 if (retval != XML_STATUS_OK) { 991 warn("couldn't get status for %s", filename); 992 retval = 1; 993 goto bailout; 994 } else 995 retval = 0; 996 997 if (status_data.error != 0) { 998 warnx("%s", status_data.error_str); 999 retval = 1; 1000 goto bailout; 1001 } 1002 1003 for (i = 0; i < nitems(req_status_items); i++) { 1004 char *name; 1005 1006 name = __DECONST(char *, req_status_items[i].name); 1007 req_status_items[i].entry = mt_status_entry_find(&status_data, 1008 name); 1009 if (req_status_items[i].entry == NULL) { 1010 errx(1, "Cannot find status entry %s", 1011 req_status_items[i].name); 1012 } 1013 } 1014 1015 *max_iosize = req_status_items[CAMDD_TS_EFF_IOSIZE].entry->value_unsigned; 1016 *max_blk= req_status_items[CAMDD_TS_MAX_BLK].entry->value_unsigned; 1017 *min_blk= req_status_items[CAMDD_TS_MIN_BLK].entry->value_unsigned; 1018 *blk_gran = req_status_items[CAMDD_TS_BLK_GRAN].entry->value_unsigned; 1019bailout: 1020 1021 free(xml_str); 1022 mt_status_free(&status_data); 1023 1024 return (retval); 1025} 1026 1027struct camdd_dev * 1028camdd_probe_file(int fd, struct camdd_io_opts *io_opts, int retry_count, 1029 int timeout) 1030{ 1031 struct camdd_dev *dev = NULL; 1032 struct camdd_dev_file *file_dev; 1033 uint64_t blocksize = io_opts->blocksize; 1034 1035 dev = camdd_alloc_dev(CAMDD_DEV_FILE, NULL, 0, retry_count, timeout); 1036 if (dev == NULL) 1037 goto bailout; 1038 1039 file_dev = &dev->dev_spec.file; 1040 file_dev->fd = fd; 1041 strlcpy(file_dev->filename, io_opts->dev_name, 1042 sizeof(file_dev->filename)); 1043 strlcpy(dev->device_name, io_opts->dev_name, sizeof(dev->device_name)); 1044 if (blocksize == 0) 1045 dev->blocksize = CAMDD_FILE_DEFAULT_BLOCK; 1046 else 1047 dev->blocksize = blocksize; 1048 1049 if ((io_opts->queue_depth != 0) 1050 && (io_opts->queue_depth != 1)) { 1051 warnx("Queue depth %ju for %s ignored, only 1 outstanding " 1052 "command supported", (uintmax_t)io_opts->queue_depth, 1053 io_opts->dev_name); 1054 } 1055 dev->target_queue_depth = CAMDD_FILE_DEFAULT_DEPTH; 1056 dev->run = camdd_file_run; 1057 dev->fetch = NULL; 1058 1059 /* 1060 * We can effectively access files on byte boundaries. We'll reset 1061 * this for devices like disks that can be accessed on sector 1062 * boundaries. 1063 */ 1064 dev->sector_size = 1; 1065 1066 if ((fd != STDIN_FILENO) 1067 && (fd != STDOUT_FILENO)) { 1068 int retval; 1069 1070 retval = fstat(fd, &file_dev->sb); 1071 if (retval != 0) { 1072 warn("Cannot stat %s", dev->device_name); 1073 goto bailout_error; 1074 } 1075 if (S_ISREG(file_dev->sb.st_mode)) { 1076 file_dev->file_type = CAMDD_FILE_REG; 1077 } else if (S_ISCHR(file_dev->sb.st_mode)) { 1078 int type; 1079 1080 if (ioctl(fd, FIODTYPE, &type) == -1) 1081 err(1, "FIODTYPE ioctl failed on %s", 1082 dev->device_name); 1083 else { 1084 if (type & D_TAPE) 1085 file_dev->file_type = CAMDD_FILE_TAPE; 1086 else if (type & D_DISK) 1087 file_dev->file_type = CAMDD_FILE_DISK; 1088 else if (type & D_MEM) 1089 file_dev->file_type = CAMDD_FILE_MEM; 1090 else if (type & D_TTY) 1091 file_dev->file_type = CAMDD_FILE_TTY; 1092 } 1093 } else if (S_ISDIR(file_dev->sb.st_mode)) { 1094 errx(1, "cannot operate on directory %s", 1095 dev->device_name); 1096 } else if (S_ISFIFO(file_dev->sb.st_mode)) { 1097 file_dev->file_type = CAMDD_FILE_PIPE; 1098 } else 1099 errx(1, "Cannot determine file type for %s", 1100 dev->device_name); 1101 1102 switch (file_dev->file_type) { 1103 case CAMDD_FILE_REG: 1104 if (file_dev->sb.st_size != 0) 1105 dev->max_sector = file_dev->sb.st_size - 1; 1106 else 1107 dev->max_sector = 0; 1108 file_dev->file_flags |= CAMDD_FF_CAN_SEEK; 1109 break; 1110 case CAMDD_FILE_TAPE: { 1111 uint64_t max_iosize, max_blk, min_blk, blk_gran; 1112 /* 1113 * Check block limits and maximum effective iosize. 1114 * Make sure the blocksize is within the block 1115 * limits (and a multiple of the minimum blocksize) 1116 * and that the blocksize is <= maximum effective 1117 * iosize. 1118 */ 1119 retval = camdd_probe_tape(fd, dev->device_name, 1120 &max_iosize, &max_blk, &min_blk, &blk_gran); 1121 if (retval != 0) 1122 errx(1, "Unable to probe tape %s", 1123 dev->device_name); 1124 1125 /* 1126 * The blocksize needs to be <= the maximum 1127 * effective I/O size of the tape device. Note 1128 * that this also takes into account the maximum 1129 * blocksize reported by READ BLOCK LIMITS. 1130 */ 1131 if (dev->blocksize > max_iosize) { 1132 warnx("Blocksize %u too big for %s, limiting " 1133 "to %ju", dev->blocksize, dev->device_name, 1134 max_iosize); 1135 dev->blocksize = max_iosize; 1136 } 1137 1138 /* 1139 * The blocksize needs to be at least min_blk; 1140 */ 1141 if (dev->blocksize < min_blk) { 1142 warnx("Blocksize %u too small for %s, " 1143 "increasing to %ju", dev->blocksize, 1144 dev->device_name, min_blk); 1145 dev->blocksize = min_blk; 1146 } 1147 1148 /* 1149 * And the blocksize needs to be a multiple of 1150 * the block granularity. 1151 */ 1152 if ((blk_gran != 0) 1153 && (dev->blocksize % (1 << blk_gran))) { 1154 warnx("Blocksize %u for %s not a multiple of " 1155 "%d, adjusting to %d", dev->blocksize, 1156 dev->device_name, (1 << blk_gran), 1157 dev->blocksize & ~((1 << blk_gran) - 1)); 1158 dev->blocksize &= ~((1 << blk_gran) - 1); 1159 } 1160 1161 if (dev->blocksize == 0) { 1162 errx(1, "Unable to derive valid blocksize for " 1163 "%s", dev->device_name); 1164 } 1165 1166 /* 1167 * For tape drives, set the sector size to the 1168 * blocksize so that we make sure not to write 1169 * less than the blocksize out to the drive. 1170 */ 1171 dev->sector_size = dev->blocksize; 1172 break; 1173 } 1174 case CAMDD_FILE_DISK: { 1175 off_t media_size; 1176 unsigned int sector_size; 1177 1178 file_dev->file_flags |= CAMDD_FF_CAN_SEEK; 1179 1180 if (ioctl(fd, DIOCGSECTORSIZE, §or_size) == -1) { 1181 err(1, "DIOCGSECTORSIZE ioctl failed on %s", 1182 dev->device_name); 1183 } 1184 1185 if (sector_size == 0) { 1186 errx(1, "DIOCGSECTORSIZE ioctl returned " 1187 "invalid sector size %u for %s", 1188 sector_size, dev->device_name); 1189 } 1190 1191 if (ioctl(fd, DIOCGMEDIASIZE, &media_size) == -1) { 1192 err(1, "DIOCGMEDIASIZE ioctl failed on %s", 1193 dev->device_name); 1194 } 1195 1196 if (media_size == 0) { 1197 errx(1, "DIOCGMEDIASIZE ioctl returned " 1198 "invalid media size %ju for %s", 1199 (uintmax_t)media_size, dev->device_name); 1200 } 1201 1202 if (dev->blocksize % sector_size) { 1203 errx(1, "%s blocksize %u not a multiple of " 1204 "sector size %u", dev->device_name, 1205 dev->blocksize, sector_size); 1206 } 1207 1208 dev->sector_size = sector_size; 1209 dev->max_sector = (media_size / sector_size) - 1; 1210 break; 1211 } 1212 case CAMDD_FILE_MEM: 1213 file_dev->file_flags |= CAMDD_FF_CAN_SEEK; 1214 break; 1215 default: 1216 break; 1217 } 1218 } 1219 1220 if ((io_opts->offset != 0) 1221 && ((file_dev->file_flags & CAMDD_FF_CAN_SEEK) == 0)) { 1222 warnx("Offset %ju specified for %s, but we cannot seek on %s", 1223 io_opts->offset, io_opts->dev_name, io_opts->dev_name); 1224 goto bailout_error; 1225 } 1226#if 0 1227 else if ((io_opts->offset != 0) 1228 && ((io_opts->offset % dev->sector_size) != 0)) { 1229 warnx("Offset %ju for %s is not a multiple of the " 1230 "sector size %u", io_opts->offset, 1231 io_opts->dev_name, dev->sector_size); 1232 goto bailout_error; 1233 } else { 1234 dev->start_offset_bytes = io_opts->offset; 1235 } 1236#endif 1237 1238bailout: 1239 return (dev); 1240 1241bailout_error: 1242 camdd_free_dev(dev); 1243 return (NULL); 1244} 1245 1246/* 1247 * Need to implement this. Do a basic probe: 1248 * - Check the inquiry data, make sure we're talking to a device that we 1249 * can reasonably expect to talk to -- direct, RBC, CD, WORM. 1250 * - Send a test unit ready, make sure the device is available. 1251 * - Get the capacity and block size. 1252 */ 1253struct camdd_dev * 1254camdd_probe_pass(struct cam_device *cam_dev, struct camdd_io_opts *io_opts, 1255 camdd_argmask arglist, int probe_retry_count, 1256 int probe_timeout, int io_retry_count, int io_timeout) 1257{ 1258 union ccb *ccb; 1259 uint64_t maxsector; 1260 uint32_t cpi_maxio, max_iosize, pass_numblocks; 1261 uint32_t block_len; 1262 struct scsi_read_capacity_data rcap; 1263 struct scsi_read_capacity_data_long rcaplong; 1264 struct camdd_dev *dev; 1265 struct camdd_dev_pass *pass_dev; 1266 struct kevent ke; 1267 int scsi_dev_type; 1268 1269 dev = NULL; 1270 1271 scsi_dev_type = SID_TYPE(&cam_dev->inq_data); 1272 maxsector = 0; 1273 block_len = 0; 1274 1275 /* 1276 * For devices that support READ CAPACITY, we'll attempt to get the 1277 * capacity. Otherwise, we really don't support tape or other 1278 * devices via SCSI passthrough, so just return an error in that case. 1279 */ 1280 switch (scsi_dev_type) { 1281 case T_DIRECT: 1282 case T_WORM: 1283 case T_CDROM: 1284 case T_OPTICAL: 1285 case T_RBC: 1286 case T_ZBC_HM: 1287 break; 1288 default: 1289 errx(1, "Unsupported SCSI device type %d", scsi_dev_type); 1290 break; /*NOTREACHED*/ 1291 } 1292 1293 ccb = cam_getccb(cam_dev); 1294 1295 if (ccb == NULL) { 1296 warnx("%s: error allocating ccb", __func__); 1297 goto bailout; 1298 } 1299 1300 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->csio); 1301 1302 scsi_read_capacity(&ccb->csio, 1303 /*retries*/ probe_retry_count, 1304 /*cbfcnp*/ NULL, 1305 /*tag_action*/ MSG_SIMPLE_Q_TAG, 1306 &rcap, 1307 SSD_FULL_SIZE, 1308 /*timeout*/ probe_timeout ? probe_timeout : 5000); 1309 1310 /* Disable freezing the device queue */ 1311 ccb->ccb_h.flags |= CAM_DEV_QFRZDIS; 1312 1313 if (arglist & CAMDD_ARG_ERR_RECOVER) 1314 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER; 1315 1316 if (cam_send_ccb(cam_dev, ccb) < 0) { 1317 warn("error sending READ CAPACITY command"); 1318 1319 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, 1320 CAM_EPF_ALL, stderr); 1321 1322 goto bailout; 1323 } 1324 1325 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1326 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr); 1327 goto bailout; 1328 } 1329 1330 maxsector = scsi_4btoul(rcap.addr); 1331 block_len = scsi_4btoul(rcap.length); 1332 1333 /* 1334 * A last block of 2^32-1 means that the true capacity is over 2TB, 1335 * and we need to issue the long READ CAPACITY to get the real 1336 * capacity. Otherwise, we're all set. 1337 */ 1338 if (maxsector != 0xffffffff) 1339 goto rcap_done; 1340 1341 scsi_read_capacity_16(&ccb->csio, 1342 /*retries*/ probe_retry_count, 1343 /*cbfcnp*/ NULL, 1344 /*tag_action*/ MSG_SIMPLE_Q_TAG, 1345 /*lba*/ 0, 1346 /*reladdr*/ 0, 1347 /*pmi*/ 0, 1348 (uint8_t *)&rcaplong, 1349 sizeof(rcaplong), 1350 /*sense_len*/ SSD_FULL_SIZE, 1351 /*timeout*/ probe_timeout ? probe_timeout : 5000); 1352 1353 /* Disable freezing the device queue */ 1354 ccb->ccb_h.flags |= CAM_DEV_QFRZDIS; 1355 1356 if (arglist & CAMDD_ARG_ERR_RECOVER) 1357 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER; 1358 1359 if (cam_send_ccb(cam_dev, ccb) < 0) { 1360 warn("error sending READ CAPACITY (16) command"); 1361 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, 1362 CAM_EPF_ALL, stderr); 1363 goto bailout; 1364 } 1365 1366 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1367 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr); 1368 goto bailout; 1369 } 1370 1371 maxsector = scsi_8btou64(rcaplong.addr); 1372 block_len = scsi_4btoul(rcaplong.length); 1373 1374rcap_done: 1375 if (block_len == 0) { 1376 warnx("Sector size for %s%u is 0, cannot continue", 1377 cam_dev->device_name, cam_dev->dev_unit_num); 1378 goto bailout_error; 1379 } 1380 1381 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->cpi); 1382 1383 ccb->ccb_h.func_code = XPT_PATH_INQ; 1384 ccb->ccb_h.flags = CAM_DIR_NONE; 1385 ccb->ccb_h.retry_count = 1; 1386 1387 if (cam_send_ccb(cam_dev, ccb) < 0) { 1388 warn("error sending XPT_PATH_INQ CCB"); 1389 1390 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, 1391 CAM_EPF_ALL, stderr); 1392 goto bailout; 1393 } 1394 1395 EV_SET(&ke, cam_dev->fd, EVFILT_READ, EV_ADD|EV_ENABLE, 0, 0, 0); 1396 1397 dev = camdd_alloc_dev(CAMDD_DEV_PASS, &ke, 1, io_retry_count, 1398 io_timeout); 1399 if (dev == NULL) 1400 goto bailout; 1401 1402 pass_dev = &dev->dev_spec.pass; 1403 pass_dev->scsi_dev_type = scsi_dev_type; 1404 pass_dev->dev = cam_dev; 1405 pass_dev->max_sector = maxsector; 1406 pass_dev->block_len = block_len; 1407 pass_dev->cpi_maxio = ccb->cpi.maxio; 1408 snprintf(dev->device_name, sizeof(dev->device_name), "%s%u", 1409 pass_dev->dev->device_name, pass_dev->dev->dev_unit_num); 1410 dev->sector_size = block_len; 1411 dev->max_sector = maxsector; 1412 1413 1414 /* 1415 * Determine the optimal blocksize to use for this device. 1416 */ 1417 1418 /* 1419 * If the controller has not specified a maximum I/O size, 1420 * just go with 128K as a somewhat conservative value. 1421 */ 1422 if (pass_dev->cpi_maxio == 0) 1423 cpi_maxio = 131072; 1424 else 1425 cpi_maxio = pass_dev->cpi_maxio; 1426 1427 /* 1428 * If the controller has a large maximum I/O size, limit it 1429 * to something smaller so that the kernel doesn't have trouble 1430 * allocating buffers to copy data in and out for us. 1431 * XXX KDM this is until we have unmapped I/O support in the kernel. 1432 */ 1433 max_iosize = min(cpi_maxio, CAMDD_PASS_MAX_BLOCK); 1434 1435 /* 1436 * If we weren't able to get a block size for some reason, 1437 * default to 512 bytes. 1438 */ 1439 block_len = pass_dev->block_len; 1440 if (block_len == 0) 1441 block_len = 512; 1442 1443 /* 1444 * Figure out how many blocksize chunks will fit in the 1445 * maximum I/O size. 1446 */ 1447 pass_numblocks = max_iosize / block_len; 1448 1449 /* 1450 * And finally, multiple the number of blocks by the LBA 1451 * length to get our maximum block size; 1452 */ 1453 dev->blocksize = pass_numblocks * block_len; 1454 1455 if (io_opts->blocksize != 0) { 1456 if ((io_opts->blocksize % dev->sector_size) != 0) { 1457 warnx("Blocksize %ju for %s is not a multiple of " 1458 "sector size %u", (uintmax_t)io_opts->blocksize, 1459 dev->device_name, dev->sector_size); 1460 goto bailout_error; 1461 } 1462 dev->blocksize = io_opts->blocksize; 1463 } 1464 dev->target_queue_depth = CAMDD_PASS_DEFAULT_DEPTH; 1465 if (io_opts->queue_depth != 0) 1466 dev->target_queue_depth = io_opts->queue_depth; 1467 1468 if (io_opts->offset != 0) { 1469 if (io_opts->offset > (dev->max_sector * dev->sector_size)) { 1470 warnx("Offset %ju is past the end of device %s", 1471 io_opts->offset, dev->device_name); 1472 goto bailout_error; 1473 } 1474#if 0 1475 else if ((io_opts->offset % dev->sector_size) != 0) { 1476 warnx("Offset %ju for %s is not a multiple of the " 1477 "sector size %u", io_opts->offset, 1478 dev->device_name, dev->sector_size); 1479 goto bailout_error; 1480 } 1481 dev->start_offset_bytes = io_opts->offset; 1482#endif 1483 } 1484 1485 dev->min_cmd_size = io_opts->min_cmd_size; 1486 1487 dev->run = camdd_pass_run; 1488 dev->fetch = camdd_pass_fetch; 1489 1490bailout: 1491 cam_freeccb(ccb); 1492 1493 return (dev); 1494 1495bailout_error: 1496 cam_freeccb(ccb); 1497 1498 camdd_free_dev(dev); 1499 1500 return (NULL); 1501} 1502 1503void * 1504camdd_worker(void *arg) 1505{ 1506 struct camdd_dev *dev = arg; 1507 struct camdd_buf *buf; 1508 struct timespec ts, *kq_ts; 1509 1510 ts.tv_sec = 0; 1511 ts.tv_nsec = 0; 1512 1513 pthread_mutex_lock(&dev->mutex); 1514 1515 dev->flags |= CAMDD_DEV_FLAG_ACTIVE; 1516 1517 for (;;) { 1518 struct kevent ke; 1519 int retval = 0; 1520 1521 /* 1522 * XXX KDM check the reorder queue depth? 1523 */ 1524 if (dev->write_dev == 0) { 1525 uint32_t our_depth, peer_depth, peer_bytes, our_bytes; 1526 uint32_t target_depth = dev->target_queue_depth; 1527 uint32_t peer_target_depth = 1528 dev->peer_dev->target_queue_depth; 1529 uint32_t peer_blocksize = dev->peer_dev->blocksize; 1530 1531 camdd_get_depth(dev, &our_depth, &peer_depth, 1532 &our_bytes, &peer_bytes); 1533 1534#if 0 1535 while (((our_depth < target_depth) 1536 && (peer_depth < peer_target_depth)) 1537 || ((peer_bytes + our_bytes) < 1538 (peer_blocksize * 2))) { 1539#endif 1540 while (((our_depth + peer_depth) < 1541 (target_depth + peer_target_depth)) 1542 || ((peer_bytes + our_bytes) < 1543 (peer_blocksize * 3))) { 1544 1545 retval = camdd_queue(dev, NULL); 1546 if (retval == 1) 1547 break; 1548 else if (retval != 0) { 1549 error_exit = 1; 1550 goto bailout; 1551 } 1552 1553 camdd_get_depth(dev, &our_depth, &peer_depth, 1554 &our_bytes, &peer_bytes); 1555 } 1556 } 1557 /* 1558 * See if we have any I/O that is ready to execute. 1559 */ 1560 buf = STAILQ_FIRST(&dev->run_queue); 1561 if (buf != NULL) { 1562 while (dev->target_queue_depth > dev->cur_active_io) { 1563 retval = dev->run(dev); 1564 if (retval == -1) { 1565 dev->flags |= CAMDD_DEV_FLAG_EOF; 1566 error_exit = 1; 1567 break; 1568 } else if (retval != 0) { 1569 break; 1570 } 1571 } 1572 } 1573 1574 /* 1575 * We've reached EOF, or our partner has reached EOF. 1576 */ 1577 if ((dev->flags & CAMDD_DEV_FLAG_EOF) 1578 || (dev->flags & CAMDD_DEV_FLAG_PEER_EOF)) { 1579 if (dev->write_dev != 0) { 1580 if ((STAILQ_EMPTY(&dev->work_queue)) 1581 && (dev->num_run_queue == 0) 1582 && (dev->cur_active_io == 0)) { 1583 goto bailout; 1584 } 1585 } else { 1586 /* 1587 * If we're the reader, and the writer 1588 * got EOF, he is already done. If we got 1589 * the EOF, then we need to wait until 1590 * everything is flushed out for the writer. 1591 */ 1592 if (dev->flags & CAMDD_DEV_FLAG_PEER_EOF) { 1593 goto bailout; 1594 } else if ((dev->num_peer_work_queue == 0) 1595 && (dev->num_peer_done_queue == 0) 1596 && (dev->cur_active_io == 0) 1597 && (dev->num_run_queue == 0)) { 1598 goto bailout; 1599 } 1600 } 1601 /* 1602 * XXX KDM need to do something about the pending 1603 * queue and cleanup resources. 1604 */ 1605 } 1606 1607 if ((dev->write_dev == 0) 1608 && (dev->cur_active_io == 0) 1609 && (dev->peer_bytes_queued < dev->peer_dev->blocksize)) 1610 kq_ts = &ts; 1611 else 1612 kq_ts = NULL; 1613 1614 /* 1615 * Run kevent to see if there are events to process. 1616 */ 1617 pthread_mutex_unlock(&dev->mutex); 1618 retval = kevent(dev->kq, NULL, 0, &ke, 1, kq_ts); 1619 pthread_mutex_lock(&dev->mutex); 1620 if (retval == -1) { 1621 warn("%s: error returned from kevent",__func__); 1622 goto bailout; 1623 } else if (retval != 0) { 1624 switch (ke.filter) { 1625 case EVFILT_READ: 1626 if (dev->fetch != NULL) { 1627 retval = dev->fetch(dev); 1628 if (retval == -1) { 1629 error_exit = 1; 1630 goto bailout; 1631 } 1632 } 1633 break; 1634 case EVFILT_SIGNAL: 1635 /* 1636 * We register for this so we don't get 1637 * an error as a result of a SIGINFO or a 1638 * SIGINT. It will actually get handled 1639 * by the signal handler. If we get a 1640 * SIGINT, bail out without printing an 1641 * error message. Any other signals 1642 * will result in the error message above. 1643 */ 1644 if (ke.ident == SIGINT) 1645 goto bailout; 1646 break; 1647 case EVFILT_USER: 1648 retval = 0; 1649 /* 1650 * Check to see if the other thread has 1651 * queued any I/O for us to do. (In this 1652 * case we're the writer.) 1653 */ 1654 for (buf = STAILQ_FIRST(&dev->work_queue); 1655 buf != NULL; 1656 buf = STAILQ_FIRST(&dev->work_queue)) { 1657 STAILQ_REMOVE_HEAD(&dev->work_queue, 1658 work_links); 1659 retval = camdd_queue(dev, buf); 1660 /* 1661 * We keep going unless we get an 1662 * actual error. If we get EOF, we 1663 * still want to remove the buffers 1664 * from the queue and send the back 1665 * to the reader thread. 1666 */ 1667 if (retval == -1) { 1668 error_exit = 1; 1669 goto bailout; 1670 } else 1671 retval = 0; 1672 } 1673 1674 /* 1675 * Next check to see if the other thread has 1676 * queued any completed buffers back to us. 1677 * (In this case we're the reader.) 1678 */ 1679 for (buf = STAILQ_FIRST(&dev->peer_done_queue); 1680 buf != NULL; 1681 buf = STAILQ_FIRST(&dev->peer_done_queue)){ 1682 STAILQ_REMOVE_HEAD( 1683 &dev->peer_done_queue, work_links); 1684 dev->num_peer_done_queue--; 1685 camdd_peer_done(buf); 1686 } 1687 break; 1688 default: 1689 warnx("%s: unknown kevent filter %d", 1690 __func__, ke.filter); 1691 break; 1692 } 1693 } 1694 } 1695 1696bailout: 1697 1698 dev->flags &= ~CAMDD_DEV_FLAG_ACTIVE; 1699 1700 /* XXX KDM cleanup resources here? */ 1701 1702 pthread_mutex_unlock(&dev->mutex); 1703 1704 need_exit = 1; 1705 sem_post(&camdd_sem); 1706 1707 return (NULL); 1708} 1709 1710/* 1711 * Simplistic translation of CCB status to our local status. 1712 */ 1713camdd_buf_status 1714camdd_ccb_status(union ccb *ccb) 1715{ 1716 camdd_buf_status status = CAMDD_STATUS_NONE; 1717 cam_status ccb_status; 1718 1719 ccb_status = ccb->ccb_h.status & CAM_STATUS_MASK; 1720 1721 switch (ccb_status) { 1722 case CAM_REQ_CMP: { 1723 if (ccb->csio.resid == 0) { 1724 status = CAMDD_STATUS_OK; 1725 } else if (ccb->csio.dxfer_len > ccb->csio.resid) { 1726 status = CAMDD_STATUS_SHORT_IO; 1727 } else { 1728 status = CAMDD_STATUS_EOF; 1729 } 1730 break; 1731 } 1732 case CAM_SCSI_STATUS_ERROR: { 1733 switch (ccb->csio.scsi_status) { 1734 case SCSI_STATUS_OK: 1735 case SCSI_STATUS_COND_MET: 1736 case SCSI_STATUS_INTERMED: 1737 case SCSI_STATUS_INTERMED_COND_MET: 1738 status = CAMDD_STATUS_OK; 1739 break; 1740 case SCSI_STATUS_CMD_TERMINATED: 1741 case SCSI_STATUS_CHECK_COND: 1742 case SCSI_STATUS_QUEUE_FULL: 1743 case SCSI_STATUS_BUSY: 1744 case SCSI_STATUS_RESERV_CONFLICT: 1745 default: 1746 status = CAMDD_STATUS_ERROR; 1747 break; 1748 } 1749 break; 1750 } 1751 default: 1752 status = CAMDD_STATUS_ERROR; 1753 break; 1754 } 1755 1756 return (status); 1757} 1758 1759/* 1760 * Queue a buffer to our peer's work thread for writing. 1761 * 1762 * Returns 0 for success, -1 for failure, 1 if the other thread exited. 1763 */ 1764int 1765camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf) 1766{ 1767 struct kevent ke; 1768 STAILQ_HEAD(, camdd_buf) local_queue; 1769 struct camdd_buf *buf1, *buf2; 1770 struct camdd_buf_data *data = NULL; 1771 uint64_t peer_bytes_queued = 0; 1772 int active = 1; 1773 int retval = 0; 1774 1775 STAILQ_INIT(&local_queue); 1776 1777 /* 1778 * Since we're the reader, we need to queue our I/O to the writer 1779 * in sequential order in order to make sure it gets written out 1780 * in sequential order. 1781 * 1782 * Check the next expected I/O starting offset. If this doesn't 1783 * match, put it on the reorder queue. 1784 */ 1785 if ((buf->lba * dev->sector_size) != dev->next_completion_pos_bytes) { 1786 1787 /* 1788 * If there is nothing on the queue, there is no sorting 1789 * needed. 1790 */ 1791 if (STAILQ_EMPTY(&dev->reorder_queue)) { 1792 STAILQ_INSERT_TAIL(&dev->reorder_queue, buf, links); 1793 dev->num_reorder_queue++; 1794 goto bailout; 1795 } 1796 1797 /* 1798 * Sort in ascending order by starting LBA. There should 1799 * be no identical LBAs. 1800 */ 1801 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL; 1802 buf1 = buf2) { 1803 buf2 = STAILQ_NEXT(buf1, links); 1804 if (buf->lba < buf1->lba) { 1805 /* 1806 * If we're less than the first one, then 1807 * we insert at the head of the list 1808 * because this has to be the first element 1809 * on the list. 1810 */ 1811 STAILQ_INSERT_HEAD(&dev->reorder_queue, 1812 buf, links); 1813 dev->num_reorder_queue++; 1814 break; 1815 } else if (buf->lba > buf1->lba) { 1816 if (buf2 == NULL) { 1817 STAILQ_INSERT_TAIL(&dev->reorder_queue, 1818 buf, links); 1819 dev->num_reorder_queue++; 1820 break; 1821 } else if (buf->lba < buf2->lba) { 1822 STAILQ_INSERT_AFTER(&dev->reorder_queue, 1823 buf1, buf, links); 1824 dev->num_reorder_queue++; 1825 break; 1826 } 1827 } else { 1828 errx(1, "Found buffers with duplicate LBA %ju!", 1829 buf->lba); 1830 } 1831 } 1832 goto bailout; 1833 } else { 1834 1835 /* 1836 * We're the next expected I/O completion, so put ourselves 1837 * on the local queue to be sent to the writer. We use 1838 * work_links here so that we can queue this to the 1839 * peer_work_queue before taking the buffer off of the 1840 * local_queue. 1841 */ 1842 dev->next_completion_pos_bytes += buf->len; 1843 STAILQ_INSERT_TAIL(&local_queue, buf, work_links); 1844 1845 /* 1846 * Go through the reorder queue looking for more sequential 1847 * I/O and add it to the local queue. 1848 */ 1849 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL; 1850 buf1 = STAILQ_FIRST(&dev->reorder_queue)) { 1851 /* 1852 * As soon as we see an I/O that is out of sequence, 1853 * we're done. 1854 */ 1855 if ((buf1->lba * dev->sector_size) != 1856 dev->next_completion_pos_bytes) 1857 break; 1858 1859 STAILQ_REMOVE_HEAD(&dev->reorder_queue, links); 1860 dev->num_reorder_queue--; 1861 STAILQ_INSERT_TAIL(&local_queue, buf1, work_links); 1862 dev->next_completion_pos_bytes += buf1->len; 1863 } 1864 } 1865 1866 /* 1867 * Setup the event to let the other thread know that it has work 1868 * pending. 1869 */ 1870 EV_SET(&ke, (uintptr_t)&dev->peer_dev->work_queue, EVFILT_USER, 0, 1871 NOTE_TRIGGER, 0, NULL); 1872 1873 /* 1874 * Put this on our shadow queue so that we know what we've queued 1875 * to the other thread. 1876 */ 1877 STAILQ_FOREACH_SAFE(buf1, &local_queue, work_links, buf2) { 1878 if (buf1->buf_type != CAMDD_BUF_DATA) { 1879 errx(1, "%s: should have a data buffer, not an " 1880 "indirect buffer", __func__); 1881 } 1882 data = &buf1->buf_type_spec.data; 1883 1884 /* 1885 * We only need to send one EOF to the writer, and don't 1886 * need to continue sending EOFs after that. 1887 */ 1888 if (buf1->status == CAMDD_STATUS_EOF) { 1889 if (dev->flags & CAMDD_DEV_FLAG_EOF_SENT) { 1890 STAILQ_REMOVE(&local_queue, buf1, camdd_buf, 1891 work_links); 1892 camdd_release_buf(buf1); 1893 retval = 1; 1894 continue; 1895 } 1896 dev->flags |= CAMDD_DEV_FLAG_EOF_SENT; 1897 } 1898 1899 1900 STAILQ_INSERT_TAIL(&dev->peer_work_queue, buf1, links); 1901 peer_bytes_queued += (data->fill_len - data->resid); 1902 dev->peer_bytes_queued += (data->fill_len - data->resid); 1903 dev->num_peer_work_queue++; 1904 } 1905 1906 if (STAILQ_FIRST(&local_queue) == NULL) 1907 goto bailout; 1908 1909 /* 1910 * Drop our mutex and pick up the other thread's mutex. We need to 1911 * do this to avoid deadlocks. 1912 */ 1913 pthread_mutex_unlock(&dev->mutex); 1914 pthread_mutex_lock(&dev->peer_dev->mutex); 1915 1916 if (dev->peer_dev->flags & CAMDD_DEV_FLAG_ACTIVE) { 1917 /* 1918 * Put the buffers on the other thread's incoming work queue. 1919 */ 1920 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL; 1921 buf1 = STAILQ_FIRST(&local_queue)) { 1922 STAILQ_REMOVE_HEAD(&local_queue, work_links); 1923 STAILQ_INSERT_TAIL(&dev->peer_dev->work_queue, buf1, 1924 work_links); 1925 } 1926 /* 1927 * Send an event to the other thread's kqueue to let it know 1928 * that there is something on the work queue. 1929 */ 1930 retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL); 1931 if (retval == -1) 1932 warn("%s: unable to add peer work_queue kevent", 1933 __func__); 1934 else 1935 retval = 0; 1936 } else 1937 active = 0; 1938 1939 pthread_mutex_unlock(&dev->peer_dev->mutex); 1940 pthread_mutex_lock(&dev->mutex); 1941 1942 /* 1943 * If the other side isn't active, run through the queue and 1944 * release all of the buffers. 1945 */ 1946 if (active == 0) { 1947 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL; 1948 buf1 = STAILQ_FIRST(&local_queue)) { 1949 STAILQ_REMOVE_HEAD(&local_queue, work_links); 1950 STAILQ_REMOVE(&dev->peer_work_queue, buf1, camdd_buf, 1951 links); 1952 dev->num_peer_work_queue--; 1953 camdd_release_buf(buf1); 1954 } 1955 dev->peer_bytes_queued -= peer_bytes_queued; 1956 retval = 1; 1957 } 1958 1959bailout: 1960 return (retval); 1961} 1962 1963/* 1964 * Return a buffer to the reader thread when we have completed writing it. 1965 */ 1966int 1967camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf) 1968{ 1969 struct kevent ke; 1970 int retval = 0; 1971 1972 /* 1973 * Setup the event to let the other thread know that we have 1974 * completed a buffer. 1975 */ 1976 EV_SET(&ke, (uintptr_t)&dev->peer_dev->peer_done_queue, EVFILT_USER, 0, 1977 NOTE_TRIGGER, 0, NULL); 1978 1979 /* 1980 * Drop our lock and acquire the other thread's lock before 1981 * manipulating 1982 */ 1983 pthread_mutex_unlock(&dev->mutex); 1984 pthread_mutex_lock(&dev->peer_dev->mutex); 1985 1986 /* 1987 * Put the buffer on the reader thread's peer done queue now that 1988 * we have completed it. 1989 */ 1990 STAILQ_INSERT_TAIL(&dev->peer_dev->peer_done_queue, peer_buf, 1991 work_links); 1992 dev->peer_dev->num_peer_done_queue++; 1993 1994 /* 1995 * Send an event to the peer thread to let it know that we've added 1996 * something to its peer done queue. 1997 */ 1998 retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL); 1999 if (retval == -1) 2000 warn("%s: unable to add peer_done_queue kevent", __func__); 2001 else 2002 retval = 0; 2003 2004 /* 2005 * Drop the other thread's lock and reacquire ours. 2006 */ 2007 pthread_mutex_unlock(&dev->peer_dev->mutex); 2008 pthread_mutex_lock(&dev->mutex); 2009 2010 return (retval); 2011} 2012 2013/* 2014 * Free a buffer that was written out by the writer thread and returned to 2015 * the reader thread. 2016 */ 2017void 2018camdd_peer_done(struct camdd_buf *buf) 2019{ 2020 struct camdd_dev *dev; 2021 struct camdd_buf_data *data; 2022 2023 dev = buf->dev; 2024 if (buf->buf_type != CAMDD_BUF_DATA) { 2025 errx(1, "%s: should have a data buffer, not an " 2026 "indirect buffer", __func__); 2027 } 2028 2029 data = &buf->buf_type_spec.data; 2030 2031 STAILQ_REMOVE(&dev->peer_work_queue, buf, camdd_buf, links); 2032 dev->num_peer_work_queue--; 2033 dev->peer_bytes_queued -= (data->fill_len - data->resid); 2034 2035 if (buf->status == CAMDD_STATUS_EOF) 2036 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF; 2037 2038 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 2039} 2040 2041/* 2042 * Assumes caller holds the lock for this device. 2043 */ 2044void 2045camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf, 2046 int *error_count) 2047{ 2048 int retval = 0; 2049 2050 /* 2051 * If we're the reader, we need to send the completed I/O 2052 * to the writer. If we're the writer, we need to just 2053 * free up resources, or let the reader know if we've 2054 * encountered an error. 2055 */ 2056 if (dev->write_dev == 0) { 2057 retval = camdd_queue_peer_buf(dev, buf); 2058 if (retval != 0) 2059 (*error_count)++; 2060 } else { 2061 struct camdd_buf *tmp_buf, *next_buf; 2062 2063 STAILQ_FOREACH_SAFE(tmp_buf, &buf->src_list, src_links, 2064 next_buf) { 2065 struct camdd_buf *src_buf; 2066 struct camdd_buf_indirect *indirect; 2067 2068 STAILQ_REMOVE(&buf->src_list, tmp_buf, 2069 camdd_buf, src_links); 2070 2071 tmp_buf->status = buf->status; 2072 2073 if (tmp_buf->buf_type == CAMDD_BUF_DATA) { 2074 camdd_complete_peer_buf(dev, tmp_buf); 2075 continue; 2076 } 2077 2078 indirect = &tmp_buf->buf_type_spec.indirect; 2079 src_buf = indirect->src_buf; 2080 src_buf->refcount--; 2081 /* 2082 * XXX KDM we probably need to account for 2083 * exactly how many bytes we were able to 2084 * write. Allocate the residual to the 2085 * first N buffers? Or just track the 2086 * number of bytes written? Right now the reader 2087 * doesn't do anything with a residual. 2088 */ 2089 src_buf->status = buf->status; 2090 if (src_buf->refcount <= 0) 2091 camdd_complete_peer_buf(dev, src_buf); 2092 STAILQ_INSERT_TAIL(&dev->free_indirect_queue, 2093 tmp_buf, links); 2094 } 2095 2096 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 2097 } 2098} 2099 2100/* 2101 * Fetch all completed commands from the pass(4) device. 2102 * 2103 * Returns the number of commands received, or -1 if any of the commands 2104 * completed with an error. Returns 0 if no commands are available. 2105 */ 2106int 2107camdd_pass_fetch(struct camdd_dev *dev) 2108{ 2109 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass; 2110 union ccb ccb; 2111 int retval = 0, num_fetched = 0, error_count = 0; 2112 2113 pthread_mutex_unlock(&dev->mutex); 2114 /* 2115 * XXX KDM we don't distinguish between EFAULT and ENOENT. 2116 */ 2117 while ((retval = ioctl(pass_dev->dev->fd, CAMIOGET, &ccb)) != -1) { 2118 struct camdd_buf *buf; 2119 struct camdd_buf_data *data; 2120 cam_status ccb_status; 2121 union ccb *buf_ccb; 2122 2123 buf = ccb.ccb_h.ccb_buf; 2124 data = &buf->buf_type_spec.data; 2125 buf_ccb = &data->ccb; 2126 2127 num_fetched++; 2128 2129 /* 2130 * Copy the CCB back out so we get status, sense data, etc. 2131 */ 2132 bcopy(&ccb, buf_ccb, sizeof(ccb)); 2133 2134 pthread_mutex_lock(&dev->mutex); 2135 2136 /* 2137 * We're now done, so take this off the active queue. 2138 */ 2139 STAILQ_REMOVE(&dev->active_queue, buf, camdd_buf, links); 2140 dev->cur_active_io--; 2141 2142 ccb_status = ccb.ccb_h.status & CAM_STATUS_MASK; 2143 if (ccb_status != CAM_REQ_CMP) { 2144 cam_error_print(pass_dev->dev, &ccb, CAM_ESF_ALL, 2145 CAM_EPF_ALL, stderr); 2146 } 2147 2148 data->resid = ccb.csio.resid; 2149 dev->bytes_transferred += (ccb.csio.dxfer_len - ccb.csio.resid); 2150 2151 if (buf->status == CAMDD_STATUS_NONE) 2152 buf->status = camdd_ccb_status(&ccb); 2153 if (buf->status == CAMDD_STATUS_ERROR) 2154 error_count++; 2155 else if (buf->status == CAMDD_STATUS_EOF) { 2156 /* 2157 * Once we queue this buffer to our partner thread, 2158 * he will know that we've hit EOF. 2159 */ 2160 dev->flags |= CAMDD_DEV_FLAG_EOF; 2161 } 2162 2163 camdd_complete_buf(dev, buf, &error_count); 2164 2165 /* 2166 * Unlock in preparation for the ioctl call. 2167 */ 2168 pthread_mutex_unlock(&dev->mutex); 2169 } 2170 2171 pthread_mutex_lock(&dev->mutex); 2172 2173 if (error_count > 0) 2174 return (-1); 2175 else 2176 return (num_fetched); 2177} 2178 2179/* 2180 * Returns -1 for error, 0 for success/continue, and 1 for resource 2181 * shortage/stop processing. 2182 */ 2183int 2184camdd_file_run(struct camdd_dev *dev) 2185{ 2186 struct camdd_dev_file *file_dev = &dev->dev_spec.file; 2187 struct camdd_buf_data *data; 2188 struct camdd_buf *buf; 2189 off_t io_offset; 2190 int retval = 0, write_dev = dev->write_dev; 2191 int error_count = 0, no_resources = 0, double_buf_needed = 0; 2192 uint32_t num_sectors = 0, db_len = 0; 2193 2194 buf = STAILQ_FIRST(&dev->run_queue); 2195 if (buf == NULL) { 2196 no_resources = 1; 2197 goto bailout; 2198 } else if ((dev->write_dev == 0) 2199 && (dev->flags & (CAMDD_DEV_FLAG_EOF | 2200 CAMDD_DEV_FLAG_EOF_SENT))) { 2201 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links); 2202 dev->num_run_queue--; 2203 buf->status = CAMDD_STATUS_EOF; 2204 error_count++; 2205 goto bailout; 2206 } 2207 2208 /* 2209 * If we're writing, we need to go through the source buffer list 2210 * and create an S/G list. 2211 */ 2212 if (write_dev != 0) { 2213 retval = camdd_buf_sg_create(buf, /*iovec*/ 1, 2214 dev->sector_size, &num_sectors, &double_buf_needed); 2215 if (retval != 0) { 2216 no_resources = 1; 2217 goto bailout; 2218 } 2219 } 2220 2221 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links); 2222 dev->num_run_queue--; 2223 2224 data = &buf->buf_type_spec.data; 2225 2226 /* 2227 * pread(2) and pwrite(2) offsets are byte offsets. 2228 */ 2229 io_offset = buf->lba * dev->sector_size; 2230 2231 /* 2232 * Unlock the mutex while we read or write. 2233 */ 2234 pthread_mutex_unlock(&dev->mutex); 2235 2236 /* 2237 * Note that we don't need to double buffer if we're the reader 2238 * because in that case, we have allocated a single buffer of 2239 * sufficient size to do the read. This copy is necessary on 2240 * writes because if one of the components of the S/G list is not 2241 * a sector size multiple, the kernel will reject the write. This 2242 * is unfortunate but not surprising. So this will make sure that 2243 * we're using a single buffer that is a multiple of the sector size. 2244 */ 2245 if ((double_buf_needed != 0) 2246 && (data->sg_count > 1) 2247 && (write_dev != 0)) { 2248 uint32_t cur_offset; 2249 int i; 2250 2251 if (file_dev->tmp_buf == NULL) 2252 file_dev->tmp_buf = calloc(dev->blocksize, 1); 2253 if (file_dev->tmp_buf == NULL) { 2254 buf->status = CAMDD_STATUS_ERROR; 2255 error_count++; 2256 pthread_mutex_lock(&dev->mutex); 2257 goto bailout; 2258 } 2259 for (i = 0, cur_offset = 0; i < data->sg_count; i++) { 2260 bcopy(data->iovec[i].iov_base, 2261 &file_dev->tmp_buf[cur_offset], 2262 data->iovec[i].iov_len); 2263 cur_offset += data->iovec[i].iov_len; 2264 } 2265 db_len = cur_offset; 2266 } 2267 2268 if (file_dev->file_flags & CAMDD_FF_CAN_SEEK) { 2269 if (write_dev == 0) { 2270 /* 2271 * XXX KDM is there any way we would need a S/G 2272 * list here? 2273 */ 2274 retval = pread(file_dev->fd, data->buf, 2275 buf->len, io_offset); 2276 } else { 2277 if (double_buf_needed != 0) { 2278 retval = pwrite(file_dev->fd, file_dev->tmp_buf, 2279 db_len, io_offset); 2280 } else if (data->sg_count == 0) { 2281 retval = pwrite(file_dev->fd, data->buf, 2282 data->fill_len, io_offset); 2283 } else { 2284 retval = pwritev(file_dev->fd, data->iovec, 2285 data->sg_count, io_offset); 2286 } 2287 } 2288 } else { 2289 if (write_dev == 0) { 2290 /* 2291 * XXX KDM is there any way we would need a S/G 2292 * list here? 2293 */ 2294 retval = read(file_dev->fd, data->buf, buf->len); 2295 } else { 2296 if (double_buf_needed != 0) { 2297 retval = write(file_dev->fd, file_dev->tmp_buf, 2298 db_len); 2299 } else if (data->sg_count == 0) { 2300 retval = write(file_dev->fd, data->buf, 2301 data->fill_len); 2302 } else { 2303 retval = writev(file_dev->fd, data->iovec, 2304 data->sg_count); 2305 } 2306 } 2307 } 2308 2309 /* We're done, re-acquire the lock */ 2310 pthread_mutex_lock(&dev->mutex); 2311 2312 if (retval >= (ssize_t)data->fill_len) { 2313 /* 2314 * If the bytes transferred is more than the request size, 2315 * that indicates an overrun, which should only happen at 2316 * the end of a transfer if we have to round up to a sector 2317 * boundary. 2318 */ 2319 if (buf->status == CAMDD_STATUS_NONE) 2320 buf->status = CAMDD_STATUS_OK; 2321 data->resid = 0; 2322 dev->bytes_transferred += retval; 2323 } else if (retval == -1) { 2324 warn("Error %s %s", (write_dev) ? "writing to" : 2325 "reading from", file_dev->filename); 2326 2327 buf->status = CAMDD_STATUS_ERROR; 2328 data->resid = data->fill_len; 2329 error_count++; 2330 2331 if (dev->debug == 0) 2332 goto bailout; 2333 2334 if ((double_buf_needed != 0) 2335 && (write_dev != 0)) { 2336 fprintf(stderr, "%s: fd %d, DB buf %p, len %u lba %ju " 2337 "offset %ju\n", __func__, file_dev->fd, 2338 file_dev->tmp_buf, db_len, (uintmax_t)buf->lba, 2339 (uintmax_t)io_offset); 2340 } else if (data->sg_count == 0) { 2341 fprintf(stderr, "%s: fd %d, buf %p, len %u, lba %ju " 2342 "offset %ju\n", __func__, file_dev->fd, data->buf, 2343 data->fill_len, (uintmax_t)buf->lba, 2344 (uintmax_t)io_offset); 2345 } else { 2346 int i; 2347 2348 fprintf(stderr, "%s: fd %d, len %u, lba %ju " 2349 "offset %ju\n", __func__, file_dev->fd, 2350 data->fill_len, (uintmax_t)buf->lba, 2351 (uintmax_t)io_offset); 2352 2353 for (i = 0; i < data->sg_count; i++) { 2354 fprintf(stderr, "index %d ptr %p len %zu\n", 2355 i, data->iovec[i].iov_base, 2356 data->iovec[i].iov_len); 2357 } 2358 } 2359 } else if (retval == 0) { 2360 buf->status = CAMDD_STATUS_EOF; 2361 if (dev->debug != 0) 2362 printf("%s: got EOF from %s!\n", __func__, 2363 file_dev->filename); 2364 data->resid = data->fill_len; 2365 error_count++; 2366 } else if (retval < (ssize_t)data->fill_len) { 2367 if (buf->status == CAMDD_STATUS_NONE) 2368 buf->status = CAMDD_STATUS_SHORT_IO; 2369 data->resid = data->fill_len - retval; 2370 dev->bytes_transferred += retval; 2371 } 2372 2373bailout: 2374 if (buf != NULL) { 2375 if (buf->status == CAMDD_STATUS_EOF) { 2376 struct camdd_buf *buf2; 2377 dev->flags |= CAMDD_DEV_FLAG_EOF; 2378 STAILQ_FOREACH(buf2, &dev->run_queue, links) 2379 buf2->status = CAMDD_STATUS_EOF; 2380 } 2381 2382 camdd_complete_buf(dev, buf, &error_count); 2383 } 2384 2385 if (error_count != 0) 2386 return (-1); 2387 else if (no_resources != 0) 2388 return (1); 2389 else 2390 return (0); 2391} 2392 2393/* 2394 * Execute one command from the run queue. Returns 0 for success, 1 for 2395 * stop processing, and -1 for error. 2396 */ 2397int 2398camdd_pass_run(struct camdd_dev *dev) 2399{ 2400 struct camdd_buf *buf = NULL; 2401 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass; 2402 struct camdd_buf_data *data; 2403 uint32_t num_blocks, sectors_used = 0; 2404 union ccb *ccb; 2405 int retval = 0, is_write = dev->write_dev; 2406 int double_buf_needed = 0; 2407 2408 buf = STAILQ_FIRST(&dev->run_queue); 2409 if (buf == NULL) { 2410 retval = 1; 2411 goto bailout; 2412 } 2413 2414 /* 2415 * If we're writing, we need to go through the source buffer list 2416 * and create an S/G list. 2417 */ 2418 if (is_write != 0) { 2419 retval = camdd_buf_sg_create(buf, /*iovec*/ 0,dev->sector_size, 2420 §ors_used, &double_buf_needed); 2421 if (retval != 0) { 2422 retval = -1; 2423 goto bailout; 2424 } 2425 } 2426 2427 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links); 2428 dev->num_run_queue--; 2429 2430 data = &buf->buf_type_spec.data; 2431 2432 ccb = &data->ccb; 2433 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->csio); 2434 2435 /* 2436 * In almost every case the number of blocks should be the device 2437 * block size. The exception may be at the end of an I/O stream 2438 * for a partial block or at the end of a device. 2439 */ 2440 if (is_write != 0) 2441 num_blocks = sectors_used; 2442 else 2443 num_blocks = data->fill_len / pass_dev->block_len; 2444 2445 scsi_read_write(&ccb->csio, 2446 /*retries*/ dev->retry_count, 2447 /*cbfcnp*/ NULL, 2448 /*tag_action*/ MSG_SIMPLE_Q_TAG, 2449 /*readop*/ (dev->write_dev == 0) ? SCSI_RW_READ : 2450 SCSI_RW_WRITE, 2451 /*byte2*/ 0, 2452 /*minimum_cmd_size*/ dev->min_cmd_size, 2453 /*lba*/ buf->lba, 2454 /*block_count*/ num_blocks, 2455 /*data_ptr*/ (data->sg_count != 0) ? 2456 (uint8_t *)data->segs : data->buf, 2457 /*dxfer_len*/ (num_blocks * pass_dev->block_len), 2458 /*sense_len*/ SSD_FULL_SIZE, 2459 /*timeout*/ dev->io_timeout); 2460 2461 /* Disable freezing the device queue */ 2462 ccb->ccb_h.flags |= CAM_DEV_QFRZDIS; 2463 2464 if (dev->retry_count != 0) 2465 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER; 2466 2467 if (data->sg_count != 0) { 2468 ccb->csio.sglist_cnt = data->sg_count; 2469 ccb->ccb_h.flags |= CAM_DATA_SG; 2470 } 2471 2472 /* 2473 * Store a pointer to the buffer in the CCB. The kernel will 2474 * restore this when we get it back, and we'll use it to identify 2475 * the buffer this CCB came from. 2476 */ 2477 ccb->ccb_h.ccb_buf = buf; 2478 2479 /* 2480 * Unlock our mutex in preparation for issuing the ioctl. 2481 */ 2482 pthread_mutex_unlock(&dev->mutex); 2483 /* 2484 * Queue the CCB to the pass(4) driver. 2485 */ 2486 if (ioctl(pass_dev->dev->fd, CAMIOQUEUE, ccb) == -1) { 2487 pthread_mutex_lock(&dev->mutex); 2488 2489 warn("%s: error sending CAMIOQUEUE ioctl to %s%u", __func__, 2490 pass_dev->dev->device_name, pass_dev->dev->dev_unit_num); 2491 warn("%s: CCB address is %p", __func__, ccb); 2492 retval = -1; 2493 2494 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links); 2495 } else { 2496 pthread_mutex_lock(&dev->mutex); 2497 2498 dev->cur_active_io++; 2499 STAILQ_INSERT_TAIL(&dev->active_queue, buf, links); 2500 } 2501 2502bailout: 2503 return (retval); 2504} 2505 2506int 2507camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len) 2508{ 2509 struct camdd_dev_pass *pass_dev; 2510 uint32_t num_blocks; 2511 int retval = 0; 2512 2513 pass_dev = &dev->dev_spec.pass; 2514 2515 *lba = dev->next_io_pos_bytes / dev->sector_size; 2516 *len = dev->blocksize; 2517 num_blocks = *len / dev->sector_size; 2518 2519 /* 2520 * If max_sector is 0, then we have no set limit. This can happen 2521 * if we're writing to a file in a filesystem, or reading from 2522 * something like /dev/zero. 2523 */ 2524 if ((dev->max_sector != 0) 2525 || (dev->sector_io_limit != 0)) { 2526 uint64_t max_sector; 2527 2528 if ((dev->max_sector != 0) 2529 && (dev->sector_io_limit != 0)) 2530 max_sector = min(dev->sector_io_limit, dev->max_sector); 2531 else if (dev->max_sector != 0) 2532 max_sector = dev->max_sector; 2533 else 2534 max_sector = dev->sector_io_limit; 2535 2536 2537 /* 2538 * Check to see whether we're starting off past the end of 2539 * the device. If so, we need to just send an EOF 2540 * notification to the writer. 2541 */ 2542 if (*lba > max_sector) { 2543 *len = 0; 2544 retval = 1; 2545 } else if (((*lba + num_blocks) > max_sector + 1) 2546 || ((*lba + num_blocks) < *lba)) { 2547 /* 2548 * If we get here (but pass the first check), we 2549 * can trim the request length down to go to the 2550 * end of the device. 2551 */ 2552 num_blocks = (max_sector + 1) - *lba; 2553 *len = num_blocks * dev->sector_size; 2554 retval = 1; 2555 } 2556 } 2557 2558 dev->next_io_pos_bytes += *len; 2559 2560 return (retval); 2561} 2562 2563/* 2564 * Returns 0 for success, 1 for EOF detected, and -1 for failure. 2565 */ 2566int 2567camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf) 2568{ 2569 struct camdd_buf *buf = NULL; 2570 struct camdd_buf_data *data; 2571 struct camdd_dev_pass *pass_dev; 2572 size_t new_len; 2573 struct camdd_buf_data *rb_data; 2574 int is_write = dev->write_dev; 2575 int eof_flush_needed = 0; 2576 int retval = 0; 2577 int error; 2578 2579 pass_dev = &dev->dev_spec.pass; 2580 2581 /* 2582 * If we've gotten EOF or our partner has, we should not continue 2583 * queueing I/O. If we're a writer, though, we should continue 2584 * to write any buffers that don't have EOF status. 2585 */ 2586 if ((dev->flags & CAMDD_DEV_FLAG_EOF) 2587 || ((dev->flags & CAMDD_DEV_FLAG_PEER_EOF) 2588 && (is_write == 0))) { 2589 /* 2590 * Tell the worker thread that we have seen EOF. 2591 */ 2592 retval = 1; 2593 2594 /* 2595 * If we're the writer, send the buffer back with EOF status. 2596 */ 2597 if (is_write) { 2598 read_buf->status = CAMDD_STATUS_EOF; 2599 2600 error = camdd_complete_peer_buf(dev, read_buf); 2601 } 2602 goto bailout; 2603 } 2604 2605 if (is_write == 0) { 2606 buf = camdd_get_buf(dev, CAMDD_BUF_DATA); 2607 if (buf == NULL) { 2608 retval = -1; 2609 goto bailout; 2610 } 2611 data = &buf->buf_type_spec.data; 2612 2613 retval = camdd_get_next_lba_len(dev, &buf->lba, &buf->len); 2614 if (retval != 0) { 2615 buf->status = CAMDD_STATUS_EOF; 2616 2617 if ((buf->len == 0) 2618 && ((dev->flags & (CAMDD_DEV_FLAG_EOF_SENT | 2619 CAMDD_DEV_FLAG_EOF_QUEUED)) != 0)) { 2620 camdd_release_buf(buf); 2621 goto bailout; 2622 } 2623 dev->flags |= CAMDD_DEV_FLAG_EOF_QUEUED; 2624 } 2625 2626 data->fill_len = buf->len; 2627 data->src_start_offset = buf->lba * dev->sector_size; 2628 2629 /* 2630 * Put this on the run queue. 2631 */ 2632 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2633 dev->num_run_queue++; 2634 2635 /* We're done. */ 2636 goto bailout; 2637 } 2638 2639 /* 2640 * Check for new EOF status from the reader. 2641 */ 2642 if ((read_buf->status == CAMDD_STATUS_EOF) 2643 || (read_buf->status == CAMDD_STATUS_ERROR)) { 2644 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF; 2645 if ((STAILQ_FIRST(&dev->pending_queue) == NULL) 2646 && (read_buf->len == 0)) { 2647 camdd_complete_peer_buf(dev, read_buf); 2648 retval = 1; 2649 goto bailout; 2650 } else 2651 eof_flush_needed = 1; 2652 } 2653 2654 /* 2655 * See if we have a buffer we're composing with pieces from our 2656 * partner thread. 2657 */ 2658 buf = STAILQ_FIRST(&dev->pending_queue); 2659 if (buf == NULL) { 2660 uint64_t lba; 2661 ssize_t len; 2662 2663 retval = camdd_get_next_lba_len(dev, &lba, &len); 2664 if (retval != 0) { 2665 read_buf->status = CAMDD_STATUS_EOF; 2666 2667 if (len == 0) { 2668 dev->flags |= CAMDD_DEV_FLAG_EOF; 2669 error = camdd_complete_peer_buf(dev, read_buf); 2670 goto bailout; 2671 } 2672 } 2673 2674 /* 2675 * If we don't have a pending buffer, we need to grab a new 2676 * one from the free list or allocate another one. 2677 */ 2678 buf = camdd_get_buf(dev, CAMDD_BUF_DATA); 2679 if (buf == NULL) { 2680 retval = 1; 2681 goto bailout; 2682 } 2683 2684 buf->lba = lba; 2685 buf->len = len; 2686 2687 STAILQ_INSERT_TAIL(&dev->pending_queue, buf, links); 2688 dev->num_pending_queue++; 2689 } 2690 2691 data = &buf->buf_type_spec.data; 2692 2693 rb_data = &read_buf->buf_type_spec.data; 2694 2695 if ((rb_data->src_start_offset != dev->next_peer_pos_bytes) 2696 && (dev->debug != 0)) { 2697 printf("%s: WARNING: reader offset %#jx != expected offset " 2698 "%#jx\n", __func__, (uintmax_t)rb_data->src_start_offset, 2699 (uintmax_t)dev->next_peer_pos_bytes); 2700 } 2701 dev->next_peer_pos_bytes = rb_data->src_start_offset + 2702 (rb_data->fill_len - rb_data->resid); 2703 2704 new_len = (rb_data->fill_len - rb_data->resid) + data->fill_len; 2705 if (new_len < buf->len) { 2706 /* 2707 * There are three cases here: 2708 * 1. We need more data to fill up a block, so we put 2709 * this I/O on the queue and wait for more I/O. 2710 * 2. We have a pending buffer in the queue that is 2711 * smaller than our blocksize, but we got an EOF. So we 2712 * need to go ahead and flush the write out. 2713 * 3. We got an error. 2714 */ 2715 2716 /* 2717 * Increment our fill length. 2718 */ 2719 data->fill_len += (rb_data->fill_len - rb_data->resid); 2720 2721 /* 2722 * Add the new read buffer to the list for writing. 2723 */ 2724 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links); 2725 2726 /* Increment the count */ 2727 buf->src_count++; 2728 2729 if (eof_flush_needed == 0) { 2730 /* 2731 * We need to exit, because we don't have enough 2732 * data yet. 2733 */ 2734 goto bailout; 2735 } else { 2736 /* 2737 * Take the buffer off of the pending queue. 2738 */ 2739 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, 2740 links); 2741 dev->num_pending_queue--; 2742 2743 /* 2744 * If we need an EOF flush, but there is no data 2745 * to flush, go ahead and return this buffer. 2746 */ 2747 if (data->fill_len == 0) { 2748 camdd_complete_buf(dev, buf, /*error_count*/0); 2749 retval = 1; 2750 goto bailout; 2751 } 2752 2753 /* 2754 * Put this on the next queue for execution. 2755 */ 2756 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2757 dev->num_run_queue++; 2758 } 2759 } else if (new_len == buf->len) { 2760 /* 2761 * We have enough data to completey fill one block, 2762 * so we're ready to issue the I/O. 2763 */ 2764 2765 /* 2766 * Take the buffer off of the pending queue. 2767 */ 2768 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, links); 2769 dev->num_pending_queue--; 2770 2771 /* 2772 * Add the new read buffer to the list for writing. 2773 */ 2774 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links); 2775 2776 /* Increment the count */ 2777 buf->src_count++; 2778 2779 /* 2780 * Increment our fill length. 2781 */ 2782 data->fill_len += (rb_data->fill_len - rb_data->resid); 2783 2784 /* 2785 * Put this on the next queue for execution. 2786 */ 2787 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2788 dev->num_run_queue++; 2789 } else { 2790 struct camdd_buf *idb; 2791 struct camdd_buf_indirect *indirect; 2792 uint32_t len_to_go, cur_offset; 2793 2794 2795 idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT); 2796 if (idb == NULL) { 2797 retval = 1; 2798 goto bailout; 2799 } 2800 indirect = &idb->buf_type_spec.indirect; 2801 indirect->src_buf = read_buf; 2802 read_buf->refcount++; 2803 indirect->offset = 0; 2804 indirect->start_ptr = rb_data->buf; 2805 /* 2806 * We've already established that there is more 2807 * data in read_buf than we have room for in our 2808 * current write request. So this particular chunk 2809 * of the request should just be the remainder 2810 * needed to fill up a block. 2811 */ 2812 indirect->len = buf->len - (data->fill_len - data->resid); 2813 2814 camdd_buf_add_child(buf, idb); 2815 2816 /* 2817 * This buffer is ready to execute, so we can take 2818 * it off the pending queue and put it on the run 2819 * queue. 2820 */ 2821 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, 2822 links); 2823 dev->num_pending_queue--; 2824 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links); 2825 dev->num_run_queue++; 2826 2827 cur_offset = indirect->offset + indirect->len; 2828 2829 /* 2830 * The resulting I/O would be too large to fit in 2831 * one block. We need to split this I/O into 2832 * multiple pieces. Allocate as many buffers as needed. 2833 */ 2834 for (len_to_go = rb_data->fill_len - rb_data->resid - 2835 indirect->len; len_to_go > 0;) { 2836 struct camdd_buf *new_buf; 2837 struct camdd_buf_data *new_data; 2838 uint64_t lba; 2839 ssize_t len; 2840 2841 retval = camdd_get_next_lba_len(dev, &lba, &len); 2842 if ((retval != 0) 2843 && (len == 0)) { 2844 /* 2845 * The device has already been marked 2846 * as EOF, and there is no space left. 2847 */ 2848 goto bailout; 2849 } 2850 2851 new_buf = camdd_get_buf(dev, CAMDD_BUF_DATA); 2852 if (new_buf == NULL) { 2853 retval = 1; 2854 goto bailout; 2855 } 2856 2857 new_buf->lba = lba; 2858 new_buf->len = len; 2859 2860 idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT); 2861 if (idb == NULL) { 2862 retval = 1; 2863 goto bailout; 2864 } 2865 2866 indirect = &idb->buf_type_spec.indirect; 2867 2868 indirect->src_buf = read_buf; 2869 read_buf->refcount++; 2870 indirect->offset = cur_offset; 2871 indirect->start_ptr = rb_data->buf + cur_offset; 2872 indirect->len = min(len_to_go, new_buf->len); 2873#if 0 2874 if (((indirect->len % dev->sector_size) != 0) 2875 || ((indirect->offset % dev->sector_size) != 0)) { 2876 warnx("offset %ju len %ju not aligned with " 2877 "sector size %u", indirect->offset, 2878 (uintmax_t)indirect->len, dev->sector_size); 2879 } 2880#endif 2881 cur_offset += indirect->len; 2882 len_to_go -= indirect->len; 2883 2884 camdd_buf_add_child(new_buf, idb); 2885 2886 new_data = &new_buf->buf_type_spec.data; 2887 2888 if ((new_data->fill_len == new_buf->len) 2889 || (eof_flush_needed != 0)) { 2890 STAILQ_INSERT_TAIL(&dev->run_queue, 2891 new_buf, links); 2892 dev->num_run_queue++; 2893 } else if (new_data->fill_len < buf->len) { 2894 STAILQ_INSERT_TAIL(&dev->pending_queue, 2895 new_buf, links); 2896 dev->num_pending_queue++; 2897 } else { 2898 warnx("%s: too much data in new " 2899 "buffer!", __func__); 2900 retval = 1; 2901 goto bailout; 2902 } 2903 } 2904 } 2905 2906bailout: 2907 return (retval); 2908} 2909 2910void 2911camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth, 2912 uint32_t *peer_depth, uint32_t *our_bytes, uint32_t *peer_bytes) 2913{ 2914 *our_depth = dev->cur_active_io + dev->num_run_queue; 2915 if (dev->num_peer_work_queue > 2916 dev->num_peer_done_queue) 2917 *peer_depth = dev->num_peer_work_queue - 2918 dev->num_peer_done_queue; 2919 else 2920 *peer_depth = 0; 2921 *our_bytes = *our_depth * dev->blocksize; 2922 *peer_bytes = dev->peer_bytes_queued; 2923} 2924 2925void 2926camdd_sig_handler(int sig) 2927{ 2928 if (sig == SIGINFO) 2929 need_status = 1; 2930 else { 2931 need_exit = 1; 2932 error_exit = 1; 2933 } 2934 2935 sem_post(&camdd_sem); 2936} 2937 2938void 2939camdd_print_status(struct camdd_dev *camdd_dev, struct camdd_dev *other_dev, 2940 struct timespec *start_time) 2941{ 2942 struct timespec done_time; 2943 uint64_t total_ns; 2944 long double mb_sec, total_sec; 2945 int error = 0; 2946 2947 error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &done_time); 2948 if (error != 0) { 2949 warn("Unable to get done time"); 2950 return; 2951 } 2952 2953 timespecsub(&done_time, start_time); 2954 2955 total_ns = done_time.tv_nsec + (done_time.tv_sec * 1000000000); 2956 total_sec = total_ns; 2957 total_sec /= 1000000000; 2958 2959 fprintf(stderr, "%ju bytes %s %s\n%ju bytes %s %s\n" 2960 "%.4Lf seconds elapsed\n", 2961 (uintmax_t)camdd_dev->bytes_transferred, 2962 (camdd_dev->write_dev == 0) ? "read from" : "written to", 2963 camdd_dev->device_name, 2964 (uintmax_t)other_dev->bytes_transferred, 2965 (other_dev->write_dev == 0) ? "read from" : "written to", 2966 other_dev->device_name, total_sec); 2967 2968 mb_sec = min(other_dev->bytes_transferred,camdd_dev->bytes_transferred); 2969 mb_sec /= 1024 * 1024; 2970 mb_sec *= 1000000000; 2971 mb_sec /= total_ns; 2972 fprintf(stderr, "%.2Lf MB/sec\n", mb_sec); 2973} 2974 2975int 2976camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts, uint64_t max_io, 2977 int retry_count, int timeout) 2978{ 2979 struct cam_device *new_cam_dev = NULL; 2980 struct camdd_dev *devs[2]; 2981 struct timespec start_time; 2982 pthread_t threads[2]; 2983 int unit = 0; 2984 int error = 0; 2985 int i; 2986 2987 if (num_io_opts != 2) { 2988 warnx("Must have one input and one output path"); 2989 error = 1; 2990 goto bailout; 2991 } 2992 2993 bzero(devs, sizeof(devs)); 2994 2995 for (i = 0; i < num_io_opts; i++) { 2996 switch (io_opts[i].dev_type) { 2997 case CAMDD_DEV_PASS: { 2998 if (isdigit(io_opts[i].dev_name[0])) { 2999 camdd_argmask new_arglist = CAMDD_ARG_NONE; 3000 int bus = 0, target = 0, lun = 0; 3001 int rv; 3002 3003 /* device specified as bus:target[:lun] */ 3004 rv = parse_btl(io_opts[i].dev_name, &bus, 3005 &target, &lun, &new_arglist); 3006 if (rv < 2) { 3007 warnx("numeric device specification " 3008 "must be either bus:target, or " 3009 "bus:target:lun"); 3010 error = 1; 3011 goto bailout; 3012 } 3013 /* default to 0 if lun was not specified */ 3014 if ((new_arglist & CAMDD_ARG_LUN) == 0) { 3015 lun = 0; 3016 new_arglist |= CAMDD_ARG_LUN; 3017 } 3018 new_cam_dev = cam_open_btl(bus, target, lun, 3019 O_RDWR, NULL); 3020 } else { 3021 char name[30]; 3022 3023 if (cam_get_device(io_opts[i].dev_name, name, 3024 sizeof name, &unit) == -1) { 3025 warnx("%s", cam_errbuf); 3026 error = 1; 3027 goto bailout; 3028 } 3029 new_cam_dev = cam_open_spec_device(name, unit, 3030 O_RDWR, NULL); 3031 } 3032 3033 if (new_cam_dev == NULL) { 3034 warnx("%s", cam_errbuf); 3035 error = 1; 3036 goto bailout; 3037 } 3038 3039 devs[i] = camdd_probe_pass(new_cam_dev, 3040 /*io_opts*/ &io_opts[i], 3041 CAMDD_ARG_ERR_RECOVER, 3042 /*probe_retry_count*/ 3, 3043 /*probe_timeout*/ 5000, 3044 /*io_retry_count*/ retry_count, 3045 /*io_timeout*/ timeout); 3046 if (devs[i] == NULL) { 3047 warn("Unable to probe device %s%u", 3048 new_cam_dev->device_name, 3049 new_cam_dev->dev_unit_num); 3050 error = 1; 3051 goto bailout; 3052 } 3053 break; 3054 } 3055 case CAMDD_DEV_FILE: { 3056 int fd = -1; 3057 3058 if (io_opts[i].dev_name[0] == '-') { 3059 if (io_opts[i].write_dev != 0) 3060 fd = STDOUT_FILENO; 3061 else 3062 fd = STDIN_FILENO; 3063 } else { 3064 if (io_opts[i].write_dev != 0) { 3065 fd = open(io_opts[i].dev_name, 3066 O_RDWR | O_CREAT, S_IWUSR |S_IRUSR); 3067 } else { 3068 fd = open(io_opts[i].dev_name, 3069 O_RDONLY); 3070 } 3071 } 3072 if (fd == -1) { 3073 warn("error opening file %s", 3074 io_opts[i].dev_name); 3075 error = 1; 3076 goto bailout; 3077 } 3078 3079 devs[i] = camdd_probe_file(fd, &io_opts[i], 3080 retry_count, timeout); 3081 if (devs[i] == NULL) { 3082 error = 1; 3083 goto bailout; 3084 } 3085 3086 break; 3087 } 3088 default: 3089 warnx("Unknown device type %d (%s)", 3090 io_opts[i].dev_type, io_opts[i].dev_name); 3091 error = 1; 3092 goto bailout; 3093 break; /*NOTREACHED */ 3094 } 3095 3096 devs[i]->write_dev = io_opts[i].write_dev; 3097 3098 devs[i]->start_offset_bytes = io_opts[i].offset; 3099 3100 if (max_io != 0) { 3101 devs[i]->sector_io_limit = 3102 (devs[i]->start_offset_bytes / 3103 devs[i]->sector_size) + 3104 (max_io / devs[i]->sector_size) - 1; 3105 devs[i]->sector_io_limit = 3106 (devs[i]->start_offset_bytes / 3107 devs[i]->sector_size) + 3108 (max_io / devs[i]->sector_size) - 1; 3109 } 3110 3111 devs[i]->next_io_pos_bytes = devs[i]->start_offset_bytes; 3112 devs[i]->next_completion_pos_bytes =devs[i]->start_offset_bytes; 3113 } 3114 3115 devs[0]->peer_dev = devs[1]; 3116 devs[1]->peer_dev = devs[0]; 3117 devs[0]->next_peer_pos_bytes = devs[0]->peer_dev->next_io_pos_bytes; 3118 devs[1]->next_peer_pos_bytes = devs[1]->peer_dev->next_io_pos_bytes; 3119 3120 sem_init(&camdd_sem, /*pshared*/ 0, 0); 3121 3122 signal(SIGINFO, camdd_sig_handler); 3123 signal(SIGINT, camdd_sig_handler); 3124 3125 error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &start_time); 3126 if (error != 0) { 3127 warn("Unable to get start time"); 3128 goto bailout; 3129 } 3130 3131 for (i = 0; i < num_io_opts; i++) { 3132 error = pthread_create(&threads[i], NULL, camdd_worker, 3133 (void *)devs[i]); 3134 if (error != 0) { 3135 warnc(error, "pthread_create() failed"); 3136 goto bailout; 3137 } 3138 } 3139 3140 for (;;) { 3141 if ((sem_wait(&camdd_sem) == -1) 3142 || (need_exit != 0)) { 3143 struct kevent ke; 3144 3145 for (i = 0; i < num_io_opts; i++) { 3146 EV_SET(&ke, (uintptr_t)&devs[i]->work_queue, 3147 EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL); 3148 3149 devs[i]->flags |= CAMDD_DEV_FLAG_EOF; 3150 3151 error = kevent(devs[i]->kq, &ke, 1, NULL, 0, 3152 NULL); 3153 if (error == -1) 3154 warn("%s: unable to wake up thread", 3155 __func__); 3156 error = 0; 3157 } 3158 break; 3159 } else if (need_status != 0) { 3160 camdd_print_status(devs[0], devs[1], &start_time); 3161 need_status = 0; 3162 } 3163 } 3164 for (i = 0; i < num_io_opts; i++) { 3165 pthread_join(threads[i], NULL); 3166 } 3167 3168 camdd_print_status(devs[0], devs[1], &start_time); 3169 3170bailout: 3171 3172 for (i = 0; i < num_io_opts; i++) 3173 camdd_free_dev(devs[i]); 3174 3175 return (error + error_exit); 3176} 3177 3178void 3179usage(void) 3180{ 3181 fprintf(stderr, 3182"usage: camdd <-i|-o pass=pass0,bs=1M,offset=1M,depth=4>\n" 3183" <-i|-o file=/tmp/file,bs=512K,offset=1M>\n" 3184" <-i|-o file=/dev/da0,bs=512K,offset=1M>\n" 3185" <-i|-o file=/dev/nsa0,bs=512K>\n" 3186" [-C retry_count][-E][-m max_io_amt][-t timeout_secs][-v][-h]\n" 3187"Option description\n" 3188"-i <arg=val> Specify input device/file and parameters\n" 3189"-o <arg=val> Specify output device/file and parameters\n" 3190"Input and Output parameters\n" 3191"pass=name Specify a pass(4) device like pass0 or /dev/pass0\n" 3192"file=name Specify a file or device, /tmp/foo, /dev/da0, /dev/null\n" 3193" or - for stdin/stdout\n" 3194"bs=blocksize Specify blocksize in bytes, or using K, M, G, etc. suffix\n" 3195"offset=len Specify starting offset in bytes or using K, M, G suffix\n" 3196" NOTE: offset cannot be specified on tapes, pipes, stdin/out\n" 3197"depth=N Specify a numeric queue depth. This only applies to pass(4)\n" 3198"mcs=N Specify a minimum cmd size for pass(4) read/write commands\n" 3199"Optional arguments\n" 3200"-C retry_cnt Specify a retry count for pass(4) devices\n" 3201"-E Enable CAM error recovery for pass(4) devices\n" 3202"-m max_io Specify the maximum amount to be transferred in bytes or\n" 3203" using K, G, M, etc. suffixes\n" 3204"-t timeout Specify the I/O timeout to use with pass(4) devices\n" 3205"-v Enable verbose error recovery\n" 3206"-h Print this message\n"); 3207} 3208 3209 3210int 3211camdd_parse_io_opts(char *args, int is_write, struct camdd_io_opts *io_opts) 3212{ 3213 char *tmpstr, *tmpstr2; 3214 char *orig_tmpstr = NULL; 3215 int retval = 0; 3216 3217 io_opts->write_dev = is_write; 3218 3219 tmpstr = strdup(args); 3220 if (tmpstr == NULL) { 3221 warn("strdup failed"); 3222 retval = 1; 3223 goto bailout; 3224 } 3225 orig_tmpstr = tmpstr; 3226 while ((tmpstr2 = strsep(&tmpstr, ",")) != NULL) { 3227 char *name, *value; 3228 3229 /* 3230 * If the user creates an empty parameter by putting in two 3231 * commas, skip over it and look for the next field. 3232 */ 3233 if (*tmpstr2 == '\0') 3234 continue; 3235 3236 name = strsep(&tmpstr2, "="); 3237 if (*name == '\0') { 3238 warnx("Got empty I/O parameter name"); 3239 retval = 1; 3240 goto bailout; 3241 } 3242 value = strsep(&tmpstr2, "="); 3243 if ((value == NULL) 3244 || (*value == '\0')) { 3245 warnx("Empty I/O parameter value for %s", name); 3246 retval = 1; 3247 goto bailout; 3248 } 3249 if (strncasecmp(name, "file", 4) == 0) { 3250 io_opts->dev_type = CAMDD_DEV_FILE; 3251 io_opts->dev_name = strdup(value); 3252 if (io_opts->dev_name == NULL) { 3253 warn("Error allocating memory"); 3254 retval = 1; 3255 goto bailout; 3256 } 3257 } else if (strncasecmp(name, "pass", 4) == 0) { 3258 io_opts->dev_type = CAMDD_DEV_PASS; 3259 io_opts->dev_name = strdup(value); 3260 if (io_opts->dev_name == NULL) { 3261 warn("Error allocating memory"); 3262 retval = 1; 3263 goto bailout; 3264 } 3265 } else if ((strncasecmp(name, "bs", 2) == 0) 3266 || (strncasecmp(name, "blocksize", 9) == 0)) { 3267 retval = expand_number(value, &io_opts->blocksize); 3268 if (retval == -1) { 3269 warn("expand_number(3) failed on %s=%s", name, 3270 value); 3271 retval = 1; 3272 goto bailout; 3273 } 3274 } else if (strncasecmp(name, "depth", 5) == 0) { 3275 char *endptr; 3276 3277 io_opts->queue_depth = strtoull(value, &endptr, 0); 3278 if (*endptr != '\0') { 3279 warnx("invalid queue depth %s", value); 3280 retval = 1; 3281 goto bailout; 3282 } 3283 } else if (strncasecmp(name, "mcs", 3) == 0) { 3284 char *endptr; 3285 3286 io_opts->min_cmd_size = strtol(value, &endptr, 0); 3287 if ((*endptr != '\0') 3288 || ((io_opts->min_cmd_size > 16) 3289 || (io_opts->min_cmd_size < 0))) { 3290 warnx("invalid minimum cmd size %s", value); 3291 retval = 1; 3292 goto bailout; 3293 } 3294 } else if (strncasecmp(name, "offset", 6) == 0) { 3295 retval = expand_number(value, &io_opts->offset); 3296 if (retval == -1) { 3297 warn("expand_number(3) failed on %s=%s", name, 3298 value); 3299 retval = 1; 3300 goto bailout; 3301 } 3302 } else if (strncasecmp(name, "debug", 5) == 0) { 3303 char *endptr; 3304 3305 io_opts->debug = strtoull(value, &endptr, 0); 3306 if (*endptr != '\0') { 3307 warnx("invalid debug level %s", value); 3308 retval = 1; 3309 goto bailout; 3310 } 3311 } else { 3312 warnx("Unrecognized parameter %s=%s", name, value); 3313 } 3314 } 3315bailout: 3316 free(orig_tmpstr); 3317 3318 return (retval); 3319} 3320 3321int 3322main(int argc, char **argv) 3323{ 3324 int c; 3325 camdd_argmask arglist = CAMDD_ARG_NONE; 3326 int timeout = 0, retry_count = 1; 3327 int error = 0; 3328 uint64_t max_io = 0; 3329 struct camdd_io_opts *opt_list = NULL; 3330 3331 if (argc == 1) { 3332 usage(); 3333 exit(1); 3334 } 3335 3336 opt_list = calloc(2, sizeof(struct camdd_io_opts)); 3337 if (opt_list == NULL) { 3338 warn("Unable to allocate option list"); 3339 error = 1; 3340 goto bailout; 3341 } 3342 3343 while ((c = getopt(argc, argv, "C:Ehi:m:o:t:v")) != -1){ 3344 switch (c) { 3345 case 'C': 3346 retry_count = strtol(optarg, NULL, 0); 3347 if (retry_count < 0) 3348 errx(1, "retry count %d is < 0", 3349 retry_count); 3350 arglist |= CAMDD_ARG_RETRIES; 3351 break; 3352 case 'E': 3353 arglist |= CAMDD_ARG_ERR_RECOVER; 3354 break; 3355 case 'i': 3356 case 'o': 3357 if (((c == 'i') 3358 && (opt_list[0].dev_type != CAMDD_DEV_NONE)) 3359 || ((c == 'o') 3360 && (opt_list[1].dev_type != CAMDD_DEV_NONE))) { 3361 errx(1, "Only one input and output path " 3362 "allowed"); 3363 } 3364 error = camdd_parse_io_opts(optarg, (c == 'o') ? 1 : 0, 3365 (c == 'o') ? &opt_list[1] : &opt_list[0]); 3366 if (error != 0) 3367 goto bailout; 3368 break; 3369 case 'm': 3370 error = expand_number(optarg, &max_io); 3371 if (error == -1) { 3372 warn("invalid maximum I/O amount %s", optarg); 3373 error = 1; 3374 goto bailout; 3375 } 3376 break; 3377 case 't': 3378 timeout = strtol(optarg, NULL, 0); 3379 if (timeout < 0) 3380 errx(1, "invalid timeout %d", timeout); 3381 /* Convert the timeout from seconds to ms */ 3382 timeout *= 1000; 3383 arglist |= CAMDD_ARG_TIMEOUT; 3384 break; 3385 case 'v': 3386 arglist |= CAMDD_ARG_VERBOSE; 3387 break; 3388 case 'h': 3389 default: 3390 usage(); 3391 exit(1); 3392 break; /*NOTREACHED*/ 3393 } 3394 } 3395 3396 if ((opt_list[0].dev_type == CAMDD_DEV_NONE) 3397 || (opt_list[1].dev_type == CAMDD_DEV_NONE)) 3398 errx(1, "Must specify both -i and -o"); 3399 3400 /* 3401 * Set the timeout if the user hasn't specified one. 3402 */ 3403 if (timeout == 0) 3404 timeout = CAMDD_PASS_RW_TIMEOUT; 3405 3406 error = camdd_rw(opt_list, 2, max_io, retry_count, timeout); 3407 3408bailout: 3409 free(opt_list); 3410 3411 exit(error); 3412} 3413