Lines Matching defs:io
443 static int ctl_do_mode_select(union ctl_io *io);
452 static void ctl_hndl_per_res_out_on_other_sc(union ctl_io *io);
466 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len);
478 static void ctl_failover_lun(union ctl_io *io);
483 static int ctl_target_reset(union ctl_io *io);
486 static int ctl_lun_reset(union ctl_io *io);
487 static int ctl_abort_task(union ctl_io *io);
488 static int ctl_abort_task_set(union ctl_io *io);
489 static int ctl_query_task(union ctl_io *io, int task_set);
492 static int ctl_i_t_nexus_reset(union ctl_io *io);
493 static int ctl_query_async_event(union ctl_io *io);
494 static void ctl_run_task(union ctl_io *io);
500 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
502 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
503 static void ctl_datamove_remote_write(union ctl_io *io);
504 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
506 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
507 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
509 static void ctl_datamove_remote_read(union ctl_io *io);
510 static void ctl_datamove_remote(union ctl_io *io);
511 static void ctl_process_done(union ctl_io *io);
515 static void ctl_enqueue_incoming(union ctl_io *io);
516 static void ctl_enqueue_rtr(union ctl_io *io);
517 static void ctl_enqueue_done(union ctl_io *io);
518 static void ctl_enqueue_isc(union ctl_io *io);
614 ctl_ha_datamove(union ctl_io *io)
616 struct ctl_lun *lun = CTL_LUN(io);
624 msg.hdr.original_sc = io->io_hdr.original_sc;
625 msg.hdr.serializing_sc = io;
626 msg.hdr.nexus = io->io_hdr.nexus;
627 msg.hdr.status = io->io_hdr.status;
628 msg.dt.flags = io->io_hdr.flags;
638 if (io->scsiio.kern_sg_entries == 0) {
641 if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
642 msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
646 (void *)vtophys(io->scsiio.kern_data_ptr);
649 KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
651 msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
653 msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
656 msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
660 msg.dt.kern_data_len = io->scsiio.kern_data_len;
661 msg.dt.kern_total_len = io->scsiio.kern_total_len;
662 msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
663 msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
677 sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
681 if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
689 KASSERT((io->io_hdr.flags &
704 io->io_hdr.port_status = 31341;
705 io->scsiio.be_move_done(io);
718 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
721 io->io_hdr.port_status = 31342;
722 io->scsiio.be_move_done(io);
725 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
726 io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
732 ctl_ha_done(union ctl_io *io)
736 if (io->io_hdr.io_type == CTL_IO_SCSI) {
739 msg.hdr.original_sc = io->io_hdr.original_sc;
740 msg.hdr.nexus = io->io_hdr.nexus;
741 msg.hdr.status = io->io_hdr.status;
742 msg.scsi.scsi_status = io->scsiio.scsi_status;
743 msg.scsi.tag_num = io->scsiio.tag_num;
744 msg.scsi.tag_type = io->scsiio.tag_type;
745 msg.scsi.sense_len = io->scsiio.sense_len;
746 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
747 io->scsiio.sense_len);
752 ctl_free_io(io);
1029 union ctl_io *io;
1042 io = ctl_alloc_io(softc->othersc_pool);
1044 ctl_zero_io(io);
1045 io->io_hdr.msg_type = CTL_MSG_FAILOVER;
1046 io->io_hdr.nexus.targ_mapped_lun = lun->lun;
1047 ctl_enqueue_isc(io);
1382 union ctl_io *io;
1407 io = ctl_alloc_io(softc->othersc_pool);
1408 ctl_zero_io(io);
1410 io->io_hdr.io_type = CTL_IO_SCSI;
1411 io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
1412 io->io_hdr.original_sc = msg->hdr.original_sc;
1413 io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
1423 io->io_hdr.flags |= CTL_FLAG_INT_COPY;
1424 io->io_hdr.nexus = msg->hdr.nexus;
1427 io->io_hdr.nexus.targ_port,
1428 io->io_hdr.nexus.initid,
1429 io->io_hdr.nexus.targ_lun);
1431 io->scsiio.tag_num = msg->scsi.tag_num;
1432 io->scsiio.tag_type = msg->scsi.tag_type;
1434 io->io_hdr.start_time = time_uptime;
1435 getbinuptime(&io->io_hdr.start_bt);
1437 io->scsiio.cdb_len = msg->scsi.cdb_len;
1438 memcpy(io->scsiio.cdb, msg->scsi.cdb,
1443 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
1444 io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
1445 io->io_hdr.flags |=
1448 ctl_enqueue_isc(io);
1456 io = msg->hdr.original_sc;
1457 if (io == NULL) {
1462 io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
1463 io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1468 io->io_hdr.serializing_sc = msg->hdr.serializing_sc;
1470 io->io_hdr.status = msg->hdr.status;
1474 getbinuptime(&io->io_hdr.dma_start_bt);
1481 io->io_hdr.remote_sglist = sgl;
1482 io->io_hdr.local_sglist =
1485 io->scsiio.kern_data_ptr = (uint8_t *)sgl;
1487 io->scsiio.kern_sg_entries =
1489 io->scsiio.rem_sg_entries =
1491 io->scsiio.kern_data_len =
1493 io->scsiio.kern_total_len =
1495 io->scsiio.kern_data_resid =
1497 io->scsiio.kern_rel_offset =
1499 io->io_hdr.flags &= ~CTL_FLAG_BUS_ADDR;
1500 io->io_hdr.flags |= msg->dt.flags &
1504 io->scsiio.kern_data_ptr;
1524 ctl_enqueue_isc(io);
1540 io = msg->hdr.serializing_sc;
1541 io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
1542 io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1543 io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1544 io->io_hdr.port_status = msg->scsi.port_status;
1545 io->scsiio.kern_data_resid = msg->scsi.kern_data_resid;
1547 io->io_hdr.status = msg->hdr.status;
1548 io->scsiio.scsi_status = msg->scsi.scsi_status;
1549 io->scsiio.sense_len = msg->scsi.sense_len;
1550 memcpy(&io->scsiio.sense_data,
1554 io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
1556 ctl_enqueue_isc(io);
1562 io = msg->hdr.original_sc;
1563 if (io == NULL) {
1568 io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1569 io->io_hdr.msg_type = CTL_MSG_R2R;
1570 io->io_hdr.serializing_sc = msg->hdr.serializing_sc;
1571 ctl_enqueue_isc(io);
1589 io = msg->hdr.original_sc;
1590 if (io == NULL) {
1595 ctl_copy_sense_data(msg, io);
1601 io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
1602 io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
1604 /* io = msg->hdr.serializing_sc; */
1605 io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
1606 ctl_enqueue_isc(io);
2361 union ctl_io *io;
2364 for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2365 (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2378 entry->tag_num = io->scsiio.tag_num;
2381 entry->start_bt = io->io_hdr.start_bt;
2383 bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2384 entry->cdb_len = io->scsiio.cdb_len;
2385 if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2388 if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2391 if (io->io_hdr.flags & CTL_FLAG_ABORT)
2394 if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2397 if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
3861 union ctl_io *io;
3863 io = uma_zalloc(pool->zone, M_WAITOK);
3864 if (io != NULL) {
3865 io->io_hdr.pool = pool_ref;
3866 CTL_SOFTC(io) = pool->ctl_softc;
3868 return (io);
3875 union ctl_io *io;
3877 io = uma_zalloc(pool->zone, M_NOWAIT);
3878 if (io != NULL) {
3879 io->io_hdr.pool = pool_ref;
3880 CTL_SOFTC(io) = pool->ctl_softc;
3882 return (io);
3886 ctl_free_io(union ctl_io *io)
3890 if (io == NULL)
3893 pool = (struct ctl_io_pool *)io->io_hdr.pool;
3894 uma_zfree(pool->zone, io);
3898 ctl_zero_io(union ctl_io *io)
3902 if (io == NULL)
3908 pool = io->io_hdr.pool;
3909 memset(io, 0, sizeof(*io));
3910 io->io_hdr.pool = pool;
3911 CTL_SOFTC(io) = pool->ctl_softc;
5060 ctl_config_move_done(union ctl_io *io)
5065 KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
5066 ("Config I/O type isn't CTL_IO_SCSI (%d)!", io->io_hdr.io_type));
5068 if ((io->io_hdr.port_status != 0) &&
5069 ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5070 (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5071 ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
5072 /*retry_count*/ io->io_hdr.port_status);
5073 } else if (io->scsiio.kern_data_resid != 0 &&
5074 (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
5075 ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5076 (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5077 ctl_set_invalid_field_ciu(&io->scsiio);
5081 ctl_data_print(io);
5082 if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) ||
5083 ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
5084 (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) ||
5085 ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
5091 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5092 free(io->scsiio.kern_data_ptr, M_CTL);
5093 ctl_done(io);
5113 retval = ctl_scsiio(&io->scsiio);
5123 ctl_data_submit_done(union ctl_io *io)
5133 if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5134 (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5135 ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5136 (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5137 io->scsiio.io_cont(io);
5140 ctl_done(io);
5148 ctl_config_write_done(union ctl_io *io)
5160 if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5161 (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5162 ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5163 (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5164 io->scsiio.io_cont(io);
5172 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5173 buf = io->scsiio.kern_data_ptr;
5176 ctl_done(io);
5182 ctl_config_read_done(union ctl_io *io)
5189 if ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0 ||
5190 ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
5191 (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
5192 if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5193 buf = io->scsiio.kern_data_ptr;
5196 ctl_done(io);
5207 if (io->io_hdr.flags & CTL_FLAG_IO_CONT) {
5208 io->scsiio.io_cont(io);
5212 ctl_datamove(io);
5968 ctl_do_mode_select(union ctl_io *io)
5970 struct ctl_lun *lun = CTL_LUN(io);
5979 ctsio = &io->scsiio;
6170 * completed the io for us.
8471 ctl_hndl_per_res_out_on_other_sc(union ctl_io *io)
8473 struct ctl_softc *softc = CTL_SOFTC(io);
8474 union ctl_ha_msg *msg = (union ctl_ha_msg *)&io->presio.pr_msg;
8782 ctl_cnw_cont(union ctl_io *io)
8784 struct ctl_lun *lun = CTL_LUN(io);
8789 ctsio = &io->scsiio;
10507 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len)
10509 if (io->io_hdr.io_type != CTL_IO_SCSI)
10512 switch (io->scsiio.cdb[0]) {
10516 cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10526 cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10538 cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10547 cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10557 cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10566 cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10576 cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10585 cdb = (struct scsi_write_atomic_16 *)io->scsiio.cdb;
10594 cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10603 cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10612 cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10621 cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10630 cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10639 cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10653 cdb = (struct scsi_get_lba_status *)io->scsiio.cdb;
10682 ctl_extent_check_unmap(union ctl_io *io, uint64_t lba2, uint64_t len2)
10690 if (io->io_hdr.io_type != CTL_IO_SCSI ||
10691 io->scsiio.cdb[0] != UNMAP)
10696 &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
10697 if ((io->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0 ||
11176 ctl_failover_io(union ctl_io *io, int have_lock)
11178 ctl_set_busy(&io->scsiio);
11179 ctl_done(io);
11187 struct ctl_io_hdr *io, *next_io;
11208 TAILQ_FOREACH_SAFE(io, &lun->ooa_queue, ooa_links, next_io) {
11210 if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11211 if (io->flags & CTL_FLAG_IO_ACTIVE) {
11212 io->flags |= CTL_FLAG_ABORT;
11213 io->flags |= CTL_FLAG_FAILOVER;
11215 io->msg_type = CTL_MSG_DATAMOVE_DONE;
11216 io->flags &= ~CTL_FLAG_DMA_INPROG;
11217 io->flags |= CTL_FLAG_IO_ACTIVE;
11218 io->port_status = 31340;
11219 ctl_enqueue_isc((union ctl_io *)io);
11223 if (io->flags & CTL_FLAG_SENT_2OTHER_SC) {
11224 io->flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11225 if (io->flags & CTL_FLAG_IO_ACTIVE) {
11226 io->flags |= CTL_FLAG_FAILOVER;
11228 ctl_set_busy(&((union ctl_io *)io)->
11230 ctl_done((union ctl_io *)io);
11235 TAILQ_FOREACH_SAFE(io, &lun->blocked_queue, blocked_links,
11238 if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11239 TAILQ_REMOVE(&lun->blocked_queue, io,
11241 io->flags &= ~CTL_FLAG_BLOCKED;
11242 TAILQ_REMOVE(&lun->ooa_queue, io, ooa_links);
11243 ctl_free_io((union ctl_io *)io);
11246 TAILQ_FOREACH_SAFE(io, &lun->ooa_queue, ooa_links, next_io) {
11248 if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
11249 TAILQ_REMOVE(&lun->ooa_queue, io, ooa_links);
11250 ctl_free_io((union ctl_io *)io);
11253 if (io->flags & CTL_FLAG_SENT_2OTHER_SC) {
11254 io->flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11255 if (!(io->flags & CTL_FLAG_IO_ACTIVE)) {
11256 ctl_set_busy(&((union ctl_io *)io)->
11258 ctl_done((union ctl_io *)io);
11596 ctl_target_reset(union ctl_io *io)
11598 struct ctl_softc *softc = CTL_SOFTC(io);
11599 struct ctl_port *port = CTL_PORT(io);
11604 if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11607 msg_info.hdr.nexus = io->io_hdr.nexus;
11608 msg_info.task.task_action = io->taskio.task_action;
11616 initidx = ctl_get_initindex(&io->io_hdr.nexus);
11617 if (io->taskio.task_action == CTL_TASK_TARGET_RESET)
11629 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11691 ctl_lun_reset(union ctl_io *io)
11693 struct ctl_softc *softc = CTL_SOFTC(io);
11697 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11698 initidx = ctl_get_initindex(&io->io_hdr.nexus);
11703 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11708 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11710 if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0) {
11714 msg_info.hdr.nexus = io->io_hdr.nexus;
11768 ctl_abort_task_set(union ctl_io *io)
11770 struct ctl_softc *softc = CTL_SOFTC(io);
11777 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11782 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11788 if (io->taskio.task_action == CTL_TASK_ABORT_TASK_SET) {
11789 ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
11790 io->io_hdr.nexus.initid,
11791 (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11794 (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
11797 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11838 ctl_i_t_nexus_reset(union ctl_io *io)
11840 struct ctl_softc *softc = CTL_SOFTC(io);
11843 if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11846 msg_info.hdr.nexus = io->io_hdr.nexus;
11855 initidx = ctl_get_initindex(&io->io_hdr.nexus);
11857 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11862 ctl_abort_task(union ctl_io *io)
11864 struct ctl_softc *softc = CTL_SOFTC(io);
11879 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11884 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
11890 lun->lun, io->taskio.tag_num, io->taskio.tag_type);
11923 if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
11924 || (xio->io_hdr.nexus.initid != io->io_hdr.nexus.initid)
11938 && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
11939 || (xio->scsiio.tag_num == io->taskio.tag_num))
11948 if (xio->scsiio.tag_num == io->taskio.tag_num) {
11951 if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0 &&
11955 msg_info.hdr.nexus = io->io_hdr.nexus;
11957 msg_info.task.tag_num = io->taskio.tag_num;
11958 msg_info.task.tag_type = io->taskio.tag_type;
11984 io->io_hdr.nexus.initid,
11985 io->io_hdr.nexus.targ_port,
11986 io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
11987 io->taskio.tag_type);
11990 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
11995 ctl_query_task(union ctl_io *io, int task_set)
11997 struct ctl_softc *softc = CTL_SOFTC(io);
12003 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12008 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
12016 if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
12017 || (xio->io_hdr.nexus.initid != io->io_hdr.nexus.initid)
12021 if (task_set || xio->scsiio.tag_num == io->taskio.tag_num) {
12028 io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
12030 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
12035 ctl_query_async_event(union ctl_io *io)
12037 struct ctl_softc *softc = CTL_SOFTC(io);
12042 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12047 io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
12052 initidx = ctl_get_initindex(&io->io_hdr.nexus);
12053 ua = ctl_build_qae(lun, initidx, io->taskio.task_resp);
12056 io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
12058 io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
12063 ctl_run_task(union ctl_io *io)
12068 KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
12069 ("ctl_run_task: Unextected io_type %d\n", io->io_hdr.io_type));
12070 io->taskio.task_status = CTL_TASK_FUNCTION_NOT_SUPPORTED;
12071 bzero(io->taskio.task_resp, sizeof(io->taskio.task_resp));
12072 switch (io->taskio.task_action) {
12074 retval = ctl_abort_task(io);
12078 retval = ctl_abort_task_set(io);
12083 retval = ctl_i_t_nexus_reset(io);
12086 retval = ctl_lun_reset(io);
12090 retval = ctl_target_reset(io);
12097 retval = ctl_query_task(io, 0);
12100 retval = ctl_query_task(io, 1);
12103 retval = ctl_query_async_event(io);
12107 __func__, io->taskio.task_action);
12111 io->io_hdr.status = CTL_SUCCESS;
12113 io->io_hdr.status = CTL_ERROR;
12114 ctl_done(io);
12122 ctl_handle_isc(union ctl_io *io)
12124 struct ctl_softc *softc = CTL_SOFTC(io);
12129 targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12130 switch (io->io_hdr.msg_type) {
12132 ctl_serialize_other_sc_cmd(&io->scsiio);
12135 entry = ctl_get_cmd_entry(&io->scsiio, NULL);
12138 ctl_done(io);
12142 if (ctl_scsiio_lun_check(lun, entry, &io->scsiio) != 0) {
12144 ctl_done(io);
12147 io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
12149 ctl_enqueue_rtr(io);
12153 ctl_done(io);
12158 ctl_free_io(io);
12162 TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
12165 ctl_free_io(io);
12168 ctl_hndl_per_res_out_on_other_sc(io);
12169 ctl_free_io(io);
12172 ctl_done(io);
12175 ctl_datamove_remote(io);
12178 io->scsiio.be_move_done(io);
12181 ctl_failover_lun(io);
12182 ctl_free_io(io);
12186 __func__, io->io_hdr.msg_type);
12187 ctl_free_io(io);
12261 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
12273 pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
12279 ctl_set_aborted(&io->scsiio);
12282 ctl_set_medium_error(&io->scsiio,
12283 (io->io_hdr.flags & CTL_FLAG_DATA_MASK) !=
12289 ctl_set_ua(&io->scsiio, 0x29, 0x00);
12297 bcopy(&desc->custom_sense, &io->scsiio.sense_data,
12299 sizeof(io->scsiio.sense_data)));
12300 io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
12301 io->scsiio.sense_len = SSD_FULL_SIZE;
12302 io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
12330 union ctl_io *io;
12332 io = (union ctl_io *)arg;
12334 ctl_datamove(io);
12339 ctl_datamove(union ctl_io *io)
12341 void (*fe_datamove)(union ctl_io *io);
12343 mtx_assert(&((struct ctl_softc *)CTL_SOFTC(io))->ctl_lock, MA_NOTOWNED);
12348 io->scsiio.kern_data_resid = io->scsiio.kern_data_len;
12351 if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12356 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12360 switch (io->io_hdr.io_type) {
12362 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12366 io->scsiio.tag_num, io->scsiio.tag_type);
12370 "Tag Type: %d\n", io->taskio.task_action,
12371 io->taskio.tag_num, io->taskio.tag_type);
12375 __func__, io->io_hdr.io_type);
12379 (intmax_t)time_uptime - io->io_hdr.start_time);
12386 if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12387 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12391 lun = CTL_LUN(io);
12395 callout_init(&io->io_hdr.delay_callout, /*mpsafe*/ 1);
12396 io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12397 callout_reset(&io->io_hdr.delay_callout,
12399 ctl_datamove_timer_wakeup, io);
12412 if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12414 io->scsiio.tag_num, io->io_hdr.nexus.initid,
12415 io->io_hdr.nexus.targ_port,
12416 io->io_hdr.nexus.targ_lun);
12417 io->io_hdr.port_status = 31337;
12423 io->scsiio.be_move_done(io);
12428 if (io->scsiio.kern_data_len == 0) {
12429 io->scsiio.be_move_done(io);
12433 fe_datamove = CTL_PORT(io)->fe_datamove;
12434 fe_datamove(io);
12438 ctl_send_datamove_done(union ctl_io *io, int have_lock)
12447 msg.hdr.original_sc = io;
12448 msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
12449 msg.hdr.nexus = io->io_hdr.nexus;
12450 msg.hdr.status = io->io_hdr.status;
12451 msg.scsi.kern_data_resid = io->scsiio.kern_data_resid;
12452 msg.scsi.tag_num = io->scsiio.tag_num;
12453 msg.scsi.tag_type = io->scsiio.tag_type;
12454 msg.scsi.scsi_status = io->scsiio.scsi_status;
12455 memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12456 io->scsiio.sense_len);
12457 msg.scsi.sense_len = io->scsiio.sense_len;
12458 msg.scsi.port_status = io->io_hdr.port_status;
12459 io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12460 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12461 ctl_failover_io(io, /*have_lock*/ have_lock);
12470 bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
12471 bintime_add(&io->io_hdr.dma_bt, &cur_bt);
12473 io->io_hdr.num_dmas++;
12483 union ctl_io *io;
12486 io = rq->context;
12491 ctl_set_internal_failure(&io->scsiio,
12498 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12499 free(io->io_hdr.local_sglist[i].addr, M_CTL);
12500 free(io->io_hdr.remote_sglist, M_CTL);
12501 io->io_hdr.remote_sglist = NULL;
12502 io->io_hdr.local_sglist = NULL;
12508 ctl_send_datamove_done(io, /*have_lock*/ 0);
12516 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
12520 retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
12526 ctl_datamove_remote_write(union ctl_io *io)
12529 void (*fe_datamove)(union ctl_io *io);
12537 retval = ctl_datamove_remote_sgl_setup(io);
12542 io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12548 io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
12550 fe_datamove = CTL_PORT(io)->fe_datamove;
12551 fe_datamove(io);
12555 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
12564 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12565 free(io->io_hdr.local_sglist[i].addr, M_CTL);
12566 free(io->io_hdr.remote_sglist, M_CTL);
12567 io->io_hdr.remote_sglist = NULL;
12568 io->io_hdr.local_sglist = NULL;
12571 scsi_path_string(io, path_str, sizeof(path_str));
12574 scsi_command_string(&io->scsiio, NULL, &sb);
12578 io->scsiio.tag_num, io->scsiio.tag_type);
12581 io->io_hdr.flags, io->io_hdr.status);
12591 ctl_send_datamove_done(io, /*have_lock*/ 0);
12599 union ctl_io *io;
12600 void (*fe_datamove)(union ctl_io *io);
12602 io = rq->context;
12607 ctl_set_internal_failure(&io->scsiio,
12615 io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12621 io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
12625 fe_datamove = CTL_PORT(io)->fe_datamove;
12626 fe_datamove(io);
12630 ctl_datamove_remote_sgl_setup(union ctl_io *io)
12638 local_sglist = io->io_hdr.local_sglist;
12639 len_to_go = io->scsiio.kern_data_len;
12658 io->scsiio.kern_sg_entries = i;
12662 io->scsiio.kern_sg_entries);
12663 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12672 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
12688 && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
12689 (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS))
12690 ctl_set_busy(&io->scsiio);
12692 if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
12693 (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
12704 ctl_send_datamove_done(io, /*have_lock*/ 0);
12709 local_sglist = io->io_hdr.local_sglist;
12710 remote_sglist = io->io_hdr.remote_sglist;
12726 for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
12731 rq->context = io;
12748 if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12754 KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
12778 if (total_used >= io->scsiio.kern_data_len)
12800 ctl_datamove_remote_read(union ctl_io *io)
12809 retval = ctl_datamove_remote_sgl_setup(io);
12813 retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12822 for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12823 free(io->io_hdr.local_sglist[i].addr, M_CTL);
12824 free(io->io_hdr.remote_sglist, M_CTL);
12825 io->io_hdr.remote_sglist = NULL;
12826 io->io_hdr.local_sglist = NULL;
12838 ctl_datamove_remote(union ctl_io *io)
12841 mtx_assert(&((struct ctl_softc *)CTL_SOFTC(io))->ctl_lock, MA_NOTOWNED);
12843 if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12844 ctl_failover_io(io, /*have_lock*/ 0);
12854 if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12856 io->scsiio.tag_num, io->io_hdr.nexus.initid,
12857 io->io_hdr.nexus.targ_port,
12858 io->io_hdr.nexus.targ_lun);
12859 io->io_hdr.port_status = 31338;
12860 ctl_send_datamove_done(io, /*have_lock*/ 0);
12864 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
12865 ctl_datamove_remote_write(io);
12866 else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
12867 ctl_datamove_remote_read(io);
12869 io->io_hdr.port_status = 31339;
12870 ctl_send_datamove_done(io, /*have_lock*/ 0);
12875 ctl_process_done(union ctl_io *io)
12877 struct ctl_softc *softc = CTL_SOFTC(io);
12878 struct ctl_port *port = CTL_PORT(io);
12879 struct ctl_lun *lun = CTL_LUN(io);
12880 void (*fe_done)(union ctl_io *io);
12887 if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12892 ctl_scsi_path_string(io, path_str, sizeof(path_str));
12896 switch (io->io_hdr.io_type) {
12898 ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12902 io->scsiio.tag_num, io->scsiio.tag_type);
12906 "Tag Type: %d\n", io->taskio.task_action,
12907 io->taskio.tag_num, io->taskio.tag_type);
12911 __func__, io->io_hdr.io_type);
12915 (intmax_t)time_uptime - io->io_hdr.start_time);
12921 switch (io->io_hdr.io_type) {
12926 ctl_io_error_print(io, NULL);
12927 fe_done(io);
12931 __func__, io->io_hdr.io_type);
12936 io->io_hdr.nexus.targ_mapped_lun));
12948 io->io_hdr.status == CTL_SUCCESS &&
12949 (io->io_hdr.flags & CTL_FLAG_STATUS_SENT) == 0) {
12956 (ctl_get_cmd_entry(&io->scsiio, NULL)->flags &
12958 ctl_set_sense(&io->scsiio,
12975 ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) &&
12976 ((io->io_hdr.flags & CTL_FLAG_STATUS_SENT) == 0))
12977 ctl_inject_error(lun, io);
12985 if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
12986 io->io_hdr.io_type == CTL_IO_SCSI) {
12992 bintime_sub(&bt, &io->io_hdr.start_bt);
12994 if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
12997 else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13006 io->scsiio.kern_total_len;
13009 io->io_hdr.num_dmas;
13012 &io->io_hdr.dma_bt);
13018 lun->stats.bytes[type] += io->scsiio.kern_total_len;
13020 lun->stats.dmas[type] += io->io_hdr.num_dmas;
13022 bintime_add(&lun->stats.dma_time[type], &io->io_hdr.dma_bt);
13027 port->stats.bytes[type] += io->scsiio.kern_total_len;
13029 port->stats.dmas[type] += io->io_hdr.num_dmas;
13031 bintime_add(&port->stats.dma_time[type], &io->io_hdr.dma_bt);
13040 TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13072 if (io->io_hdr.flags & CTL_FLAG_ABORT)
13073 ctl_set_task_aborted(&io->scsiio);
13078 if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS &&
13080 ctl_io_error_print(io, NULL);
13088 (io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)) {
13091 msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
13092 msg.hdr.nexus = io->io_hdr.nexus;
13098 fe_done(io);
13106 ctl_queue_sense(union ctl_io *io)
13108 struct ctl_softc *softc = CTL_SOFTC(io);
13109 struct ctl_port *port = CTL_PORT(io);
13116 targ_lun = ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13135 initidx = ctl_get_initindex(&io->io_hdr.nexus);
13144 memcpy(ps, &io->scsiio.sense_data, io->scsiio.sense_len);
13149 ctl_free_io(io);
13158 ctl_queue(union ctl_io *io)
13160 struct ctl_port *port = CTL_PORT(io);
13162 CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13165 io->io_hdr.start_time = time_uptime;
13166 getbinuptime(&io->io_hdr.start_bt);
13170 io->io_hdr.nexus.targ_mapped_lun =
13171 ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13173 switch (io->io_hdr.io_type) {
13177 ctl_io_print(io);
13178 ctl_enqueue_incoming(io);
13181 printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13192 union ctl_io *io;
13194 io = (union ctl_io *)arg;
13195 ctl_done(io);
13200 ctl_serseq_done(union ctl_io *io)
13202 struct ctl_lun *lun = CTL_LUN(io);;
13208 io->io_hdr.flags |= CTL_FLAG_SERSEQ_DONE;
13214 ctl_done(union ctl_io *io)
13221 if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13226 io->io_hdr.io_type,
13227 io->io_hdr.msg_type,
13228 io->scsiio.cdb[0],
13229 io->io_hdr.nexus.initid,
13230 io->io_hdr.nexus.targ_port,
13231 io->io_hdr.nexus.targ_lun,
13232 (io->io_hdr.io_type ==
13234 io->taskio.tag_num :
13235 io->scsiio.tag_num,
13236 io->io_hdr.flags,
13237 io->io_hdr.status);
13239 io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13246 if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
13250 if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13251 io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13253 struct ctl_lun *lun = CTL_LUN(io);
13258 callout_init(&io->io_hdr.delay_callout, /*mpsafe*/ 1);
13259 io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
13260 callout_reset(&io->io_hdr.delay_callout,
13262 ctl_done_timer_wakeup, io);
13270 ctl_enqueue_done(io);
13278 union ctl_io *io;
13295 io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
13296 if (io != NULL) {
13299 ctl_handle_isc(io);
13302 io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
13303 if (io != NULL) {
13307 ctl_process_done(io);
13310 io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
13311 if (io != NULL) {
13314 if (io->io_hdr.io_type == CTL_IO_TASK)
13315 ctl_run_task(io);
13317 ctl_scsiio_precheck(softc, &io->scsiio);
13320 io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
13321 if (io != NULL) {
13324 retval = ctl_scsiio(&io->scsiio);
13469 ctl_enqueue_incoming(union ctl_io *io)
13471 struct ctl_softc *softc = CTL_SOFTC(io);
13475 idx = (io->io_hdr.nexus.targ_port * 127 +
13476 io->io_hdr.nexus.initid) % worker_threads;
13479 STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
13485 ctl_enqueue_rtr(union ctl_io *io)
13487 struct ctl_softc *softc = CTL_SOFTC(io);
13490 thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13492 STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
13498 ctl_enqueue_done(union ctl_io *io)
13500 struct ctl_softc *softc = CTL_SOFTC(io);
13503 thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13505 STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
13511 ctl_enqueue_isc(union ctl_io *io)
13513 struct ctl_softc *softc = CTL_SOFTC(io);
13516 thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13518 STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);